#!/usr/bin/env python3
"""Bulk frame analysis: classify, compare, hunt structure and checksums."""
import json, sys, collections, math
import numpy as np


def to_bytes(bits):
    n = len(bits) // 8 * 8
    return bytes(int(bits[i:i + 8], 2) for i in range(0, n, 8))


def entropy(data):
    if not data:
        return 0.0
    c = collections.Counter(data)
    n = len(data)
    return -sum(v / n * math.log2(v / n) for v in c.values())


def crc8(d, poly, init, refin, refout, xorout):
    crc = init
    for byte in d:
        if refin:
            byte = int(f"{byte:08b}"[::-1], 2)
        crc ^= byte
        for _ in range(8):
            crc = ((crc << 1) ^ poly) & 0xFF if crc & 0x80 else (crc << 1) & 0xFF
    if refout:
        crc = int(f"{crc:08b}"[::-1], 2)
    return crc ^ xorout


def crc16(d, poly, init, refin, refout=True):
    crc = init
    for byte in d:
        if refin:
            byte = int(f"{byte:08b}"[::-1], 2)
        crc ^= byte << 8
        for _ in range(8):
            crc = ((crc << 1) ^ poly) & 0xFFFF if crc & 0x8000 else (crc << 1) & 0xFFFF
    if refout:
        crc = int(f"{crc:016b}"[::-1], 2)
    return crc


def main(path):
    rows = [json.loads(l) for l in open(path)]
    frames = [(r, to_bytes(r["payload"])) for r in rows]
    frames = [(r, b) for r, b in frames if len(b) >= 7]
    print(f"{len(rows)} frames decoded, {len(frames)} byte-aligned of >=7 bytes\n")

    beacons = [(r, b) for r, b in frames if len(b) == 7]
    data = [(r, b) for r, b in frames if len(b) > 20]
    other = [(r, b) for r, b in frames if 7 < len(b) <= 20]
    print(f"beacon (7 B): {len(beacons)}   data (>20 B): {len(data)}   short/partial: {len(other)}")
    if beacons:
        c = collections.Counter(b.hex(" ") for _, b in beacons)
        print("beacon values:", [(k, v) for k, v in c.most_common(3)])
    if not data:
        print("no data frames yet")
        return

    print(f"\ndata frame lengths: {sorted(collections.Counter(len(b) for _, b in data).items())}")

    # header fields
    hdr = collections.Counter(b[:7].hex(" ") for _, b in data)
    print(f"\nfirst 7 bytes: {len(hdr)} distinct")
    for k, v in hdr.most_common(6):
        print(f"   x{v:3d}  {k}")
    b4 = collections.Counter(b[4] for _, b in data)
    print(f"\nbyte 4 (XX) values: {len(b4)} distinct, most common {[(hex(k), v) for k, v in b4.most_common(8)]}")

    # does XX track the length?
    by_len = collections.defaultdict(list)
    for _, b in data:
        by_len[len(b)].append(b[4])
    print("XX vs frame length:")
    for L in sorted(by_len):
        vals = by_len[L]
        print(f"   len {L:3d}: n={len(vals):3d}  XX range 0x{min(vals):02X}-0x{max(vals):02X}  distinct {len(set(vals))}")

    # constant bit positions inside the data region, aligned at payload start
    L = min(len(b) for _, b in data)
    M = np.array([[int(c) for c in r["payload"][:L * 8]] for r, b in data[:400]])
    const = np.where(M.std(axis=0) == 0)[0]
    print(f"\nconstant bit positions across {M.shape[0]} data frames: {len(const)} of {M.shape[1]}")
    if len(const):
        print(f"   first: {const[:24].tolist()}  (bytes {sorted(set(const//8))[:12]})")

    # data-region entropy
    blob = b"".join(b[7:] for _, b in data)
    print(f"\ndata region: {len(blob)} bytes, entropy {entropy(blob):.2f} bits/byte")

    # XOR of consecutive frames
    pairs = [(data[i][1][7:], data[i + 1][1][7:]) for i in range(len(data) - 1)]
    pairs = [(a, b) for a, b in pairs if len(a) > 8 and len(b) > 8]
    if pairs:
        n = min(min(len(a), len(b)) for a, b in pairs)
        x = [bytes(p ^ q for p, q in zip(a[:n], b[:n])) for a, b in pairs]
        e = np.mean([entropy(v) for v in x])
        print(f"XOR of consecutive data frames: {len(pairs)} pairs, mean entropy {e:.2f} bits/byte")

    # checksum hunt on the tail
    print("\nchecksum hunt (last 2 bytes as big/little endian over the rest):")
    polys = [(0x1021, "CCITT"), (0x8005, "IBM"), (0x3D65, "DNP"), (0x0589, "DECT")]
    best = []
    for poly, name in polys:
        for init in (0x0000, 0xFFFF):
            for refin in (False, True):
                for xo in (0x0000, 0xFFFF):
                    ok = 0
                    for _, b in data:
                        if len(b) < 6:
                            continue
                        v = crc16(b[:-2], poly, init, refin, True) ^ xo
                        if v == int.from_bytes(b[-2:], "big") or v == int.from_bytes(b[-2:], "little"):
                            ok += 1
                    if ok:
                        best.append((ok, name, hex(poly), hex(init), refin, hex(xo)))
    print("   ", best if best else "no CRC-16 match in any frame")


if __name__ == "__main__":
    main(sys.argv[1] if len(sys.argv) > 1 else "/tmp/bulk_test.jsonl")