"""Compare the ACTIVE BA_ST_MVA source PDF with a freshly downloaded RICMS-BA Anexo 1 and list the affected products. Read-only; changes no dataset."""
import gzip, json, pickle, re, sys
from pathlib import Path
from pypdf import PdfReader
ROOT = Path(__file__).resolve().parents[1]; sys.path.insert(0, str(ROOT))
from central.app.datasets.ba_st_mva import parse_page  # noqa: E402

def load(p):
    recs = []
    for i, pg in enumerate(PdfReader(p).pages, 1):
        f, _ = parse_page(pg.extract_text() or "", i); recs += f
    return recs

def key(r): return (r["item"], r["cest"] or "")
old = load(ROOT / "sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf"); new = load(ROOT / "sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026_dl20260921.pdf")
sig = lambda r: (tuple(sorted(r["ncm_sh"] or [])), r["mva_original"], r["mva_adjusted"])  # noqa: E731
o = {key(r): r for r in old}; n = {key(r): r for r in new}
added = sorted(set(n) - set(o)); removed = sorted(set(o) - set(n)); changed = sorted(k for k in set(o) & set(n) if sig(o[k]) != sig(n[k]))
def prefixes(recs): return {re.sub(r"\D", "", x) for r in recs for x in (r["ncm_sh"] or []) if len(re.sub(r"\D", "", x)) in (4, 6, 8)}
po, pn = prefixes(old), prefixes(new); gone, come = po - pn, pn - po
affected = {}
for name in ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]:
    snap = pickle.load(gzip.open(ROOT / f"docs/runs/local_completion/{name}_before.pkl.gz", "rb")); cols = snap["columns"]; i = cols.index("CLASSIFICACAO_FISCAL")
    hit = [c for c, r in snap["rows"].items() if any(re.sub(r"\D", "", r[i] or "").startswith(p) for p in (gone | come))]
    affected[name] = {"products": len(hit), "examples": hit[:5]}
doc = {"old_records": len(old), "new_records": len(new), "items_added": [list(k) for k in added], "items_removed": [list(k) for k in removed], "items_changed": [{"key": list(k), "old": sig(o[k]), "new": sig(n[k])} for k in changed[:60]], "items_changed_count": len(changed),
       "ncm_prefixes_no_longer_listed": sorted(gone)[:100], "ncm_prefixes_newly_listed": sorted(come)[:100], "affected_products_by_database": affected,
       "note": "No dataset was modified; the Hub still serves the ACTIVE BA_ST_MVA (sha e07e9085...). The candidate PDF is versioned under sources/raw/bootstrap_v1/."}
(ROOT / "docs" / "BA_ST_MVA_REFRESH_DIFF_V1.json").write_text(json.dumps(doc, ensure_ascii=False, indent=1, default=list), encoding="utf-8")
print({k: (v if not isinstance(v, list) else len(v)) for k, v in doc.items() if k not in ("affected_products_by_database", "items_changed")}); print(affected)