chore: establish project baseline

scripts/ba_anexo1_refresh_diff.py

@@ -0,0 +1,30 @@
+"""Compare the ACTIVE BA_ST_MVA source PDF with a freshly downloaded RICMS-BA Anexo 1 and list the affected products. Read-only; changes no dataset."""
+import gzip, json, pickle, re, sys
+from pathlib import Path
+from pypdf import PdfReader
+ROOT = Path(__file__).resolve().parents[1]; sys.path.insert(0, str(ROOT))
+from central.app.datasets.ba_st_mva import parse_page  # noqa: E402
+
+def load(p):
+    recs = []
+    for i, pg in enumerate(PdfReader(p).pages, 1):
+        f, _ = parse_page(pg.extract_text() or "", i); recs += f
+    return recs
+
+def key(r): return (r["item"], r["cest"] or "")
+old = load(ROOT / "sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf"); new = load(ROOT / "sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026_dl20260921.pdf")
+sig = lambda r: (tuple(sorted(r["ncm_sh"] or [])), r["mva_original"], r["mva_adjusted"])  # noqa: E731
+o = {key(r): r for r in old}; n = {key(r): r for r in new}
+added = sorted(set(n) - set(o)); removed = sorted(set(o) - set(n)); changed = sorted(k for k in set(o) & set(n) if sig(o[k]) != sig(n[k]))
+def prefixes(recs): return {re.sub(r"\D", "", x) for r in recs for x in (r["ncm_sh"] or []) if len(re.sub(r"\D", "", x)) in (4, 6, 8)}
+po, pn = prefixes(old), prefixes(new); gone, come = po - pn, pn - po
+affected = {}
+for name in ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]:
+    snap = pickle.load(gzip.open(ROOT / f"docs/runs/local_completion/{name}_before.pkl.gz", "rb")); cols = snap["columns"]; i = cols.index("CLASSIFICACAO_FISCAL")
+    hit = [c for c, r in snap["rows"].items() if any(re.sub(r"\D", "", r[i] or "").startswith(p) for p in (gone | come))]
+    affected[name] = {"products": len(hit), "examples": hit[:5]}
+doc = {"old_records": len(old), "new_records": len(new), "items_added": [list(k) for k in added], "items_removed": [list(k) for k in removed], "items_changed": [{"key": list(k), "old": sig(o[k]), "new": sig(n[k])} for k in changed[:60]], "items_changed_count": len(changed),
+       "ncm_prefixes_no_longer_listed": sorted(gone)[:100], "ncm_prefixes_newly_listed": sorted(come)[:100], "affected_products_by_database": affected,
+       "note": "No dataset was modified; the Hub still serves the ACTIVE BA_ST_MVA (sha e07e9085...). The candidate PDF is versioned under sources/raw/bootstrap_v1/."}
+(ROOT / "docs" / "BA_ST_MVA_REFRESH_DIFF_V1.json").write_text(json.dumps(doc, ensure_ascii=False, indent=1, default=list), encoding="utf-8")
+print({k: (v if not isinstance(v, list) else len(v)) for k, v in doc.items() if k not in ("affected_products_by_database", "items_changed")}); print(affected)