gti
d21a9d1140
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
31 lines
2.6 KiB
Python
31 lines
2.6 KiB
Python
"""Compare the ACTIVE BA_ST_MVA source PDF with a freshly downloaded RICMS-BA Anexo 1 and list the affected products. Read-only; changes no dataset."""
|
|
import gzip, json, pickle, re, sys
|
|
from pathlib import Path
|
|
from pypdf import PdfReader
|
|
ROOT = Path(__file__).resolve().parents[1]; sys.path.insert(0, str(ROOT))
|
|
from central.app.datasets.ba_st_mva import parse_page # noqa: E402
|
|
|
|
def load(p):
|
|
recs = []
|
|
for i, pg in enumerate(PdfReader(p).pages, 1):
|
|
f, _ = parse_page(pg.extract_text() or "", i); recs += f
|
|
return recs
|
|
|
|
def key(r): return (r["item"], r["cest"] or "")
|
|
old = load(ROOT / "sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf"); new = load(ROOT / "sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026_dl20260921.pdf")
|
|
sig = lambda r: (tuple(sorted(r["ncm_sh"] or [])), r["mva_original"], r["mva_adjusted"]) # noqa: E731
|
|
o = {key(r): r for r in old}; n = {key(r): r for r in new}
|
|
added = sorted(set(n) - set(o)); removed = sorted(set(o) - set(n)); changed = sorted(k for k in set(o) & set(n) if sig(o[k]) != sig(n[k]))
|
|
def prefixes(recs): return {re.sub(r"\D", "", x) for r in recs for x in (r["ncm_sh"] or []) if len(re.sub(r"\D", "", x)) in (4, 6, 8)}
|
|
po, pn = prefixes(old), prefixes(new); gone, come = po - pn, pn - po
|
|
affected = {}
|
|
for name in ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]:
|
|
snap = pickle.load(gzip.open(ROOT / f"docs/runs/local_completion/{name}_before.pkl.gz", "rb")); cols = snap["columns"]; i = cols.index("CLASSIFICACAO_FISCAL")
|
|
hit = [c for c, r in snap["rows"].items() if any(re.sub(r"\D", "", r[i] or "").startswith(p) for p in (gone | come))]
|
|
affected[name] = {"products": len(hit), "examples": hit[:5]}
|
|
doc = {"old_records": len(old), "new_records": len(new), "items_added": [list(k) for k in added], "items_removed": [list(k) for k in removed], "items_changed": [{"key": list(k), "old": sig(o[k]), "new": sig(n[k])} for k in changed[:60]], "items_changed_count": len(changed),
|
|
"ncm_prefixes_no_longer_listed": sorted(gone)[:100], "ncm_prefixes_newly_listed": sorted(come)[:100], "affected_products_by_database": affected,
|
|
"note": "No dataset was modified; the Hub still serves the ACTIVE BA_ST_MVA (sha e07e9085...). The candidate PDF is versioned under sources/raw/bootstrap_v1/."}
|
|
(ROOT / "docs" / "BA_ST_MVA_REFRESH_DIFF_V1.json").write_text(json.dumps(doc, ensure_ascii=False, indent=1, default=list), encoding="utf-8")
|
|
print({k: (v if not isinstance(v, list) else len(v)) for k, v in doc.items() if k not in ("affected_products_by_database", "items_changed")}); print(affected)
|