Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'

git-subtree-dir: fiscal git-subtree-mainline: d95165fc8f git-subtree-split: 102ba84500
2026-09-29 14:43:23 -03:00
parent d95165fc8f 102ba84500
commit d21a9d1140
868 changed files with 566394 additions and 0 deletions
@@ -0,0 +1,68 @@
+"""Semantic audit of the local-completion writes (Local Completion V2). Offline: reads docs/runs/local_completion/* only.
+Verdicts: SEMANTICALLY_PROVEN | NEEDS_REVIEW | INVALID_RULE. Emits docs/LOCAL_WRITE_SEMANTIC_AUDIT_V2.json (+ per-field tables) and rollback_plan.json {db: {field: [codes]}} for every write that is not PROVEN."""
+import csv, gzip, json, pickle, re, sys
+from collections import Counter, defaultdict
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]; ST = ROOT / "docs" / "runs" / "local_completion"
+DBS = ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]
+tipi = defaultdict(dict)
+for line in (ST / "tipi_export.txt").read_text(encoding="utf-8").splitlines():
+    ncm, ex, rate, txt = line.split("|"); tipi[ncm][ex] = (float(rate) if rate else None, txt)
+st_prefixes = {re.sub(r"\D", "", x) for x in (ST / "ba_st_ncm_export.txt").read_text(encoding="utf-8").split()}
+ST_CODES = {"010", "030", "070", "060", "201", "202", "203", "500", "10", "30", "70", "60"}
+audit = defaultdict(list); plan = defaultdict(lambda: defaultdict(list)); table = []
+for n in DBS:
+    b = pickle.load(gzip.open(ST / f"{n}_before.pkl.gz", "rb")); cols = b["columns"]; rows = b["rows"]; g = lambda code, c: rows[code][cols.index(c)]  # noqa: E731
+    dry = {}
+    for l in gzip.open(ST / f"{n}_dryrun.ndjson.gz", "rt", encoding="utf-8"):
+        r = json.loads(l); dry[(r["codigo"], r["field"])] = r
+    for l in gzip.open(ST / f"{n}_write_audit.ndjson.gz", "rt", encoding="utf-8"):
+        a = json.loads(l); code = a["product_code"]
+        for f, v in a["fields"].items():
+            d = dry[(code, f)]; ncm = re.sub(r"\D", "", str(g(code, "CLASSIFICACAO_FISCAL") or "")); rule = d["reason"]; ev = ""; verdict = "SEMANTICALLY_PROVEN"
+            if f == "COD_ORIGEM_MERCADORIA":
+                verdict, ev = "INVALID_RULE", f"CST digit is padding: CST={g(code,'CST')!r} while declared-origin products (1/2) also carry CST digit 0"
+            elif f == "IPI":
+                ncm_ok = dry[(code, "CLASSIFICACAO_FISCAL")]["status"] == "ALREADY_CORRECT" and len(ncm) == 8
+                base = tipi.get(ncm, {}).get("")
+                ex_prod = str(g(code, "EX_TIPI") or "").strip()
+                if not ncm_ok or base is None or base[0] is None or abs(base[0] - float(v["after"])) > 1e-9:
+                    verdict, ev = "INVALID_RULE", "TIPI numeric base rate does not match"
+                elif g(code, "IPI") is not None:
+                    verdict, ev = "NEEDS_REVIEW", "overwrote a non-null IPI"
+                elif ex_prod:
+                    verdict, ev = "NEEDS_REVIEW", f"product declares EX_TIPI={ex_prod}"
+                else:
+                    ev = f"NCM {ncm} valid; TIPI base row rate={base[0]} (text {base[1]!r}); EX rows for NCM: {sorted(k for k in tipi[ncm] if k)}; before NULL"
+            elif f == "CFOP":
+                cst, csosn = str(g(code, "CST") or "").strip(), str(g(code, "CSOSN") or "").strip()
+                st_decl = csosn in {"500", "201", "202", "203"} or cst in {"060", "010", "070", "030"}
+                if d["reason"] == "PROVEN_ROLE_AND_ST_CONTRADICT_DECLARED_CFOP" and v["before"] == "5405" and v["after"] == "5102":
+                    verdict, ev = ("INVALID_RULE", f"ERP declares ICMS-ST already retained (CST={cst}, CSOSN={csosn}); CFOP 5102 contradicts the product's own CST/CSOSN, official table alone cannot override it") if st_decl else ("NEEDS_REVIEW", "ST not declared")
+                elif v["after"] == "5101":
+                    ipp = dry.get((code, "IPPT"), {}).get("reason", "")
+                    strong = "COMPOSICAO" in ipp or "RECEITA_INGREDIENTES" in ipp
+                    verdict, ev = ("SEMANTICALLY_PROVEN", f"own production evidence: {ipp}") if strong else ("NEEDS_REVIEW", f"only kitchen-routing flag: {ipp}")
+                elif v["after"] == "5102" and v["before"] is None:
+                    in_st = any(ncm.startswith(p) for p in st_prefixes if len(p) in (4, 6, 8))
+                    verdict, ev = ("SEMANTICALLY_PROVEN", f"resale; CST={cst} CSOSN={csosn} not ST; NCM {ncm} not in BA ST prefixes") if (not st_decl and not in_st) else ("NEEDS_REVIEW", "possible ST")
+            elif f == "CEST":
+                cest = str(v["after"]); fmt = f"{cest[:2]}.{cest[2:5]}.{cest[5:]}"
+                verdict, ev = "SEMANTICALLY_PROVEN", f"official CEST dataset: NCM {ncm} linked exactly to {fmt} (single candidate); before blank"
+            elif f == "IPPT":
+                strong = "COMPOSICAO" in rule or "RECEITA_INGREDIENTES" in rule
+                verdict, ev = ("SEMANTICALLY_PROVEN", f"ERP evidence: {rule}") if strong else ("NEEDS_REVIEW", f"only kitchen-routing flag: {rule}")
+            else:
+                verdict, ev = "NEEDS_REVIEW", "no audit rule"
+            audit[(n, f)].append(verdict)
+            if verdict != "SEMANTICALLY_PROVEN":
+                plan[n][f].append(code)
+            if f in ("CFOP", "CEST", "IPPT"):
+                table.append({"database": n, "product": code, "field": f, "before": v["before"], "after": v["after"], "rule": rule, "evidence": ev, "verdict": verdict})
+summary = {f"{n}|{f}": dict(Counter(v)) for (n, f), v in audit.items()}
+(ROOT / "docs" / "LOCAL_WRITE_SEMANTIC_AUDIT_V2.json").write_text(json.dumps({"summary": summary, "small_field_writes": table}, ensure_ascii=False, indent=1), encoding="utf-8")
+with open(ROOT / "docs" / "LOCAL_WRITE_SEMANTIC_AUDIT_V2_CFOP_CEST_IPPT.csv", "w", newline="", encoding="utf-8") as fh:
+    w = csv.DictWriter(fh, fieldnames=list(table[0])); w.writeheader(); w.writerows(table)
+(ST / "rollback_plan.json").write_text(json.dumps(plan), encoding="utf-8")
+for k, v in summary.items(): print(k, v)