"""Semantic audit of the local-completion writes (Local Completion V2). Offline: reads docs/runs/local_completion/* only.
Verdicts: SEMANTICALLY_PROVEN | NEEDS_REVIEW | INVALID_RULE. Emits docs/LOCAL_WRITE_SEMANTIC_AUDIT_V2.json (+ per-field tables) and rollback_plan.json {db: {field: [codes]}} for every write that is not PROVEN."""
import csv, gzip, json, pickle, re, sys
from collections import Counter, defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]; ST = ROOT / "docs" / "runs" / "local_completion"
DBS = ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]
tipi = defaultdict(dict)
for line in (ST / "tipi_export.txt").read_text(encoding="utf-8").splitlines():
    ncm, ex, rate, txt = line.split("|"); tipi[ncm][ex] = (float(rate) if rate else None, txt)
st_prefixes = {re.sub(r"\D", "", x) for x in (ST / "ba_st_ncm_export.txt").read_text(encoding="utf-8").split()}
ST_CODES = {"010", "030", "070", "060", "201", "202", "203", "500", "10", "30", "70", "60"}
audit = defaultdict(list); plan = defaultdict(lambda: defaultdict(list)); table = []
for n in DBS:
    b = pickle.load(gzip.open(ST / f"{n}_before.pkl.gz", "rb")); cols = b["columns"]; rows = b["rows"]; g = lambda code, c: rows[code][cols.index(c)]  # noqa: E731
    dry = {}
    for l in gzip.open(ST / f"{n}_dryrun.ndjson.gz", "rt", encoding="utf-8"):
        r = json.loads(l); dry[(r["codigo"], r["field"])] = r
    for l in gzip.open(ST / f"{n}_write_audit.ndjson.gz", "rt", encoding="utf-8"):
        a = json.loads(l); code = a["product_code"]
        for f, v in a["fields"].items():
            d = dry[(code, f)]; ncm = re.sub(r"\D", "", str(g(code, "CLASSIFICACAO_FISCAL") or "")); rule = d["reason"]; ev = ""; verdict = "SEMANTICALLY_PROVEN"
            if f == "COD_ORIGEM_MERCADORIA":
                verdict, ev = "INVALID_RULE", f"CST digit is padding: CST={g(code,'CST')!r} while declared-origin products (1/2) also carry CST digit 0"
            elif f == "IPI":
                ncm_ok = dry[(code, "CLASSIFICACAO_FISCAL")]["status"] == "ALREADY_CORRECT" and len(ncm) == 8
                base = tipi.get(ncm, {}).get("")
                ex_prod = str(g(code, "EX_TIPI") or "").strip()
                if not ncm_ok or base is None or base[0] is None or abs(base[0] - float(v["after"])) > 1e-9:
                    verdict, ev = "INVALID_RULE", "TIPI numeric base rate does not match"
                elif g(code, "IPI") is not None:
                    verdict, ev = "NEEDS_REVIEW", "overwrote a non-null IPI"
                elif ex_prod:
                    verdict, ev = "NEEDS_REVIEW", f"product declares EX_TIPI={ex_prod}"
                else:
                    ev = f"NCM {ncm} valid; TIPI base row rate={base[0]} (text {base[1]!r}); EX rows for NCM: {sorted(k for k in tipi[ncm] if k)}; before NULL"
            elif f == "CFOP":
                cst, csosn = str(g(code, "CST") or "").strip(), str(g(code, "CSOSN") or "").strip()
                st_decl = csosn in {"500", "201", "202", "203"} or cst in {"060", "010", "070", "030"}
                if d["reason"] == "PROVEN_ROLE_AND_ST_CONTRADICT_DECLARED_CFOP" and v["before"] == "5405" and v["after"] == "5102":
                    verdict, ev = ("INVALID_RULE", f"ERP declares ICMS-ST already retained (CST={cst}, CSOSN={csosn}); CFOP 5102 contradicts the product's own CST/CSOSN, official table alone cannot override it") if st_decl else ("NEEDS_REVIEW", "ST not declared")
                elif v["after"] == "5101":
                    ipp = dry.get((code, "IPPT"), {}).get("reason", "")
                    strong = "COMPOSICAO" in ipp or "RECEITA_INGREDIENTES" in ipp
                    verdict, ev = ("SEMANTICALLY_PROVEN", f"own production evidence: {ipp}") if strong else ("NEEDS_REVIEW", f"only kitchen-routing flag: {ipp}")
                elif v["after"] == "5102" and v["before"] is None:
                    in_st = any(ncm.startswith(p) for p in st_prefixes if len(p) in (4, 6, 8))
                    verdict, ev = ("SEMANTICALLY_PROVEN", f"resale; CST={cst} CSOSN={csosn} not ST; NCM {ncm} not in BA ST prefixes") if (not st_decl and not in_st) else ("NEEDS_REVIEW", "possible ST")
            elif f == "CEST":
                cest = str(v["after"]); fmt = f"{cest[:2]}.{cest[2:5]}.{cest[5:]}"
                verdict, ev = "SEMANTICALLY_PROVEN", f"official CEST dataset: NCM {ncm} linked exactly to {fmt} (single candidate); before blank"
            elif f == "IPPT":
                strong = "COMPOSICAO" in rule or "RECEITA_INGREDIENTES" in rule
                verdict, ev = ("SEMANTICALLY_PROVEN", f"ERP evidence: {rule}") if strong else ("NEEDS_REVIEW", f"only kitchen-routing flag: {rule}")
            else:
                verdict, ev = "NEEDS_REVIEW", "no audit rule"
            audit[(n, f)].append(verdict)
            if verdict != "SEMANTICALLY_PROVEN":
                plan[n][f].append(code)
            if f in ("CFOP", "CEST", "IPPT"):
                table.append({"database": n, "product": code, "field": f, "before": v["before"], "after": v["after"], "rule": rule, "evidence": ev, "verdict": verdict})
summary = {f"{n}|{f}": dict(Counter(v)) for (n, f), v in audit.items()}
(ROOT / "docs" / "LOCAL_WRITE_SEMANTIC_AUDIT_V2.json").write_text(json.dumps({"summary": summary, "small_field_writes": table}, ensure_ascii=False, indent=1), encoding="utf-8")
with open(ROOT / "docs" / "LOCAL_WRITE_SEMANTIC_AUDIT_V2_CFOP_CEST_IPPT.csv", "w", newline="", encoding="utf-8") as fh:
    w = csv.DictWriter(fh, fieldnames=list(table[0])); w.writeheader(); w.writerows(table)
(ST / "rollback_plan.json").write_text(json.dumps(plan), encoding="utf-8")
for k, v in summary.items(): print(k, v)