GTI_GERENCIADOR_FISCAL/fiscal/scripts/run_icms_context_v1.py

"""ICMS Context Engine V1 runner (READ-ONLY): ST applicability, ST withholding, product role, CSOSN V2 dry-run.
Outputs: docs/ICMS_PRODUCT_CONTEXT_V1.json, ICMS_CONTEXT_COVERAGE_V1.json, CSOSN_DRY_RUN_V2.json,
FISCAL_CONTEXT_REQUIREMENTS_V3.json, NEXT_FISCAL_FIELD_IMPLEMENTATION_V3.md. Never writes to Firebird.
"""
import json
import os
import subprocess
import sys
from collections import Counter, defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
import run_fiscal_decision_v1 as base  # noqa: E402
from central.app.icms_context import ICMS_CONTEXT_ENGINE_VERSION, csosn_v2, product_role, st_conflict_class, st_status, withheld_status  # noqa: E402
from central.app.product_defaults import validate_against_history  # noqa: E402
from gti_fiscal_agent.firebird_reader import _connect, parse_company_context  # noqa: E402

s = lambda v: None if v is None else str(v).strip()
dg = lambda v: "".join(c for c in str(v or "") if c.isdigit())


def main():
    token = sys.argv[1]; hub = "http://localhost:8000"
    base.load_dotenv(ROOT / ".env")
    cur = _connect(os.environ).cursor()
    cur.execute("select CODIGO, PRODUTO, CLASSIFICACAO_FISCAL, CEST, CSOSN, CST, CFOP, TIPO, PROD_COZINHA, PRODUTO_SERVICO from C000025 order by CODIGO")
    products = [dict(zip(("CODIGO", "PRODUTO", "CLASSIFICACAO_FISCAL", "CEST", "CSOSN", "CST", "CFOP", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO"), r)) for r in cur.fetchall()]
    cur.execute("select ICMS_CSOSN from C000004"); company_default = s(cur.fetchone()[0])
    cur.execute("select UF, CRT from C000004"); company = parse_company_context(cur.fetchall()); uf, crt = company["uf"], company["crt"]
    cur.execute("select CODRECEITA from RECEITA_INGREDIENTES where CODRECEITA is not null"); recipes = {str(r[0]).zfill(6) for r in cur.fetchall()}
    cur.execute("select distinct CODPRODUTO from C000032 where TIPO = 'COMPOSICAO'"); composed = {s(r[0]) for r in cur.fetchall()}
    cur.execute("select CODPRODUTO, ICMS_RETIDO, ICMS_VALORRETIDO, SUB_VALOR, CFOP from C000088"); purchases = defaultdict(list)
    for cod, ret, vr, sv, cf in cur.fetchall():
        purchases[s(cod)].append({"icms_retido": s(ret), "valor_retido": vr, "sub_valor": sv, "cfop": s(cf)})
    cur.execute("select CODPRODUTO, ULTIMA_ENTRADA, NOTA_FISCAL from C000100"); stock_entry = {s(r[0]): (r[1] is not None or (s(r[2]) not in (None, "", "000000"))) for r in cur.fetchall()}
    cur.execute("select CODPRODUTO, CST, ICMS_SUB from C000062"); nf_out = defaultdict(list)
    for cod, cst, sub in cur.fetchall(): nf_out[s(cod)].append((s(cst), float(sub or 0)))
    cur.connection.close()

    sh = defaultdict(list)
    out = subprocess.run(["docker", "exec", "projeto-postgres-1", "psql", "-U", "gti", "-d", "gti_fiscal", "-At", "-F|", "-c",
                          "select e.item, n.ncm_sh from ba_st_mva_entries e join ba_st_mva_ncm_sh n on n.entry_id=e.id join datasets d on d.id=e.dataset_id where d.status='ACTIVE'"], capture_output=True, text=True, check=True).stdout
    for line in out.splitlines():
        it, n = line.split("|"); sh[it].append(dg(n))

    refs = base.build_refs(hub, token, products, uf)
    decisions = []
    for i in range(0, len(products), 100):
        chunk = products[i:i + 100]
        decisions += base.http(f"{hub}/v1/classify/batch", token, {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]})["results"]

    trusted = json.loads((ROOT / "docs" / "runs" / "trusted_historical_items.json").read_text(encoding="utf-8"))
    hist = defaultdict(list)
    for t in trusted:
        hist[s(t["cod"])].append(s(t.get("csosn")) or "")
    hist_cfop = defaultdict(list)
    for t in trusted:
        hist_cfop[s(t["cod"])].append(s(t.get("cfop")) or "")

    rows = []
    for p, d in zip(products, decisions):
        code = s(p["CODIGO"]); cls = d.get("classification") or {}
        ncm_status = d.get("status"); mva = d.get("ba_st_mva_status"); ceststat = d.get("cest_status")
        ba = cls.get("ba_st_mva") or {}
        ba_item, ba_cest = ba.get("item"), ba.get("cest")
        ncm8 = dg(p["CLASSIFICACAO_FISCAL"])
        sh_len = max((len(x) for x in sh.get(ba_item, []) if x and ncm8.startswith(x)), default=None) if ba_item else None
        st, st_reason = st_status(ncm_status, mva, ceststat, p["CEST"], ba_cest, sh_len)
        wh, wh_reason = withheld_status(purchases.get(code, []))
        purchase_ev = bool(purchases.get(code)) or stock_entry.get(code, False)
        role, role_ev = product_role(p["TIPO"], p["PROD_COZINHA"] == 1, code in recipes, code in composed, bool(s(p["PRODUTO_SERVICO"])), purchase_ev)
        cs = csosn_v2(p["CSOSN"], company_default, ncm_status, st, wh, role, refs["csosn"])
        h = hist.get(code, [])
        hv = validate_against_history(cs["proposed_csosn"], [x for x in h if x])
        share = (sum(1 for x in h if x == cs["current_value"]) / len(h)) if h else None
        rows.append({
            "CODIGO": code, "DESCRIPTION": p["PRODUTO"], "NCM": p["CLASSIFICACAO_FISCAL"], "CEST": s(p["CEST"]) or None,
            "NCM_STATUS": ncm_status, "ST_STATUS": st, "ST_REASON": st_reason, "ST_WITHHELD_STATUS": wh, "ST_WITHHELD_REASON": wh_reason,
            "PRODUCT_ROLE": role, "PRODUCT_ROLE_EVIDENCE": role_ev, "CURRENT_CSOSN": cs["current_value"], "COMPANY_CRT": crt, "COMPANY_DEFAULT_CSOSN": company_default,
            "HISTORICAL_CSOSN": dict(Counter(x for x in h if x)), "HISTORICAL_CONSISTENCY": share,
            "ERP_EMITTED_ST_TREATMENT_NF_OUT": {"cst060_items": sum(1 for c, _ in nf_out.get(code, []) if c == "060"), "icms_sub_items": sum(1 for _, v in nf_out.get(code, []) if v > 0)},
            "CSOSN_V2": cs, "CSOSN_HISTORY_VALIDATION": hv,
            "_conf": {"cest_status": ceststat, "mva": mva, "sh_len": sh_len, "erp_cest": p["CEST"], "ba_cest": ba_cest, "cfop": s(p["CFOP"]),
                      "hist_cfop_current": bool(hist_cfop.get(code)) and all(x == s(p["CFOP"]) for x in hist_cfop[code])},
        })

    # ST conflict audit (the CFOP-conflict population: ST-subject with non-ST CFOP, or 5405 without ST)
    conflicts = []
    for r in rows:
        c = r["_conf"]
        st_subject = c["mva"] == "MVA_RESOLVED"
        cfop_conflict = r["NCM_STATUS"] == "EXACT_NCM" and ((st_subject and c["cfop"] == "5102") or (c["mva"] == "NO_MVA_MAPPING_FOUND" and c["cfop"] == "5405"))
        if cfop_conflict:
            primary, flags = st_conflict_class(c["cest_status"], c["mva"], c["sh_len"], c["erp_cest"], c["ba_cest"], c["cfop"], c["hist_cfop_current"], c["mva"] == "NO_MVA_MAPPING_FOUND" and c["cfop"] == "5405")
            conflicts.append({"CODIGO": r["CODIGO"], "DESCRIPTION": r["DESCRIPTION"], "NCM": r["NCM"], "erp_cfop": c["cfop"], "primary": primary, "flags": flags, "st_status": r["ST_STATUS"]})
    for r in rows:
        r.pop("_conf")

    C = lambda key: Counter(r[key] for r in rows)
    st_c, wh_c, role_c = C("ST_STATUS"), C("ST_WITHHELD_STATUS"), C("PRODUCT_ROLE")
    cs_c = Counter(r["CSOSN_V2"]["status"] for r in rows)
    act_c = Counter(r["CSOSN_V2"]["action_class"] for r in rows)
    hv_c = Counter(r["CSOSN_HISTORY_VALIDATION"] for r in rows if r["CSOSN_V2"]["proposed_csosn"] is not None)
    ncm_blocked = sum(1 for r in rows if r["NCM_STATUS"] != "EXACT_NCM")
    ncm_only_blocked = ncm_blocked  # every NCM-invalid product is UNRESOLVED solely because of NCM (no other context is evaluated)
    already = sum(1 for r in rows if r["CSOSN_V2"]["status"] == "CSOSN_RESOLVED" and r["CSOSN_V2"]["proposed_csosn"] == r["CURRENT_CSOSN"])
    change = sum(1 for r in rows if r["CSOSN_V2"]["status"] == "CSOSN_RESOLVED" and r["CSOSN_V2"]["proposed_csosn"] != r["CURRENT_CSOSN"])
    auto_safe = 0  # none may be write-eligible in this mission; readiness below
    ready = sum(1 for r in rows if r["CSOSN_V2"]["status"] == "CSOSN_RESOLVED" and r["CSOSN_HISTORY_VALIDATION"] == "MATCH")
    conf_primary = Counter(c["primary"] for c in conflicts)
    conf_flags = Counter(f for c in conflicts for f in c["flags"])

    cov = {"artifact": "ICMS_CONTEXT_COVERAGE_V1", "firebird_writes": 0, "TOTAL_PRODUCTS": len(rows), "engine": ICMS_CONTEXT_ENGINE_VERSION,
           "ST": {k: st_c.get(k, 0) for k in ("ST_CONFIRMED", "ST_NOT_APPLICABLE", "ST_CANDIDATE", "ST_UNKNOWN")},
           "ST_reasons": dict(C("ST_REASON")),
           "ST_WITHHELD": {"ST_ALREADY_WITHHELD_CONFIRMED": wh_c.get("ST_ALREADY_WITHHELD_CONFIRMED", 0), "ST_ALREADY_WITHHELD_NOT_CONFIRMED": wh_c.get("ST_ALREADY_WITHHELD_NOT_CONFIRMED", 0), "ST_ALREADY_WITHHELD_UNKNOWN": wh_c.get("ST_WITHHOLDING_UNKNOWN", 0)},
           "PRODUCT_ROLE": {k: role_c.get(k, 0) for k in ("RESALE", "OWN_PRODUCTION", "INPUT", "SERVICE", "UNKNOWN")},
           "CSOSN": {"RESOLVED": cs_c.get("CSOSN_RESOLVED", 0), "ALREADY_CORRECT": already, "CHANGE_REQUIRED": change, "CANDIDATES": cs_c.get("CSOSN_CANDIDATE", 0),
                     "CONTEXT_REQUIRED": cs_c.get("CSOSN_CONTEXT_REQUIRED", 0), "CONFLICT": cs_c.get("CSOSN_CONFLICT", 0), "UNRESOLVED": cs_c.get("CSOSN_UNRESOLVED", 0),
                     "AUTO_APPLY_SAFE": auto_safe, "AUTO_APPLY_READY_IF_ENABLED": ready, "action_class": dict(act_c),
                     "reasons": dict(Counter(r["CSOSN_V2"]["reason_code"] for r in rows).most_common(10))},
           "CSOSN_HISTORY_of_proposals": {"MATCH": hv_c.get("MATCH", 0), "MISMATCH": hv_c.get("MISMATCH", 0), "UNVALIDATED": hv_c.get("UNVALIDATED", 0) + sum(1 for r in rows if r["CSOSN_V2"]["proposed_csosn"] is None)},
           "CSOSN_HISTORY_of_proposals_note": "MATCH/MISMATCH/UNVALIDATED counted on products WITH a proposed value; UNVALIDATED additionally counts products with no proposal (nothing to validate).",
           "NCM_BLOCKED_PRODUCTS": ncm_blocked, "NCM_BLOCKED_ONLY_BY_NCM": ncm_only_blocked,
           "ST_CONFLICT_AUDIT": {"population": len(conflicts), "primary_class": dict(conf_primary), "any_flag": dict(conf_flags)}}
    (ROOT / "docs" / "ICMS_CONTEXT_COVERAGE_V1.json").write_text(json.dumps(cov, ensure_ascii=False, indent=1), encoding="utf-8")
    (ROOT / "docs" / "ICMS_PRODUCT_CONTEXT_V1.json").write_text(json.dumps({"artifact": "ICMS_PRODUCT_CONTEXT_V1", "engine": ICMS_CONTEXT_ENGINE_VERSION, "company": {"crt": crt, "uf": uf, "default_csosn": company_default},
                                                                         "products": rows, "st_conflict_audit": conflicts}, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
    dry = {"artifact": "CSOSN_DRY_RUN_V2", "firebird_writes": 0, "CSOSN_AUTO_APPLY_SAFE": auto_safe, "CSOSN_ALREADY_CORRECT": already, "CSOSN_CHANGE_REQUIRED": change,
           "would_change": [], "states": dict(cs_c), "note": "No CSOSN proposal is write-eligible; readiness requires proven ST, withholding, role and ICMS treatment."}
    (ROOT / "docs" / "CSOSN_DRY_RUN_V2.json").write_text(json.dumps(dry, ensure_ascii=False, indent=1), encoding="utf-8")

    n = lambda key, val: sum(1 for r in rows if r[key] == val)
    req = [
        {"INPUT": "valid_ncm", "SOURCE_EXPECTED": "product master", "AVAILABLE": "PARTIAL", "AFFECTED_PRODUCTS": ncm_blocked, "AFFECTED_FIELD": "CSOSN", "NEXT_ACTION": "Manual NCM review (3 legacy + 23 invalid); not auto-corrected."},
        {"INPUT": "st_confirmed_per_product (CEST corroboration)", "SOURCE_EXPECTED": "ERP CEST field filled and equal to the BA Anexo 1 item CEST", "AVAILABLE": "PARTIAL", "AFFECTED_PRODUCTS": st_c.get("ST_CANDIDATE", 0) + st_c.get("ST_UNKNOWN", 0) - ncm_blocked, "AFFECTED_FIELD": "CSOSN", "NEXT_ACTION": "Fill/confirm CEST for ST candidates (description review) — 490 of 521 ERP CEST values are blank."},
        {"INPUT": "st_withheld_by_supplier (purchase evidence)", "SOURCE_EXPECTED": "purchase items with ICMS_RETIDO/ST values (C000088)", "AVAILABLE": "NO (12 purchase items, all ICMS_RETIDO='N', ST values 0)", "AFFECTED_PRODUCTS": st_c.get("ST_CONFIRMED", 0) + st_c.get("ST_CANDIDATE", 0), "AFFECTED_FIELD": "CSOSN(500 vs other)", "NEXT_ACTION": "Import supplier NF-e XML (ICMS ST/CSOSN 500 on entrada) or obtain vendor confirmation of how retention is recorded."},
        {"INPUT": "product_role (resale/own production)", "SOURCE_EXPECTED": "C000025.TIPO/PROD_COZINHA + recipe + purchase docs", "AVAILABLE": "PARTIAL", "AFFECTED_PRODUCTS": role_c.get("UNKNOWN", 0), "AFFECTED_FIELD": "CSOSN,CFOP", "NEXT_ACTION": "TIPO_ITEM/PRODUTO_SERVICO are unused; only 7 INPUT and ~13 own-production products are objectively evidenced."},
        {"INPUT": "product_icms_treatment (taxed vs exempt/not taxed)", "SOURCE_EXPECTED": "official per-NCM ICMS treatment source (none acquired)", "AVAILABLE": False, "AFFECTED_PRODUCTS": st_c.get("ST_NOT_APPLICABLE", 0), "AFFECTED_FIELD": "CSOSN(102 vs 400/300/103)", "NEXT_ACTION": "Decide with accountant/vendor: ERP CST=040 with CSOSN=102 is the emitted practice; needs a documented rule."},
    ]
    (ROOT / "docs" / "FISCAL_CONTEXT_REQUIREMENTS_V3.json").write_text(json.dumps({"artifact": "FISCAL_CONTEXT_REQUIREMENTS_V3", "requirements": req}, ensure_ascii=False, indent=1), encoding="utf-8")
    md = ["# NEXT_FISCAL_FIELD_IMPLEMENTATION_V3", "",
          "Mechanical ordering: mapping confirmed, rule available, context available, products affected, history validation, auto-apply possible.", "",
          "| FIELD | ERP_MAPPING | CONTEXT_AVAILABLE | RULE_AVAILABLE | POTENTIAL_PRODUCTS | HISTORICAL_VALIDATION | NEXT_ACTION |", "|---|---|---|---|---|---|---|",
          f"| CSOSN | CONFIRMED (HIGH) | NO (retention + ICMS treatment missing) | PARTIAL (ST→500 rule needs withholding proof) | {cs_c.get('CSOSN_CANDIDATE',0)+cs_c.get('CSOSN_CONTEXT_REQUIRED',0)+cs_c.get('CSOSN_CONFLICT',0)} | trusted 1146 items | Acquire supplier ST retention evidence; get accountant decision on 102 vs 400 |",
          "| CEST | CONFIRMED (HIGH) | PARTIAL | PARTIAL | " + str(st_c.get("ST_CANDIDATE", 0)) + " | weak | CEST fill/confirm drives ST_CONFIRMED |",
          "| CFOP | CONFIRMED (HIGH) | PARTIAL | YES (ST family) | 29 conflicts | trusted history | Depends on ST_CONFIRMED + retention |",
          "| IPI | CONFIRMED (HIGH) | YES | YES | 107 | n/a | Maintained |", "",
          "Blocking inputs with counts: `docs/FISCAL_CONTEXT_REQUIREMENTS_V3.json`."]
    (ROOT / "docs" / "NEXT_FISCAL_FIELD_IMPLEMENTATION_V3.md").write_text("\n".join(md) + "\n", encoding="utf-8")
    print(json.dumps({k: cov[k] for k in ("ST", "ST_reasons", "ST_WITHHELD", "PRODUCT_ROLE", "CSOSN", "CSOSN_HISTORY_of_proposals", "NCM_BLOCKED_PRODUCTS", "ST_CONFLICT_AUDIT")}, ensure_ascii=True, indent=1))


main()