Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
This commit is contained in:
gti
@@ -0,0 +1,155 @@
|
||||
"""ICMS Context Engine V1 runner (READ-ONLY): ST applicability, ST withholding, product role, CSOSN V2 dry-run.
|
||||
Outputs: docs/ICMS_PRODUCT_CONTEXT_V1.json, ICMS_CONTEXT_COVERAGE_V1.json, CSOSN_DRY_RUN_V2.json,
|
||||
FISCAL_CONTEXT_REQUIREMENTS_V3.json, NEXT_FISCAL_FIELD_IMPLEMENTATION_V3.md. Never writes to Firebird.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
import run_fiscal_decision_v1 as base # noqa: E402
|
||||
from central.app.icms_context import ICMS_CONTEXT_ENGINE_VERSION, csosn_v2, product_role, st_conflict_class, st_status, withheld_status # noqa: E402
|
||||
from central.app.product_defaults import validate_against_history # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import _connect, parse_company_context # noqa: E402
|
||||
|
||||
s = lambda v: None if v is None else str(v).strip()
|
||||
dg = lambda v: "".join(c for c in str(v or "") if c.isdigit())
|
||||
|
||||
|
||||
def main():
|
||||
token = sys.argv[1]; hub = "http://localhost:8000"
|
||||
base.load_dotenv(ROOT / ".env")
|
||||
cur = _connect(os.environ).cursor()
|
||||
cur.execute("select CODIGO, PRODUTO, CLASSIFICACAO_FISCAL, CEST, CSOSN, CST, CFOP, TIPO, PROD_COZINHA, PRODUTO_SERVICO from C000025 order by CODIGO")
|
||||
products = [dict(zip(("CODIGO", "PRODUTO", "CLASSIFICACAO_FISCAL", "CEST", "CSOSN", "CST", "CFOP", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO"), r)) for r in cur.fetchall()]
|
||||
cur.execute("select ICMS_CSOSN from C000004"); company_default = s(cur.fetchone()[0])
|
||||
cur.execute("select UF, CRT from C000004"); company = parse_company_context(cur.fetchall()); uf, crt = company["uf"], company["crt"]
|
||||
cur.execute("select CODRECEITA from RECEITA_INGREDIENTES where CODRECEITA is not null"); recipes = {str(r[0]).zfill(6) for r in cur.fetchall()}
|
||||
cur.execute("select distinct CODPRODUTO from C000032 where TIPO = 'COMPOSICAO'"); composed = {s(r[0]) for r in cur.fetchall()}
|
||||
cur.execute("select CODPRODUTO, ICMS_RETIDO, ICMS_VALORRETIDO, SUB_VALOR, CFOP from C000088"); purchases = defaultdict(list)
|
||||
for cod, ret, vr, sv, cf in cur.fetchall():
|
||||
purchases[s(cod)].append({"icms_retido": s(ret), "valor_retido": vr, "sub_valor": sv, "cfop": s(cf)})
|
||||
cur.execute("select CODPRODUTO, ULTIMA_ENTRADA, NOTA_FISCAL from C000100"); stock_entry = {s(r[0]): (r[1] is not None or (s(r[2]) not in (None, "", "000000"))) for r in cur.fetchall()}
|
||||
cur.execute("select CODPRODUTO, CST, ICMS_SUB from C000062"); nf_out = defaultdict(list)
|
||||
for cod, cst, sub in cur.fetchall(): nf_out[s(cod)].append((s(cst), float(sub or 0)))
|
||||
cur.connection.close()
|
||||
|
||||
sh = defaultdict(list)
|
||||
out = subprocess.run(["docker", "exec", "projeto-postgres-1", "psql", "-U", "gti", "-d", "gti_fiscal", "-At", "-F|", "-c",
|
||||
"select e.item, n.ncm_sh from ba_st_mva_entries e join ba_st_mva_ncm_sh n on n.entry_id=e.id join datasets d on d.id=e.dataset_id where d.status='ACTIVE'"], capture_output=True, text=True, check=True).stdout
|
||||
for line in out.splitlines():
|
||||
it, n = line.split("|"); sh[it].append(dg(n))
|
||||
|
||||
refs = base.build_refs(hub, token, products, uf)
|
||||
decisions = []
|
||||
for i in range(0, len(products), 100):
|
||||
chunk = products[i:i + 100]
|
||||
decisions += base.http(f"{hub}/v1/classify/batch", token, {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]})["results"]
|
||||
|
||||
trusted = json.loads((ROOT / "docs" / "runs" / "trusted_historical_items.json").read_text(encoding="utf-8"))
|
||||
hist = defaultdict(list)
|
||||
for t in trusted:
|
||||
hist[s(t["cod"])].append(s(t.get("csosn")) or "")
|
||||
hist_cfop = defaultdict(list)
|
||||
for t in trusted:
|
||||
hist_cfop[s(t["cod"])].append(s(t.get("cfop")) or "")
|
||||
|
||||
rows = []
|
||||
for p, d in zip(products, decisions):
|
||||
code = s(p["CODIGO"]); cls = d.get("classification") or {}
|
||||
ncm_status = d.get("status"); mva = d.get("ba_st_mva_status"); ceststat = d.get("cest_status")
|
||||
ba = cls.get("ba_st_mva") or {}
|
||||
ba_item, ba_cest = ba.get("item"), ba.get("cest")
|
||||
ncm8 = dg(p["CLASSIFICACAO_FISCAL"])
|
||||
sh_len = max((len(x) for x in sh.get(ba_item, []) if x and ncm8.startswith(x)), default=None) if ba_item else None
|
||||
st, st_reason = st_status(ncm_status, mva, ceststat, p["CEST"], ba_cest, sh_len)
|
||||
wh, wh_reason = withheld_status(purchases.get(code, []))
|
||||
purchase_ev = bool(purchases.get(code)) or stock_entry.get(code, False)
|
||||
role, role_ev = product_role(p["TIPO"], p["PROD_COZINHA"] == 1, code in recipes, code in composed, bool(s(p["PRODUTO_SERVICO"])), purchase_ev)
|
||||
cs = csosn_v2(p["CSOSN"], company_default, ncm_status, st, wh, role, refs["csosn"])
|
||||
h = hist.get(code, [])
|
||||
hv = validate_against_history(cs["proposed_csosn"], [x for x in h if x])
|
||||
share = (sum(1 for x in h if x == cs["current_value"]) / len(h)) if h else None
|
||||
rows.append({
|
||||
"CODIGO": code, "DESCRIPTION": p["PRODUTO"], "NCM": p["CLASSIFICACAO_FISCAL"], "CEST": s(p["CEST"]) or None,
|
||||
"NCM_STATUS": ncm_status, "ST_STATUS": st, "ST_REASON": st_reason, "ST_WITHHELD_STATUS": wh, "ST_WITHHELD_REASON": wh_reason,
|
||||
"PRODUCT_ROLE": role, "PRODUCT_ROLE_EVIDENCE": role_ev, "CURRENT_CSOSN": cs["current_value"], "COMPANY_CRT": crt, "COMPANY_DEFAULT_CSOSN": company_default,
|
||||
"HISTORICAL_CSOSN": dict(Counter(x for x in h if x)), "HISTORICAL_CONSISTENCY": share,
|
||||
"ERP_EMITTED_ST_TREATMENT_NF_OUT": {"cst060_items": sum(1 for c, _ in nf_out.get(code, []) if c == "060"), "icms_sub_items": sum(1 for _, v in nf_out.get(code, []) if v > 0)},
|
||||
"CSOSN_V2": cs, "CSOSN_HISTORY_VALIDATION": hv,
|
||||
"_conf": {"cest_status": ceststat, "mva": mva, "sh_len": sh_len, "erp_cest": p["CEST"], "ba_cest": ba_cest, "cfop": s(p["CFOP"]),
|
||||
"hist_cfop_current": bool(hist_cfop.get(code)) and all(x == s(p["CFOP"]) for x in hist_cfop[code])},
|
||||
})
|
||||
|
||||
# ST conflict audit (the CFOP-conflict population: ST-subject with non-ST CFOP, or 5405 without ST)
|
||||
conflicts = []
|
||||
for r in rows:
|
||||
c = r["_conf"]
|
||||
st_subject = c["mva"] == "MVA_RESOLVED"
|
||||
cfop_conflict = r["NCM_STATUS"] == "EXACT_NCM" and ((st_subject and c["cfop"] == "5102") or (c["mva"] == "NO_MVA_MAPPING_FOUND" and c["cfop"] == "5405"))
|
||||
if cfop_conflict:
|
||||
primary, flags = st_conflict_class(c["cest_status"], c["mva"], c["sh_len"], c["erp_cest"], c["ba_cest"], c["cfop"], c["hist_cfop_current"], c["mva"] == "NO_MVA_MAPPING_FOUND" and c["cfop"] == "5405")
|
||||
conflicts.append({"CODIGO": r["CODIGO"], "DESCRIPTION": r["DESCRIPTION"], "NCM": r["NCM"], "erp_cfop": c["cfop"], "primary": primary, "flags": flags, "st_status": r["ST_STATUS"]})
|
||||
for r in rows:
|
||||
r.pop("_conf")
|
||||
|
||||
C = lambda key: Counter(r[key] for r in rows)
|
||||
st_c, wh_c, role_c = C("ST_STATUS"), C("ST_WITHHELD_STATUS"), C("PRODUCT_ROLE")
|
||||
cs_c = Counter(r["CSOSN_V2"]["status"] for r in rows)
|
||||
act_c = Counter(r["CSOSN_V2"]["action_class"] for r in rows)
|
||||
hv_c = Counter(r["CSOSN_HISTORY_VALIDATION"] for r in rows if r["CSOSN_V2"]["proposed_csosn"] is not None)
|
||||
ncm_blocked = sum(1 for r in rows if r["NCM_STATUS"] != "EXACT_NCM")
|
||||
ncm_only_blocked = ncm_blocked # every NCM-invalid product is UNRESOLVED solely because of NCM (no other context is evaluated)
|
||||
already = sum(1 for r in rows if r["CSOSN_V2"]["status"] == "CSOSN_RESOLVED" and r["CSOSN_V2"]["proposed_csosn"] == r["CURRENT_CSOSN"])
|
||||
change = sum(1 for r in rows if r["CSOSN_V2"]["status"] == "CSOSN_RESOLVED" and r["CSOSN_V2"]["proposed_csosn"] != r["CURRENT_CSOSN"])
|
||||
auto_safe = 0 # none may be write-eligible in this mission; readiness below
|
||||
ready = sum(1 for r in rows if r["CSOSN_V2"]["status"] == "CSOSN_RESOLVED" and r["CSOSN_HISTORY_VALIDATION"] == "MATCH")
|
||||
conf_primary = Counter(c["primary"] for c in conflicts)
|
||||
conf_flags = Counter(f for c in conflicts for f in c["flags"])
|
||||
|
||||
cov = {"artifact": "ICMS_CONTEXT_COVERAGE_V1", "firebird_writes": 0, "TOTAL_PRODUCTS": len(rows), "engine": ICMS_CONTEXT_ENGINE_VERSION,
|
||||
"ST": {k: st_c.get(k, 0) for k in ("ST_CONFIRMED", "ST_NOT_APPLICABLE", "ST_CANDIDATE", "ST_UNKNOWN")},
|
||||
"ST_reasons": dict(C("ST_REASON")),
|
||||
"ST_WITHHELD": {"ST_ALREADY_WITHHELD_CONFIRMED": wh_c.get("ST_ALREADY_WITHHELD_CONFIRMED", 0), "ST_ALREADY_WITHHELD_NOT_CONFIRMED": wh_c.get("ST_ALREADY_WITHHELD_NOT_CONFIRMED", 0), "ST_ALREADY_WITHHELD_UNKNOWN": wh_c.get("ST_WITHHOLDING_UNKNOWN", 0)},
|
||||
"PRODUCT_ROLE": {k: role_c.get(k, 0) for k in ("RESALE", "OWN_PRODUCTION", "INPUT", "SERVICE", "UNKNOWN")},
|
||||
"CSOSN": {"RESOLVED": cs_c.get("CSOSN_RESOLVED", 0), "ALREADY_CORRECT": already, "CHANGE_REQUIRED": change, "CANDIDATES": cs_c.get("CSOSN_CANDIDATE", 0),
|
||||
"CONTEXT_REQUIRED": cs_c.get("CSOSN_CONTEXT_REQUIRED", 0), "CONFLICT": cs_c.get("CSOSN_CONFLICT", 0), "UNRESOLVED": cs_c.get("CSOSN_UNRESOLVED", 0),
|
||||
"AUTO_APPLY_SAFE": auto_safe, "AUTO_APPLY_READY_IF_ENABLED": ready, "action_class": dict(act_c),
|
||||
"reasons": dict(Counter(r["CSOSN_V2"]["reason_code"] for r in rows).most_common(10))},
|
||||
"CSOSN_HISTORY_of_proposals": {"MATCH": hv_c.get("MATCH", 0), "MISMATCH": hv_c.get("MISMATCH", 0), "UNVALIDATED": hv_c.get("UNVALIDATED", 0) + sum(1 for r in rows if r["CSOSN_V2"]["proposed_csosn"] is None)},
|
||||
"CSOSN_HISTORY_of_proposals_note": "MATCH/MISMATCH/UNVALIDATED counted on products WITH a proposed value; UNVALIDATED additionally counts products with no proposal (nothing to validate).",
|
||||
"NCM_BLOCKED_PRODUCTS": ncm_blocked, "NCM_BLOCKED_ONLY_BY_NCM": ncm_only_blocked,
|
||||
"ST_CONFLICT_AUDIT": {"population": len(conflicts), "primary_class": dict(conf_primary), "any_flag": dict(conf_flags)}}
|
||||
(ROOT / "docs" / "ICMS_CONTEXT_COVERAGE_V1.json").write_text(json.dumps(cov, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
(ROOT / "docs" / "ICMS_PRODUCT_CONTEXT_V1.json").write_text(json.dumps({"artifact": "ICMS_PRODUCT_CONTEXT_V1", "engine": ICMS_CONTEXT_ENGINE_VERSION, "company": {"crt": crt, "uf": uf, "default_csosn": company_default},
|
||||
"products": rows, "st_conflict_audit": conflicts}, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
dry = {"artifact": "CSOSN_DRY_RUN_V2", "firebird_writes": 0, "CSOSN_AUTO_APPLY_SAFE": auto_safe, "CSOSN_ALREADY_CORRECT": already, "CSOSN_CHANGE_REQUIRED": change,
|
||||
"would_change": [], "states": dict(cs_c), "note": "No CSOSN proposal is write-eligible; readiness requires proven ST, withholding, role and ICMS treatment."}
|
||||
(ROOT / "docs" / "CSOSN_DRY_RUN_V2.json").write_text(json.dumps(dry, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
|
||||
n = lambda key, val: sum(1 for r in rows if r[key] == val)
|
||||
req = [
|
||||
{"INPUT": "valid_ncm", "SOURCE_EXPECTED": "product master", "AVAILABLE": "PARTIAL", "AFFECTED_PRODUCTS": ncm_blocked, "AFFECTED_FIELD": "CSOSN", "NEXT_ACTION": "Manual NCM review (3 legacy + 23 invalid); not auto-corrected."},
|
||||
{"INPUT": "st_confirmed_per_product (CEST corroboration)", "SOURCE_EXPECTED": "ERP CEST field filled and equal to the BA Anexo 1 item CEST", "AVAILABLE": "PARTIAL", "AFFECTED_PRODUCTS": st_c.get("ST_CANDIDATE", 0) + st_c.get("ST_UNKNOWN", 0) - ncm_blocked, "AFFECTED_FIELD": "CSOSN", "NEXT_ACTION": "Fill/confirm CEST for ST candidates (description review) — 490 of 521 ERP CEST values are blank."},
|
||||
{"INPUT": "st_withheld_by_supplier (purchase evidence)", "SOURCE_EXPECTED": "purchase items with ICMS_RETIDO/ST values (C000088)", "AVAILABLE": "NO (12 purchase items, all ICMS_RETIDO='N', ST values 0)", "AFFECTED_PRODUCTS": st_c.get("ST_CONFIRMED", 0) + st_c.get("ST_CANDIDATE", 0), "AFFECTED_FIELD": "CSOSN(500 vs other)", "NEXT_ACTION": "Import supplier NF-e XML (ICMS ST/CSOSN 500 on entrada) or obtain vendor confirmation of how retention is recorded."},
|
||||
{"INPUT": "product_role (resale/own production)", "SOURCE_EXPECTED": "C000025.TIPO/PROD_COZINHA + recipe + purchase docs", "AVAILABLE": "PARTIAL", "AFFECTED_PRODUCTS": role_c.get("UNKNOWN", 0), "AFFECTED_FIELD": "CSOSN,CFOP", "NEXT_ACTION": "TIPO_ITEM/PRODUTO_SERVICO are unused; only 7 INPUT and ~13 own-production products are objectively evidenced."},
|
||||
{"INPUT": "product_icms_treatment (taxed vs exempt/not taxed)", "SOURCE_EXPECTED": "official per-NCM ICMS treatment source (none acquired)", "AVAILABLE": False, "AFFECTED_PRODUCTS": st_c.get("ST_NOT_APPLICABLE", 0), "AFFECTED_FIELD": "CSOSN(102 vs 400/300/103)", "NEXT_ACTION": "Decide with accountant/vendor: ERP CST=040 with CSOSN=102 is the emitted practice; needs a documented rule."},
|
||||
]
|
||||
(ROOT / "docs" / "FISCAL_CONTEXT_REQUIREMENTS_V3.json").write_text(json.dumps({"artifact": "FISCAL_CONTEXT_REQUIREMENTS_V3", "requirements": req}, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
md = ["# NEXT_FISCAL_FIELD_IMPLEMENTATION_V3", "",
|
||||
"Mechanical ordering: mapping confirmed, rule available, context available, products affected, history validation, auto-apply possible.", "",
|
||||
"| FIELD | ERP_MAPPING | CONTEXT_AVAILABLE | RULE_AVAILABLE | POTENTIAL_PRODUCTS | HISTORICAL_VALIDATION | NEXT_ACTION |", "|---|---|---|---|---|---|---|",
|
||||
f"| CSOSN | CONFIRMED (HIGH) | NO (retention + ICMS treatment missing) | PARTIAL (ST→500 rule needs withholding proof) | {cs_c.get('CSOSN_CANDIDATE',0)+cs_c.get('CSOSN_CONTEXT_REQUIRED',0)+cs_c.get('CSOSN_CONFLICT',0)} | trusted 1146 items | Acquire supplier ST retention evidence; get accountant decision on 102 vs 400 |",
|
||||
"| CEST | CONFIRMED (HIGH) | PARTIAL | PARTIAL | " + str(st_c.get("ST_CANDIDATE", 0)) + " | weak | CEST fill/confirm drives ST_CONFIRMED |",
|
||||
"| CFOP | CONFIRMED (HIGH) | PARTIAL | YES (ST family) | 29 conflicts | trusted history | Depends on ST_CONFIRMED + retention |",
|
||||
"| IPI | CONFIRMED (HIGH) | YES | YES | 107 | n/a | Maintained |", "",
|
||||
"Blocking inputs with counts: `docs/FISCAL_CONTEXT_REQUIREMENTS_V3.json`."]
|
||||
(ROOT / "docs" / "NEXT_FISCAL_FIELD_IMPLEMENTATION_V3.md").write_text("\n".join(md) + "\n", encoding="utf-8")
|
||||
print(json.dumps({k: cov[k] for k in ("ST", "ST_reasons", "ST_WITHHELD", "PRODUCT_ROLE", "CSOSN", "CSOSN_HISTORY_of_proposals", "NCM_BLOCKED_PRODUCTS", "ST_CONFLICT_AUDIT")}, ensure_ascii=True, indent=1))
|
||||
|
||||
|
||||
main()
|
||||
Reference in New Issue
Block a user