chore: establish project baseline
This commit is contained in:
OpenCode
@@ -0,0 +1,129 @@
|
||||
"""Read-only historical fiscal analysis over TESTESOFICIAIS.FDB: correlates emitted
|
||||
NFC-e items (NFCE_ITENS) and sale/purchase item tables with C000025 product defaults.
|
||||
No DML/DDL. Output: docs/runs/erp_fiscal_history_analysis.json
|
||||
"""
|
||||
import json, os, sys
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent.firebird_reader import _connect # noqa: E402
|
||||
|
||||
for l in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
||||
if "=" in l and not l.startswith("#"):
|
||||
k, _, v = l.partition("="); os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def digits(v):
|
||||
return "".join(c for c in str(v or "") if c.isdigit())
|
||||
|
||||
|
||||
def main():
|
||||
conn = _connect(os.environ); cur = conn.cursor()
|
||||
cur.execute("select CODIGO, CFOP, CST, CSOSN, GRUPO_ICMS, CEST, CLASSIFICACAO_FISCAL, COD_SAIDA_PIS, INCIDENCIA_PISCOFINS, ALIQUOTA from C000025")
|
||||
prod = {s(r[0]): dict(zip(("CFOP", "CST", "CSOSN", "GRUPO_ICMS", "CEST", "NCM", "PIS", "INC", "ALIQ"), map(s, r[1:]))) for r in cur.fetchall()}
|
||||
|
||||
res = {"products": len(prod)}
|
||||
cur.execute("select min(DATA), max(DATA), count(*) from NFCE_ITENS"); mn, mx, n = cur.fetchone()
|
||||
res["nfce_itens"] = {"rows": n, "date_min": str(mn), "date_max": str(mx)}
|
||||
cur.execute("select count(*), min(DATA), max(DATA) from NFCE"); r = cur.fetchone(); res["nfce_docs"] = {"rows": r[0], "min": str(r[1]), "max": str(r[2])}
|
||||
|
||||
cur.execute("select CODPRODUTO, CFOP, CST, GRUPO_ICMS, CSOSN, NCM, CEST, CST_PIS_COFINS, ALIQUOTA, VALOR from NFCE_ITENS")
|
||||
rows = cur.fetchall()
|
||||
joined = 0; agree = defaultdict(Counter); per_prod = defaultdict(lambda: defaultdict(Counter)); orphan = 0
|
||||
for cod, cfop, cst, gicms, csosn, ncm, cest, pispc, aliq, val in rows:
|
||||
p = prod.get(s(cod))
|
||||
if p is None:
|
||||
orphan += 1; continue
|
||||
joined += 1
|
||||
item = {"CFOP": s(cfop), "CST": s(cst), "GRUPO_ICMS": s(gicms), "CSOSN": s(csosn), "NCM": s(ncm), "CEST": s(cest), "PIS": s(pispc)}
|
||||
for f, iv in item.items():
|
||||
if iv is None:
|
||||
agree[f]["ITEM_NULL"] += 1; continue
|
||||
pv = p.get(f if f != "PIS" else "PIS")
|
||||
if f == "CEST":
|
||||
agree[f]["MATCH" if digits(iv) == digits(pv) else "MISMATCH"] += 1
|
||||
elif f == "PIS":
|
||||
agree[f]["MATCH" if iv.lstrip("0") == (pv or "").lstrip("0") else "MISMATCH"] += 1
|
||||
else:
|
||||
agree[f]["MATCH" if iv == pv else "MISMATCH"] += 1
|
||||
per_prod[s(cod)][f][iv] += 1
|
||||
res["nfce_join"] = {"items_joined_to_C000025": joined, "orphan_items": orphan,
|
||||
"agreement_item_vs_product_default": {f: dict(c) for f, c in agree.items()}}
|
||||
res["nfce_products_seen"] = len(per_prod)
|
||||
res["variability"] = {f: sum(1 for pp in per_prod.values() if len(pp[f]) > 1) for f in ("CFOP", "CST", "GRUPO_ICMS", "CSOSN", "CEST", "NCM")}
|
||||
res["nfce_item_value_distributions"] = {}
|
||||
for f, idx in (("CFOP", 1), ("CST", 2), ("GRUPO_ICMS", 3), ("CSOSN", 4), ("CST_PIS_COFINS", 7)):
|
||||
res["nfce_item_value_distributions"][f] = Counter(s(r[idx]) for r in rows).most_common(8)
|
||||
|
||||
# Sale/purchase item tables (C000032 sales, C000062 purchase items) vs product defaults
|
||||
for table, extra in (("C000032", "CST, CFOP"), ("C000062", "CST, CFOP")):
|
||||
cur.execute(f"select CODPRODUTO, {extra} from {table}")
|
||||
c = Counter(); cfopc = Counter(); cstc = Counter()
|
||||
for cod, cst, cfop in cur.fetchall():
|
||||
p = prod.get(s(cod))
|
||||
if p is None: c["ORPHAN"] += 1; continue
|
||||
c["JOINED"] += 1
|
||||
if s(cst): cstc["MATCH" if s(cst) == p["CST"] else "MISMATCH"] += 1
|
||||
if s(cfop): cfopc["MATCH" if s(cfop) == p["CFOP"] else "MISMATCH"] += 1
|
||||
res[table] = {"joined": dict(c), "cst_vs_product": dict(cstc), "cfop_vs_product": dict(cfopc)}
|
||||
|
||||
# Natureza de operação (C000030) and operation-context sources
|
||||
cur.execute("select TIPO, count(*) from C000030 group by TIPO"); res["C000030_tipo"] = dict(cur.fetchall())
|
||||
cur.execute("select substring(CFOP from 1 for 1), count(*) from C000030 group by 1 order by 1"); res["C000030_cfop_first_digit"] = dict(cur.fetchall())
|
||||
cur.execute("select CFOP from C000030 where CFOP in ('5102','5405','5656','5929','6102','1102')"); res["C000030_has_cfops"] = [s(r[0]) for r in cur.fetchall()]
|
||||
cur.execute("select IND_CFOP_ENTRADA, IND_CFOP_SAIDA, IND_CFOP_PERDA from C000001"); res["C000001_cfop_defaults"] = [s(x) for x in cur.fetchone()]
|
||||
cur.execute("select OPTANTE_SIMPLES, OPTANTE_SUPER_SIMPLES, CRT, ICMS_CSOSN, SUBSTITUTO_TRIBUTARIO, CONTRIBUINTE_IPI, UF from C000004"); res["C000004"] = [s(x) for x in cur.fetchone()]
|
||||
cur.execute("select EMITENTE_CRT from EMISSOR"); res["EMISSOR_CRT"] = [s(x) for x in cur.fetchone()]
|
||||
cur.execute("select CFOP, count(*) from C000062 group by CFOP"); res["C000062_cfop"] = {s(a): b for a, b in cur.fetchall()}
|
||||
cur.execute("select CST, count(*) from C000062 group by CST"); res["C000062_cst"] = {s(a): b for a, b in cur.fetchall()}
|
||||
cur.execute("select count(*), sum(case when MARGEM_AGREGADA <> 0 then 1 else 0 end), sum(case when ICMS_SUB <> 0 then 1 else 0 end) from C000062"); res["C000062_mva_usage"] = list(cur.fetchone())
|
||||
cur.execute("select count(*) from PRODUTOS_MARGEM_AGREGADA"); res["PRODUTOS_MARGEM_AGREGADA_rows"] = cur.fetchone()[0]
|
||||
cur.execute("select count(*) from C000025 where MARGEM_AGREGADA is not null"); res["C000025_MARGEM_AGREGADA_nonnull"] = cur.fetchone()[0]
|
||||
|
||||
# CEST history for the 15 CEST-candidate products (from the decision matrix)
|
||||
matrix = json.loads((ROOT / "docs" / "FISCAL_DECISION_MATRIX_V1.json").read_text(encoding="utf-8"))
|
||||
cand = [m for m in matrix["products"] if m["fields"]["CEST"]["status"] == "RULE_CANDIDATE"]
|
||||
cest_hist = []
|
||||
for m in cand:
|
||||
code = s(m["CODIGO"])
|
||||
hist = Counter(digits(k) for k, c in per_prod.get(code, {}).get("CEST", {}).items() for _ in range(c) if digits(k))
|
||||
cest_hist.append({"CODIGO": code, "DESCRIPTION": m["DESCRIPTION"], "NCM": m["NCM"], "erp_current_cest": m["fields"]["CEST"]["current_value"], "historical_cest": dict(hist)})
|
||||
res["cest_candidates_history"] = cest_hist
|
||||
conn.close()
|
||||
|
||||
# Shadow validation: decision matrix engine value vs historical NFC-e item value
|
||||
val = {}
|
||||
for dom, hist_field in (("NCM", "NCM"), ("CEST", "CEST"), ("CFOP", "CFOP"), ("CST_ICMS", "CST"), ("CSOSN", "CSOSN"), ("PIS_CST", "PIS")):
|
||||
val[dom] = Counter()
|
||||
by_code = {s(m["CODIGO"]): m["fields"] for m in matrix["products"]}
|
||||
conn = _connect(os.environ); cur = conn.cursor()
|
||||
cur.execute("select first 5000 CODPRODUTO, NCM, CEST, CFOP, CST, CSOSN, CST_PIS_COFINS from NFCE_ITENS where NCM is not null")
|
||||
n_sim = 0
|
||||
for cod, ncm, cest, cfop, cst, csosn, pispc in cur.fetchall():
|
||||
f = by_code.get(s(cod))
|
||||
if f is None: continue
|
||||
n_sim += 1
|
||||
hist = {"NCM": s(ncm), "CEST": s(cest), "CFOP": s(cfop), "CST_ICMS": s(cst), "CSOSN": s(csosn), "PIS_CST": s(pispc)}
|
||||
for dom, hv in hist.items():
|
||||
rec = f[dom]
|
||||
ev = rec["proposed_value"] if rec["status"] == "RULE_RESOLVED" else None
|
||||
if rec["status"] == "RULE_RESOLVED" and dom == "NCM":
|
||||
ev = rec["current_value"]
|
||||
if ev is None:
|
||||
val[dom]["UNRESOLVED"] += 1
|
||||
elif digits(ev) == digits(hv) if dom in ("NCM", "CEST") else str(ev) == str(hv):
|
||||
val[dom]["MATCH"] += 1
|
||||
else:
|
||||
val[dom]["MISMATCH"] += 1
|
||||
res["shadow_validation"] = {"items": n_sim, **{d: dict(c) for d, c in val.items()}}
|
||||
conn.close()
|
||||
print(json.dumps(res, ensure_ascii=False, indent=1, default=str))
|
||||
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Read-only extras: (1) history agreement restricted to identity-consistent joins (item NCM == product NCM),
|
||||
(2) CEST historical support for the CEST-candidate products against the Hub's official candidates."""
|
||||
import json, os, sys, urllib.request
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent.firebird_reader import _connect
|
||||
for l in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
||||
if "=" in l and not l.startswith("#"):
|
||||
k, _, v = l.partition("="); os.environ.setdefault(k.strip(), v.strip())
|
||||
s = lambda v: None if v is None else str(v).strip()
|
||||
dg = lambda v: "".join(c for c in str(v or "") if c.isdigit())
|
||||
c = _connect(os.environ).cursor()
|
||||
c.execute("select CODIGO, PRODUTO, CLASSIFICACAO_FISCAL, CFOP, CST, CSOSN, CEST, COD_SAIDA_PIS from C000025")
|
||||
P = {s(r[0]): dict(zip(("D", "NCM", "CFOP", "CST", "CSOSN", "CEST", "PIS"), map(s, r[1:]))) for r in c.fetchall()}
|
||||
c.execute("select CODPRODUTO, NCM, CFOP, CST, CSOSN, CEST, CST_PIS_COFINS, GRUPO_ICMS from NFCE_ITENS where NCM is not null")
|
||||
consistent = Counter(); inconsistent = 0; agree = {f: Counter() for f in ("CFOP", "CST", "CSOSN", "CEST", "PIS")}
|
||||
hist_cest = {}
|
||||
for cod, ncm, cfop, cst, csosn, cest, pis, gi in c.fetchall():
|
||||
p = P.get(s(cod))
|
||||
if not p: continue
|
||||
if s(ncm) != p["NCM"]:
|
||||
inconsistent += 1
|
||||
else:
|
||||
consistent["items"] += 1
|
||||
for f, iv in (("CFOP", s(cfop)), ("CST", s(cst)), ("CSOSN", s(csosn)), ("CEST", s(cest)), ("PIS", s(pis))):
|
||||
pv = p[f]
|
||||
if iv is None: continue
|
||||
eq = dg(iv) == dg(pv) if f == "CEST" else (iv.lstrip("0") == (pv or "").lstrip("0") if f == "PIS" else iv == pv)
|
||||
agree[f]["MATCH" if eq else "MISMATCH"] += 1
|
||||
if dg(cest):
|
||||
hist_cest.setdefault(s(cod), Counter())[dg(cest)] += 1
|
||||
out = {"ncm_identity_consistent_items": consistent["items"], "ncm_identity_inconsistent_items": inconsistent,
|
||||
"agreement_consistent_joins_only": {f: dict(v) for f, v in agree.items()}}
|
||||
|
||||
# also all-time (any NCM) CEST history per product
|
||||
c.execute("select CODPRODUTO, CEST from NFCE_ITENS where CEST is not null and trim(CEST) <> ''")
|
||||
allc = {}
|
||||
for cod, cest in c.fetchall():
|
||||
allc.setdefault(s(cod), Counter())[dg(cest)] += 1
|
||||
|
||||
tok = [l.split("=", 1)[1].strip() for l in open(ROOT / ".env", encoding="utf-8") if l.startswith("AGENT_DEV_TOKEN=")][0]
|
||||
matrix = json.loads((ROOT / "docs" / "FISCAL_DECISION_MATRIX_V1.json").read_text(encoding="utf-8"))
|
||||
cands = [m for m in matrix["products"] if m["fields"]["CEST"]["status"] == "RULE_CANDIDATE"]
|
||||
req = urllib.request.Request("http://localhost:8000/v1/classify/batch", data=json.dumps({"products": [{"current_ncm": m["NCM"], "description": m["DESCRIPTION"] or ""} for m in cands]}).encode(),
|
||||
headers={"Content-Type": "application/json", "Authorization": "Bearer " + tok})
|
||||
decs = json.load(urllib.request.urlopen(req, timeout=600))["results"]
|
||||
rows = []
|
||||
for m, d in zip(cands, decs):
|
||||
code = s(m["CODIGO"]); off = {dg(x["code"]): x for x in d["classification"].get("cest_candidates", [])}
|
||||
h = dict(allc.get(code, {}))
|
||||
supported = [k for k in h if k in off]
|
||||
mva_ceps = {dg(x.get("cest")) for x in d["classification"].get("ba_st_mva_candidates", [])}
|
||||
rows.append({"CODIGO": code, "DESCRIPTION": m["DESCRIPTION"], "NCM": m["NCM"], "official_candidates": len(off), "historical_cest": h,
|
||||
"historical_in_official_candidates": supported, "historical_in_ba_st_mva": [k for k in h if k in mva_ceps],
|
||||
"product_ncm_matches_item_ncm": None,
|
||||
"verdict": "CEST_HISTORICALLY_SUPPORTED" if len(h) == 1 and len(supported) == 1 else ("HISTORY_MULTIPLE_OR_UNSUPPORTED" if h else "NO_HISTORY")})
|
||||
out["cest_candidates"] = rows
|
||||
out["cest_historically_supported"] = sum(1 for r in rows if r["verdict"] == "CEST_HISTORICALLY_SUPPORTED")
|
||||
print(json.dumps(out, ensure_ascii=True, indent=1))
|
||||
@@ -0,0 +1,85 @@
|
||||
"""READ-ONLY company-context audit + resolver retest over every .FDB in a directory (default ./Banco).
|
||||
Writes docs/COMPANY_CONTEXT_RESOLUTION_V1.json (evidence) and docs/MULTI_DATABASE_COMPANY_CONTEXT_V1.json (outcomes).
|
||||
The full CNPJ is never written: masked value + short SHA-256 fingerprint only.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
import fdb # noqa: E402
|
||||
from gti_fiscal_agent import company_context as cc # noqa: E402
|
||||
|
||||
for line in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def evidence_for(path):
|
||||
conn = fdb.connect(database=str(path), user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
return cc.collect_evidence(conn.cursor())
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def table(ev):
|
||||
rows = []
|
||||
emissor_cnpjs = {cc.digits(e["cnpj"]) for e in ev["emissor"]}
|
||||
docs_by_cnpj = {}
|
||||
for d in ev["nfce_keys"]:
|
||||
docs_by_cnpj[cc.digits(d["cnpj"])] = docs_by_cnpj.get(cc.digits(d["cnpj"]), 0) + d["n"]
|
||||
for i, c in enumerate(ev["c000004"]):
|
||||
cn = cc.digits(c.get("cnpj"))
|
||||
em = next((e for e in ev["emissor"] if cc.digits(e["cnpj"]) == cn), None)
|
||||
n_docs = docs_by_cnpj.get(cn, 0); n_fil = ev["nf_by_filial"].get(c.get("code"), 0)
|
||||
active = bool(em) and n_docs > 0
|
||||
rows.append({"C000004_CODE": c.get("code"), "CNPJ": cc.mask_cnpj(cn), "CNPJ_FINGERPRINT": cc.cnpj_fingerprint(cn), "UF_C000004": c.get("uf"), "CRT_C000004": c.get("crt"), "OPTANTE_SIMPLES": c.get("optante"),
|
||||
"EMISSOR_ID": (ev["emissor"].index(em) + 1) if em else None, "UF_EMISSOR": em.get("uf") if em else None, "CRT_EMISSOR": em.get("crt") if em else None,
|
||||
"DOCUMENT_COUNT": {"nfce_keys_with_this_cnpj": n_docs, "nf_C000061_with_this_CODFILIAL": n_fil}, "IS_ACTIVE_CANDIDATE": active,
|
||||
"EVIDENCE": [x for x in (("EMISSOR.EMITENTE_CNPJ equals this CNPJ" if em else "no EMISSOR row with this CNPJ"), (f"{n_docs} emitted NFC-e keys carry this CNPJ" if n_docs else "no emitted NFC-e key carries this CNPJ"),
|
||||
(f"{n_fil} NF-e rows use CODFILIAL={c.get('code')}" if n_fil else None)) if x]})
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
directory = Path(sys.argv[1]) if len(sys.argv) > 1 else ROOT / "Banco"
|
||||
evid, outcomes = {}, {}
|
||||
for p in sorted(directory.glob("*.FDB")):
|
||||
ev = evidence_for(p)
|
||||
rows = table(ev)
|
||||
no_cfg = cc.resolve(ev, {})
|
||||
crt_cfg = cc.resolve(ev, {"crt_source": "EMISSOR"})
|
||||
multi = len(ev["c000004"]) > 1
|
||||
neg = {}
|
||||
if multi: # negative tests on real data: wrong company / unknown CNPJ must not resolve silently
|
||||
other = next((c for c in ev["c000004"] if not next((e for e in ev["emissor"] if cc.digits(e["cnpj"]) == cc.digits(c["cnpj"])), None)), None)
|
||||
neg["configured_non_emitting_company"] = cc.resolve(ev, {"company_code": other["code"], "crt_source": "EMISSOR"})["classification"] if other else None
|
||||
neg["configured_unknown_cnpj"] = cc.resolve(ev, {"company_cnpj": "00000000000000"})["classification"]
|
||||
crt_authority = {"EMISSOR_values": sorted({e.get("crt") for e in ev["emissor"] if e.get("crt")}), "C000004_selected_row": no_cfg.get("crt_candidates") or {"C000004": no_cfg.get("company_crt")}}
|
||||
evid[p.stem] = {"c000004_rows": len(ev["c000004"]), "emissor_rows": len(ev["emissor"]), "companies": rows, "nfce_key_groups": [{"cUF": d["cuf"], "cnpj": cc.mask_cnpj(d["cnpj"]), "model": "65", "n": d["n"]} for d in ev["nfce_keys"]],
|
||||
"nf_by_filial": ev["nf_by_filial"]}
|
||||
outcomes[p.stem] = {"without_config": cc.preflight_report(no_cfg), "with_COMPANY_CRT_SOURCE=EMISSOR": cc.preflight_report(crt_cfg), "negative_tests": neg,
|
||||
"crt_source_status": no_cfg.get("crt_status") or ("CRT_SOURCE_CONFLICT" if "CRT_SOURCE_CONFLICT" in " ".join(no_cfg["reasons"]) else "CRT_SOURCE_UNKNOWN"),
|
||||
"uf_source_status": no_cfg.get("uf_status") or "UF_SOURCE_UNKNOWN", "resolution_class": no_cfg["classification"]}
|
||||
classes = {}
|
||||
for k, v in outcomes.items():
|
||||
classes.setdefault(v["resolution_class"], []).append(k)
|
||||
authority = {
|
||||
"CRT": {"emission_authority": "EMISSOR.EMITENTE_CRT", "evidence": ["GtiVendas_NFCe.exe (NFC-e) and Envio NF-e.exe (NF-e) bind QrEmissor.EMITENTE_CRT/EMITENTE_UF/EMITENTE_CNPJ and never reference C000004.CRT or OPTANTE_SIMPLES",
|
||||
"SGV_5.0.6.124.exe (ERP) binds C000004.CRT (qrFilialCRT/QrSPEDFilialCRT) and OPTANTE_SIMPLES for the filial registry/SPED and has no EMITENTE_* binding",
|
||||
"emitted XML is not stored in the database (C000061.NFE_XML and XML_MES.XML hold file paths), so the CRT tag of issued documents could not be read directly"],
|
||||
"status": "CRT_SOURCE_CONFLICT where the two values differ; the resolver refuses to choose without COMPANY_CRT_SOURCE"},
|
||||
"UF": {"status": "UF_SOURCE_CONFIRMED", "evidence": "EMISSOR.EMITENTE_UF, C000004.UF and the cUF of 100% of emitted NFC-e keys agree in all 4 databases"},
|
||||
"COMPANY": {"status": "confirmed", "evidence": "EMISSOR.EMITENTE_CNPJ equals exactly one C000004 CNPJ in every database and 100% of emitted NFC-e keys carry that CNPJ; other C000004 rows (RINCAO) have no documents"}}
|
||||
(ROOT / "docs" / "COMPANY_CONTEXT_RESOLUTION_V1.json").write_text(json.dumps({"artifact": "COMPANY_CONTEXT_RESOLUTION_V1", "firebird_writes": 0, "authority": authority, "databases": evid}, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
(ROOT / "docs" / "MULTI_DATABASE_COMPANY_CONTEXT_V1.json").write_text(json.dumps({"artifact": "MULTI_DATABASE_COMPANY_CONTEXT_V1", "firebird_writes": 0, "resolution_classes": classes, "databases": outcomes}, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
print(json.dumps({k: (v["resolution_class"], v["with_COMPANY_CRT_SOURCE=EMISSOR"]["COMPANY_CONTEXT"], v["with_COMPANY_CRT_SOURCE=EMISSOR"]["company_crt"], v["negative_tests"]) for k, v in outcomes.items()}, ensure_ascii=False))
|
||||
print(classes)
|
||||
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Read-only audit of fiscal-field semantics in C000025 (types, value distributions, comments)."""
|
||||
import json, os, sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent.firebird_reader import _connect # noqa: E402
|
||||
|
||||
FIELDS = ["CLASSIFICACAO_FISCAL","NCM","IPI","EX_TIPI","CEST","CFOP_ENTRADA","CFOP","CST_ENTRADA","CST","ALIQUOTA","CSOSN",
|
||||
"COD_ENTRADA_PIS","COD_SAIDA_PIS","ALIQ_PIS","COD_ENTRADA_COFINS","COD_SAIDA_COFINS","ALIQ_COFINS","CST_IPI","ORIGEM","IAT","INCIDENCIA_PISCOFINS","IPPT"]
|
||||
|
||||
def load_env():
|
||||
for line in (ROOT/".env").read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, _, v = line.partition("="); os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
def main():
|
||||
load_env()
|
||||
conn = _connect(os.environ); cur = conn.cursor()
|
||||
cur.execute("""SELECT rf.RDB$FIELD_NAME, f.RDB$FIELD_TYPE, f.RDB$FIELD_LENGTH, f.RDB$FIELD_PRECISION, f.RDB$FIELD_SCALE, f.RDB$CHARACTER_LENGTH, rf.RDB$DESCRIPTION, rf.RDB$DEFAULT_SOURCE
|
||||
FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON rf.RDB$FIELD_SOURCE=f.RDB$FIELD_NAME WHERE rf.RDB$RELATION_NAME='C000025'""")
|
||||
meta = {}
|
||||
for r in cur.fetchall():
|
||||
name = r[0].strip()
|
||||
if name in FIELDS:
|
||||
meta[name] = {"type_code": r[1], "len": r[5] or r[2], "precision": r[3], "scale": r[4], "comment": str(r[6]) if r[6] else None, "default": str(r[7]) if r[7] else None}
|
||||
out = {}
|
||||
for f in FIELDS:
|
||||
if f not in meta:
|
||||
out[f] = {"exists": False}; continue
|
||||
cur.execute(f"SELECT {f}, COUNT(*) FROM C000025 GROUP BY {f}")
|
||||
rows = sorted(((str(v).strip() if v is not None else None, c) for v, c in cur.fetchall()), key=lambda x: -x[1])
|
||||
out[f] = {"exists": True, **meta[f], "distinct": len(rows), "top": rows[:12]}
|
||||
# relations that may hold context (company/tax config), names only
|
||||
cur.execute("SELECT RDB$RELATION_NAME FROM RDB$RELATIONS WHERE RDB$SYSTEM_FLAG=0 AND (RDB$RELATION_NAME LIKE '%EMPRESA%' OR RDB$RELATION_NAME LIKE '%CFOP%' OR RDB$RELATION_NAME LIKE '%TRIB%' OR RDB$RELATION_NAME LIKE '%ICMS%' OR RDB$RELATION_NAME LIKE '%CST%' OR RDB$RELATION_NAME LIKE '%NATUREZA%')")
|
||||
out["_related_tables"] = [r[0].strip() for r in cur.fetchall()]
|
||||
conn.close()
|
||||
print(json.dumps(out, ensure_ascii=False, indent=1, default=str))
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,129 @@
|
||||
"""READ-ONLY audit of any SGV Firebird database (restaurante / mercado / oficina ...).
|
||||
|
||||
python audit_firebird_database.py <FDB> [--json out.json]
|
||||
|
||||
Credentials come from the process environment (FB_USER / FB_PASSWORD, optional FIREBIRD_CLIENT_LIBRARY) or the
|
||||
local ignored .env — never from the command line, never printed. The connection uses read-only isolation and the
|
||||
script executes SELECT statements only. It never enables writes for the audited database.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src")); sys.path.insert(0, str(ROOT / "src"))
|
||||
|
||||
FISCAL_FIELDS = ["CLASSIFICACAO_FISCAL", "NCM", "CFOP", "CFOP_ENTRADA", "CST", "CST_ENTRADA", "ALIQUOTA", "CEST", "CSOSN", "IPI", "EX_TIPI", "CST_IPI",
|
||||
"COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS", "ORIGEM", "COD_ORIGEM_MERCADORIA",
|
||||
"IAT", "INCIDENCIA_PISCOFINS", "IPPT", "GRUPO_ICMS", "SITUACAO_TRIBUTARIA", "MARGEM_AGREGADA"]
|
||||
# Firebird RDB$FIELD_TYPE -> family
|
||||
_FAMILY = {7: "int", 8: "int", 16: "int", 10: "float", 11: "float", 27: "float", 12: "date", 13: "time", 35: "timestamp", 14: "text", 37: "text", 261: "blob"}
|
||||
# expected type family per field (from docs/ERP_FISCAL_FIELD_MAP_V1.json); text fields may also be char
|
||||
EXPECTED_FAMILY = {"CLASSIFICACAO_FISCAL": "text", "NCM": "text", "CFOP": "text", "CFOP_ENTRADA": ("text", "int"), "CST": "text", "CST_ENTRADA": ("text", "int"), "ALIQUOTA": ("float", "int"),
|
||||
"CEST": "text", "CSOSN": "text", "IPI": ("float", "int"), "EX_TIPI": "text", "CST_IPI": ("text", "int"), "COD_ENTRADA_PIS": "text", "COD_SAIDA_PIS": "text",
|
||||
"ALIQ_PIS": "text", "COD_ENTRADA_COFINS": "text", "COD_SAIDA_COFINS": "text", "ALIQ_COFINS": "text", "ORIGEM": "text", "COD_ORIGEM_MERCADORIA": "int",
|
||||
"IAT": "text", "INCIDENCIA_PISCOFINS": "text", "IPPT": "text", "GRUPO_ICMS": "text", "SITUACAO_TRIBUTARIA": "text", "MARGEM_AGREGADA": ("float", "int")}
|
||||
|
||||
|
||||
def type_family(code):
|
||||
return _FAMILY.get(code, f"other:{code}")
|
||||
|
||||
|
||||
def ncm_bucket(value):
|
||||
v = "" if value is None else str(value).strip()
|
||||
if not v:
|
||||
return "blank"
|
||||
return "eight_digits" if re.fullmatch(r"\d{8}", v) else "other_format"
|
||||
|
||||
|
||||
def compat(columns):
|
||||
"""columns: {FIELD: type_code}. Returns per-field compatibility against the expected families."""
|
||||
out = {}
|
||||
for f, exp in EXPECTED_FAMILY.items():
|
||||
if f not in columns:
|
||||
out[f] = "MISSING"
|
||||
continue
|
||||
fam = type_family(columns[f]); exp = (exp,) if isinstance(exp, str) else exp
|
||||
out[f] = "OK" if fam in exp else f"TYPE_DIFFERS({fam} not in {'/'.join(exp)})"
|
||||
return out
|
||||
|
||||
|
||||
def load_env():
|
||||
env_file = ROOT / ".env"
|
||||
if env_file.exists():
|
||||
for line in env_file.read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, _, v = line.partition("="); os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def audit(path, env):
|
||||
import fdb
|
||||
from gti_discovery.fingerprint import build_fingerprint
|
||||
from gti_fiscal_agent.firebird_reader import parse_company_context
|
||||
conn = fdb.connect(database=path, user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
res = {"database": Path(path).name, "mode": "READ_ONLY", "firebird_server_version": getattr(conn, "server_version", None), "ods": f"{getattr(conn, 'ods_version', '?')}.{getattr(conn, 'ods_minor_version', '?')}"}
|
||||
cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH), f.RDB$FIELD_SCALE
|
||||
FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE
|
||||
JOIN RDB$RELATIONS r ON r.RDB$RELATION_NAME = rf.RDB$RELATION_NAME WHERE COALESCE(r.RDB$SYSTEM_FLAG, 0) = 0""")
|
||||
meta = {}
|
||||
for t, c, ty, ln, sc in cur.fetchall():
|
||||
meta.setdefault(t, []).append({"name": c, "type": ty, "length": ln, "scale": sc})
|
||||
res["schema_fingerprint"] = build_fingerprint({"tables": [{"name": t, "columns": cols} for t, cols in sorted(meta.items())]})
|
||||
res["tables"] = len(meta)
|
||||
# company context
|
||||
if "C000004" in meta:
|
||||
try:
|
||||
cur.execute("SELECT UF, CRT FROM C000004"); res["company"] = {"status": "OK", **parse_company_context(cur.fetchall())}
|
||||
except ValueError as e:
|
||||
res["company"] = {"status": "UNAVAILABLE", "error": str(e)}
|
||||
else:
|
||||
res["company"] = {"status": "C000004_ABSENT"}
|
||||
# products
|
||||
res["C000025_present"] = "C000025" in meta
|
||||
if res["C000025_present"]:
|
||||
cols = {c["name"]: c["type"] for c in meta["C000025"]}
|
||||
res["field_mapping_compatibility"] = compat(cols)
|
||||
res["field_mapping_compatibility_summary"] = {k: sum(1 for v in res["field_mapping_compatibility"].values() if (v == k or v.startswith(k))) for k in ("OK", "MISSING", "TYPE_DIFFERS")}
|
||||
cur.execute("SELECT COUNT(*) FROM C000025"); res["product_count"] = cur.fetchone()[0]
|
||||
ncm_col = "CLASSIFICACAO_FISCAL" if "CLASSIFICACAO_FISCAL" in cols else None
|
||||
if ncm_col:
|
||||
cur.execute(f"SELECT {ncm_col}, COUNT(*) FROM C000025 GROUP BY {ncm_col}")
|
||||
b = {"eight_digits": 0, "blank": 0, "other_format": 0}
|
||||
for v, n in cur.fetchall():
|
||||
b[ncm_bucket(v)] += n
|
||||
res["ncm_coverage"] = {**b, "share_eight_digits": round(b["eight_digits"] / res["product_count"], 4) if res["product_count"] else None}
|
||||
pop = {}
|
||||
for f in FISCAL_FIELDS:
|
||||
if f in cols:
|
||||
cur.execute(f"SELECT COUNT(*) FROM C000025 WHERE {f} IS NOT NULL AND TRIM(CAST({f} AS VARCHAR(60))) <> ''")
|
||||
pop[f] = cur.fetchone()[0]
|
||||
res["fiscal_field_population"] = pop
|
||||
return res
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("fdb"); ap.add_argument("--json")
|
||||
a = ap.parse_args(argv)
|
||||
load_env()
|
||||
if not os.environ.get("FB_USER") or not os.environ.get("FB_PASSWORD"):
|
||||
raise SystemExit("FB_USER/FB_PASSWORD must be set in the environment (never on the command line)")
|
||||
if not Path(a.fdb).is_file():
|
||||
raise SystemExit(f"database file not found: {a.fdb}")
|
||||
res = audit(a.fdb, os.environ)
|
||||
text = json.dumps(res, ensure_ascii=False, indent=1, default=str)
|
||||
if a.json:
|
||||
Path(a.json).write_text(text, encoding="utf-8")
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Semantic audit of the local-completion writes (Local Completion V2). Offline: reads docs/runs/local_completion/* only.
|
||||
Verdicts: SEMANTICALLY_PROVEN | NEEDS_REVIEW | INVALID_RULE. Emits docs/LOCAL_WRITE_SEMANTIC_AUDIT_V2.json (+ per-field tables) and rollback_plan.json {db: {field: [codes]}} for every write that is not PROVEN."""
|
||||
import csv, gzip, json, pickle, re, sys
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]; ST = ROOT / "docs" / "runs" / "local_completion"
|
||||
DBS = ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]
|
||||
tipi = defaultdict(dict)
|
||||
for line in (ST / "tipi_export.txt").read_text(encoding="utf-8").splitlines():
|
||||
ncm, ex, rate, txt = line.split("|"); tipi[ncm][ex] = (float(rate) if rate else None, txt)
|
||||
st_prefixes = {re.sub(r"\D", "", x) for x in (ST / "ba_st_ncm_export.txt").read_text(encoding="utf-8").split()}
|
||||
ST_CODES = {"010", "030", "070", "060", "201", "202", "203", "500", "10", "30", "70", "60"}
|
||||
audit = defaultdict(list); plan = defaultdict(lambda: defaultdict(list)); table = []
|
||||
for n in DBS:
|
||||
b = pickle.load(gzip.open(ST / f"{n}_before.pkl.gz", "rb")); cols = b["columns"]; rows = b["rows"]; g = lambda code, c: rows[code][cols.index(c)] # noqa: E731
|
||||
dry = {}
|
||||
for l in gzip.open(ST / f"{n}_dryrun.ndjson.gz", "rt", encoding="utf-8"):
|
||||
r = json.loads(l); dry[(r["codigo"], r["field"])] = r
|
||||
for l in gzip.open(ST / f"{n}_write_audit.ndjson.gz", "rt", encoding="utf-8"):
|
||||
a = json.loads(l); code = a["product_code"]
|
||||
for f, v in a["fields"].items():
|
||||
d = dry[(code, f)]; ncm = re.sub(r"\D", "", str(g(code, "CLASSIFICACAO_FISCAL") or "")); rule = d["reason"]; ev = ""; verdict = "SEMANTICALLY_PROVEN"
|
||||
if f == "COD_ORIGEM_MERCADORIA":
|
||||
verdict, ev = "INVALID_RULE", f"CST digit is padding: CST={g(code,'CST')!r} while declared-origin products (1/2) also carry CST digit 0"
|
||||
elif f == "IPI":
|
||||
ncm_ok = dry[(code, "CLASSIFICACAO_FISCAL")]["status"] == "ALREADY_CORRECT" and len(ncm) == 8
|
||||
base = tipi.get(ncm, {}).get("")
|
||||
ex_prod = str(g(code, "EX_TIPI") or "").strip()
|
||||
if not ncm_ok or base is None or base[0] is None or abs(base[0] - float(v["after"])) > 1e-9:
|
||||
verdict, ev = "INVALID_RULE", "TIPI numeric base rate does not match"
|
||||
elif g(code, "IPI") is not None:
|
||||
verdict, ev = "NEEDS_REVIEW", "overwrote a non-null IPI"
|
||||
elif ex_prod:
|
||||
verdict, ev = "NEEDS_REVIEW", f"product declares EX_TIPI={ex_prod}"
|
||||
else:
|
||||
ev = f"NCM {ncm} valid; TIPI base row rate={base[0]} (text {base[1]!r}); EX rows for NCM: {sorted(k for k in tipi[ncm] if k)}; before NULL"
|
||||
elif f == "CFOP":
|
||||
cst, csosn = str(g(code, "CST") or "").strip(), str(g(code, "CSOSN") or "").strip()
|
||||
st_decl = csosn in {"500", "201", "202", "203"} or cst in {"060", "010", "070", "030"}
|
||||
if d["reason"] == "PROVEN_ROLE_AND_ST_CONTRADICT_DECLARED_CFOP" and v["before"] == "5405" and v["after"] == "5102":
|
||||
verdict, ev = ("INVALID_RULE", f"ERP declares ICMS-ST already retained (CST={cst}, CSOSN={csosn}); CFOP 5102 contradicts the product's own CST/CSOSN, official table alone cannot override it") if st_decl else ("NEEDS_REVIEW", "ST not declared")
|
||||
elif v["after"] == "5101":
|
||||
ipp = dry.get((code, "IPPT"), {}).get("reason", "")
|
||||
strong = "COMPOSICAO" in ipp or "RECEITA_INGREDIENTES" in ipp
|
||||
verdict, ev = ("SEMANTICALLY_PROVEN", f"own production evidence: {ipp}") if strong else ("NEEDS_REVIEW", f"only kitchen-routing flag: {ipp}")
|
||||
elif v["after"] == "5102" and v["before"] is None:
|
||||
in_st = any(ncm.startswith(p) for p in st_prefixes if len(p) in (4, 6, 8))
|
||||
verdict, ev = ("SEMANTICALLY_PROVEN", f"resale; CST={cst} CSOSN={csosn} not ST; NCM {ncm} not in BA ST prefixes") if (not st_decl and not in_st) else ("NEEDS_REVIEW", "possible ST")
|
||||
elif f == "CEST":
|
||||
cest = str(v["after"]); fmt = f"{cest[:2]}.{cest[2:5]}.{cest[5:]}"
|
||||
verdict, ev = "SEMANTICALLY_PROVEN", f"official CEST dataset: NCM {ncm} linked exactly to {fmt} (single candidate); before blank"
|
||||
elif f == "IPPT":
|
||||
strong = "COMPOSICAO" in rule or "RECEITA_INGREDIENTES" in rule
|
||||
verdict, ev = ("SEMANTICALLY_PROVEN", f"ERP evidence: {rule}") if strong else ("NEEDS_REVIEW", f"only kitchen-routing flag: {rule}")
|
||||
else:
|
||||
verdict, ev = "NEEDS_REVIEW", "no audit rule"
|
||||
audit[(n, f)].append(verdict)
|
||||
if verdict != "SEMANTICALLY_PROVEN":
|
||||
plan[n][f].append(code)
|
||||
if f in ("CFOP", "CEST", "IPPT"):
|
||||
table.append({"database": n, "product": code, "field": f, "before": v["before"], "after": v["after"], "rule": rule, "evidence": ev, "verdict": verdict})
|
||||
summary = {f"{n}|{f}": dict(Counter(v)) for (n, f), v in audit.items()}
|
||||
(ROOT / "docs" / "LOCAL_WRITE_SEMANTIC_AUDIT_V2.json").write_text(json.dumps({"summary": summary, "small_field_writes": table}, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
with open(ROOT / "docs" / "LOCAL_WRITE_SEMANTIC_AUDIT_V2_CFOP_CEST_IPPT.csv", "w", newline="", encoding="utf-8") as fh:
|
||||
w = csv.DictWriter(fh, fieldnames=list(table[0])); w.writeheader(); w.writerows(table)
|
||||
(ST / "rollback_plan.json").write_text(json.dumps(plan), encoding="utf-8")
|
||||
for k, v in summary.items(): print(k, v)
|
||||
@@ -0,0 +1,154 @@
|
||||
"""READ-ONLY schema/capability/population audit of every .FDB in a directory (default: ./Banco).
|
||||
|
||||
python scripts/audit_multi_databases.py [DIR]
|
||||
|
||||
Uses read-only isolation and SELECT only; credentials from FB_USER/FB_PASSWORD (env or local ignored .env).
|
||||
Writes docs/runs/multi_db/<DB>.json (raw) and docs/MULTI_DATABASE_SCHEMA_AUDIT_V1.json,
|
||||
docs/MULTI_DATABASE_FIELD_COMPATIBILITY_V1.json. Never modifies a database.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src")); sys.path.insert(0, str(ROOT / "src"))
|
||||
import sgv_capabilities as cap # noqa: E402
|
||||
|
||||
REFERENCE = json.loads((ROOT / "docs" / "runs" / "audit_TESTESOFICIAIS.json").read_text(encoding="utf-8"))["schema_fingerprint"]
|
||||
SIGNAL_TABLES = ["MESAS", "R000002", "RECEITA_INGREDIENTES", "DELIVERY_M", "C000051", "NFCE", "NFCE_ITENS", "C000032", "C000062", "C000088", "C000100", "C000030", "C000061"]
|
||||
CONTEXT_TABLES = ["C000030", "C000032", "C000062", "C000088", "C000100", "NFCE_ITENS", "RECEITA_INGREDIENTES"]
|
||||
|
||||
|
||||
def load_env():
|
||||
f = ROOT / ".env"
|
||||
if f.exists():
|
||||
for line in f.read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def audit_one(path):
|
||||
import fdb
|
||||
from gti_discovery.fingerprint import build_fingerprint
|
||||
from gti_fiscal_agent.firebird_reader import parse_company_context
|
||||
conn = fdb.connect(database=str(path), user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
res = {"file": path.name, "size_mb": round(path.stat().st_size / 1e6, 1), "mode": "READ_ONLY", "firebird_server_version": getattr(conn, "server_version", None),
|
||||
"ods": f"{getattr(conn, 'ods_version', '?')}.{getattr(conn, 'ods_minor_version', '?')}"}
|
||||
cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH), f.RDB$FIELD_SCALE
|
||||
FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE JOIN RDB$RELATIONS r ON r.RDB$RELATION_NAME = rf.RDB$RELATION_NAME
|
||||
WHERE COALESCE(r.RDB$SYSTEM_FLAG, 0) = 0""")
|
||||
meta = {}
|
||||
for t, c, ty, ln, sc in cur.fetchall():
|
||||
meta.setdefault(t, []).append({"name": c, "type": ty, "length": ln, "scale": sc})
|
||||
res["tables"] = len(meta)
|
||||
res["schema_fingerprint"] = build_fingerprint({"tables": [{"name": t, "columns": cols} for t, cols in sorted(meta.items())]})
|
||||
res["C000004_present"] = "C000004" in meta; res["C000025_present"] = "C000025" in meta
|
||||
company = {"status": "C000004_ABSENT"}
|
||||
if res["C000004_present"]:
|
||||
names = {c["name"] for c in meta["C000004"]}
|
||||
try:
|
||||
cur.execute("SELECT UF, CRT FROM C000004"); company = {"status": "OK", **parse_company_context(cur.fetchall())}
|
||||
except ValueError as e:
|
||||
company = {"status": "UNAVAILABLE", "error": str(e)}
|
||||
for col in ("OPTANTE_SIMPLES", "OPTANTE_SUPER_SIMPLES", "SUBSTITUTO_TRIBUTARIO", "CONTRIBUINTE_IPI", "ICMS_CSOSN"):
|
||||
if col in names:
|
||||
cur.execute(f"SELECT FIRST 1 {col} FROM C000004"); row = cur.fetchone(); company[col] = None if row is None or row[0] is None else str(row[0]).strip()
|
||||
for col in ("FANTASIA", "NOME_FANTASIA", "RAZAO_SOCIAL", "EMPRESA", "NOME"): # short display name only, never CNPJ
|
||||
if col in names:
|
||||
cur.execute(f"SELECT FIRST 1 {col} FROM C000004"); row = cur.fetchone(); company["name"] = None if row is None or row[0] is None else str(row[0]).strip()[:40]; break
|
||||
cur.execute("SELECT CODIGO, UF, CRT FROM C000004")
|
||||
rows = [[None if v is None else str(v).strip() for v in r] for r in cur.fetchall()]
|
||||
company["rows"] = len(rows); company["rows_summary"] = rows
|
||||
if "FILIAL" in names:
|
||||
cur.execute("SELECT FIRST 1 FILIAL FROM C000004 ORDER BY CODIGO"); r0 = cur.fetchone()
|
||||
company["filial_name_first_row"] = None if r0 is None or r0[0] is None else str(r0[0]).strip()[:50]
|
||||
if "EMISSOR" in meta and any(c["name"] == "EMITENTE_CRT" for c in meta["EMISSOR"]):
|
||||
cur.execute("SELECT EMITENTE_CRT FROM EMISSOR"); company["emissor_crt"] = sorted({str(r[0]).strip() for r in cur.fetchall() if r[0] is not None})
|
||||
res["company"] = company
|
||||
counts = {}
|
||||
for t in SIGNAL_TABLES:
|
||||
if t in meta:
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{t}"'); counts[t] = cur.fetchone()[0]
|
||||
else:
|
||||
counts[t] = None
|
||||
res["table_rows"] = counts
|
||||
if res["C000025_present"]:
|
||||
cols = {c["name"]: c["type"] for c in meta["C000025"]}
|
||||
res["C000025_columns"] = len(cols)
|
||||
full = {c["name"]: c for c in meta["C000025"]}
|
||||
res["field_types"] = {f: [full[f]["type"], full[f]["length"], full[f]["scale"]] for f in cap.WRITE_TARGETS + [cap.ORIGIN_FIELD] if f in full}
|
||||
res["capabilities"] = cap.detect_capabilities(cols)
|
||||
res["field_compatibility"] = cap.field_compatibility(cols)
|
||||
cur.execute("SELECT COUNT(*) FROM C000025"); res["product_count"] = cur.fetchone()[0]
|
||||
dist, pop = {}, {}
|
||||
for f in cap.WRITE_TARGETS + [cap.ORIGIN_FIELD]:
|
||||
if f in cols:
|
||||
cur.execute(f'SELECT "{f}", COUNT(*) FROM C000025 GROUP BY "{f}" ORDER BY 2 DESC')
|
||||
rows = [(None if v is None else str(v).strip(), n) for v, n in cur.fetchall()]
|
||||
pop[f] = sum(n for v, n in rows if v not in (None, ""))
|
||||
dist[f] = {"distinct": len(rows), "top": rows[:8]}
|
||||
res["fiscal_field_population"] = pop; res["fiscal_field_distribution"] = dist
|
||||
for extra, sql in (("kitchen_products", "SELECT COUNT(*) FROM C000025 WHERE PROD_COZINHA = 1"), ("vehicle_products", "SELECT COUNT(*) FROM C000025 WHERE VEIC_TIPO IS NOT NULL AND TRIM(VEIC_TIPO) <> ''")):
|
||||
col = "PROD_COZINHA" if "COZINHA" in sql else "VEIC_TIPO"
|
||||
if col in cols:
|
||||
cur.execute(sql); res[extra] = cur.fetchone()[0]
|
||||
grade = [t for t in meta if "GRADE" in t]
|
||||
res["grade_tables"] = {t: None for t in grade}
|
||||
gi = 0
|
||||
for t in grade:
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{t}"'); n = cur.fetchone()[0]; res["grade_tables"][t] = n; gi += n
|
||||
sig = {"restaurant_tables_rows": (counts.get("MESAS") or 0) + (counts.get("R000002") or 0), "kitchen_products": res.get("kitchen_products", 0), "recipes": counts.get("RECEITA_INGREDIENTES") or 0,
|
||||
"service_orders": counts.get("C000051") or 0, "vehicle_products": res.get("vehicle_products", 0), "grade_items": gi, "nfce_items": counts.get("NFCE_ITENS") or 0}
|
||||
seg, score, ev = cap.infer_segment(sig, " ".join(str(company.get(k) or "") for k in ("name", "filial_name_first_row")))
|
||||
res["segment_inferred"] = {"segment": seg, "score": score, "evidence": ev, "signals": sig, "note": "inferred from table population, not from the file name"}
|
||||
return res
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def main():
|
||||
load_env()
|
||||
directory = Path(sys.argv[1]) if len(sys.argv) > 1 else ROOT / "Banco"
|
||||
dbs = sorted(directory.glob("*.FDB")) + sorted(p for p in directory.glob("*.fdb") if p.suffix != ".FDB")
|
||||
out_dir = ROOT / "docs" / "runs" / "multi_db"; out_dir.mkdir(parents=True, exist_ok=True)
|
||||
audits = []
|
||||
ref_types = audit_one(ROOT / "TESTESOFICIAIS.FDB").get("field_types", {}) # reference database, read-only
|
||||
for p in dbs:
|
||||
try:
|
||||
a = audit_one(p)
|
||||
a["type_variations_vs_reference"] = {f: {"reference": ref_types.get(f), "this": v} for f, v in a.get("field_types", {}).items() if ref_types.get(f) != v}
|
||||
a["schema_class"] = cap.classify_schema(a["C000004_present"], a["C000025_present"], a["schema_fingerprint"], REFERENCE, a.get("field_compatibility", {}))
|
||||
except Exception as e: # noqa: BLE001 — record, never guess
|
||||
a = {"file": p.name, "size_mb": round(p.stat().st_size / 1e6, 1), "error": f"{type(e).__name__}: {str(e)[:200]}", "schema_class": "INCOMPATIBLE"}
|
||||
(out_dir / f"{p.stem}.json").write_text(json.dumps(a, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
audits.append(a)
|
||||
print(p.name, a.get("schema_class"), a.get("product_count"), a.get("company"), a.get("error", ""))
|
||||
ok = [a for a in audits if "error" not in a]
|
||||
schema_doc = {"artifact": "MULTI_DATABASE_SCHEMA_AUDIT_V1", "firebird_writes": 0, "directory": directory.name, "reference_fingerprint_TESTESOFICIAIS": REFERENCE, "databases_found": len(dbs), "databases_audited": len(ok),
|
||||
"classification_counts": {k: sum(1 for a in audits if a["schema_class"] == k) for k in ("REFERENCE_SCHEMA_MATCH", "COMPATIBLE_WITH_VARIATIONS", "INCOMPATIBLE")},
|
||||
"databases": [{k: a.get(k) for k in ("file", "size_mb", "firebird_server_version", "ods", "schema_fingerprint", "tables", "product_count", "company", "C000004_present", "C000025_present", "schema_class",
|
||||
"capabilities", "segment_inferred", "table_rows", "error")} for a in audits]}
|
||||
(ROOT / "docs" / "MULTI_DATABASE_SCHEMA_AUDIT_V1.json").write_text(json.dumps(schema_doc, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
fc = {}
|
||||
for f in cap.WRITE_TARGETS + [cap.ORIGIN_FIELD]:
|
||||
fc[f] = {"status_per_db": {a["file"]: a.get("field_compatibility", {}).get(f, "N/A") for a in audits},
|
||||
"population_per_db": {a["file"]: a.get("fiscal_field_population", {}).get(f) for a in ok},
|
||||
"distribution_top_per_db": {a["file"]: a.get("fiscal_field_distribution", {}).get(f, {}).get("top", [])[:5] for a in ok}}
|
||||
all_present = [f for f in cap.WRITE_TARGETS if all(a.get("field_compatibility", {}).get(f) in ("PRESENT_OK", "PRESENT_TYPE_VARIATION") for a in ok)]
|
||||
variations = {}
|
||||
for a in ok:
|
||||
for f, v in a.get("type_variations_vs_reference", {}).items():
|
||||
variations.setdefault(f, {})[a["file"]] = v
|
||||
for f, v in a.get("field_compatibility", {}).items():
|
||||
if v not in ("PRESENT_OK",):
|
||||
variations.setdefault(f, {})[a["file"]] = v
|
||||
(ROOT / "docs" / "MULTI_DATABASE_FIELD_COMPATIBILITY_V1.json").write_text(json.dumps({"artifact": "MULTI_DATABASE_FIELD_COMPATIBILITY_V1", "write_targets_present_in_all": all_present, "write_targets_count": len(cap.WRITE_TARGETS),
|
||||
"variations": variations, "fields": fc}, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Compare the ACTIVE BA_ST_MVA source PDF with a freshly downloaded RICMS-BA Anexo 1 and list the affected products. Read-only; changes no dataset."""
|
||||
import gzip, json, pickle, re, sys
|
||||
from pathlib import Path
|
||||
from pypdf import PdfReader
|
||||
ROOT = Path(__file__).resolve().parents[1]; sys.path.insert(0, str(ROOT))
|
||||
from central.app.datasets.ba_st_mva import parse_page # noqa: E402
|
||||
|
||||
def load(p):
|
||||
recs = []
|
||||
for i, pg in enumerate(PdfReader(p).pages, 1):
|
||||
f, _ = parse_page(pg.extract_text() or "", i); recs += f
|
||||
return recs
|
||||
|
||||
def key(r): return (r["item"], r["cest"] or "")
|
||||
old = load(ROOT / "sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf"); new = load(ROOT / "sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026_dl20260921.pdf")
|
||||
sig = lambda r: (tuple(sorted(r["ncm_sh"] or [])), r["mva_original"], r["mva_adjusted"]) # noqa: E731
|
||||
o = {key(r): r for r in old}; n = {key(r): r for r in new}
|
||||
added = sorted(set(n) - set(o)); removed = sorted(set(o) - set(n)); changed = sorted(k for k in set(o) & set(n) if sig(o[k]) != sig(n[k]))
|
||||
def prefixes(recs): return {re.sub(r"\D", "", x) for r in recs for x in (r["ncm_sh"] or []) if len(re.sub(r"\D", "", x)) in (4, 6, 8)}
|
||||
po, pn = prefixes(old), prefixes(new); gone, come = po - pn, pn - po
|
||||
affected = {}
|
||||
for name in ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]:
|
||||
snap = pickle.load(gzip.open(ROOT / f"docs/runs/local_completion/{name}_before.pkl.gz", "rb")); cols = snap["columns"]; i = cols.index("CLASSIFICACAO_FISCAL")
|
||||
hit = [c for c, r in snap["rows"].items() if any(re.sub(r"\D", "", r[i] or "").startswith(p) for p in (gone | come))]
|
||||
affected[name] = {"products": len(hit), "examples": hit[:5]}
|
||||
doc = {"old_records": len(old), "new_records": len(new), "items_added": [list(k) for k in added], "items_removed": [list(k) for k in removed], "items_changed": [{"key": list(k), "old": sig(o[k]), "new": sig(n[k])} for k in changed[:60]], "items_changed_count": len(changed),
|
||||
"ncm_prefixes_no_longer_listed": sorted(gone)[:100], "ncm_prefixes_newly_listed": sorted(come)[:100], "affected_products_by_database": affected,
|
||||
"note": "No dataset was modified; the Hub still serves the ACTIVE BA_ST_MVA (sha e07e9085...). The candidate PDF is versioned under sources/raw/bootstrap_v1/."}
|
||||
(ROOT / "docs" / "BA_ST_MVA_REFRESH_DIFF_V1.json").write_text(json.dumps(doc, ensure_ascii=False, indent=1, default=list), encoding="utf-8")
|
||||
print({k: (v if not isinstance(v, list) else len(v)) for k, v in doc.items() if k not in ("affected_products_by_database", "items_changed")}); print(affected)
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Build the GTI Fiscal Agent Windows package (zip) and verify it contains no secrets.
|
||||
|
||||
python scripts/build_agent_package.py -> dist/GTIFiscalAgent-<version>.zip (+ .sha256)
|
||||
|
||||
The zip root is the install folder:
|
||||
gti-agent.ps1 / gti-agent.cmd / requirements-agent.txt / agent.env.template / pilot-construcao.env.example / profiles/
|
||||
agent/{src,scripts,service}, scripts/firebird_write_executor.py, central/app/{engine modules}, VERSION, MANIFEST.json
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
VERSION = (ROOT / "release" / "VERSION").read_text(encoding="utf-8").strip()
|
||||
NAME = f"GTIFiscalAgent-{VERSION}"
|
||||
|
||||
INCLUDE = [ # (source, destination inside the package)
|
||||
*[(p, str(p.relative_to(ROOT)).replace("\\", "/")) for p in sorted((ROOT / "agent" / "src" / "gti_fiscal_agent").glob("*.py"))],
|
||||
(ROOT / "agent" / "scripts" / "run_agent_service.py", "agent/scripts/run_agent_service.py"),
|
||||
(ROOT / "agent" / "scripts" / "preflight.py", "agent/scripts/preflight.py"),
|
||||
(ROOT / "agent" / "service" / "gti_agent_windows_service.py", "agent/service/gti_agent_windows_service.py"),
|
||||
(ROOT / "scripts" / "firebird_write_executor.py", "scripts/firebird_write_executor.py"),
|
||||
(ROOT / "central" / "__init__.py", "central/__init__.py"),
|
||||
*[(ROOT / "central" / "app" / f, f"central/app/{f}") for f in ("__init__.py", "fiscal_decision.py", "dry_run_planner.py", "product_defaults.py", "icms_context.py")],
|
||||
*[(p, p.name) for p in sorted((ROOT / "deploy" / "agent-windows").glob("*")) if p.is_file()],
|
||||
*[(p, f"profiles/{p.name}") for p in sorted((ROOT / "deploy" / "agent-windows" / "profiles").glob("*")) if p.is_file()],
|
||||
(ROOT / "release" / "VERSION", "VERSION"),
|
||||
]
|
||||
SECRET_PATTERNS = [
|
||||
(re.compile(r"^[ ]*(AGENT_TOKEN|FB_PASSWORD|POSTGRES_PASSWORD)[ ]*=[ ]*\S+", re.M), "non-empty secret assignment"),
|
||||
(re.compile(r"masterkey", re.I), "default Firebird password"),
|
||||
(re.compile(r"[A-Za-z0-9_\-]{40,}"), "long token-like string"),
|
||||
]
|
||||
ALLOWED_LONG = re.compile(r"^[A-Za-z0-9_\-]{40,}$")
|
||||
|
||||
|
||||
def scan(name, data):
|
||||
if name.endswith((".ps1", ".cmd", ".py", ".txt", ".example", ".template", "VERSION")):
|
||||
text = data.decode("utf-8", "replace")
|
||||
for rx, why in SECRET_PATTERNS:
|
||||
for m in rx.finditer(text):
|
||||
token = m.group(0)
|
||||
if why == "long token-like string" and (token.startswith(("_", "-")) or "_" in token and token.count("_") > 3 or token.isalpha()):
|
||||
continue # identifiers / prose, not a secret
|
||||
if why == "long token-like string" and not re.search(r"\d", token):
|
||||
continue
|
||||
if why == "long token-like string" and re.fullmatch(r"[0-9a-f]{64}", token):
|
||||
continue # schema fingerprint (public SHA-256), not a secret
|
||||
raise SystemExit(f"secret scan failed in {name}: {why}: {token[:6]}...")
|
||||
|
||||
|
||||
def main():
|
||||
dist = ROOT / "dist"
|
||||
dist.mkdir(exist_ok=True)
|
||||
out = dist / f"{NAME}.zip"
|
||||
manifest = {"name": NAME, "version": VERSION, "files": {}}
|
||||
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as z:
|
||||
for src, dst in INCLUDE:
|
||||
data = Path(src).read_bytes()
|
||||
assert not dst.endswith((".env", ".sqlite", ".log", ".FDB")), dst
|
||||
scan(dst, data)
|
||||
manifest["files"][dst] = hashlib.sha256(data).hexdigest()
|
||||
z.writestr(f"{NAME}/{dst}", data)
|
||||
z.writestr(f"{NAME}/MANIFEST.json", json.dumps(manifest, indent=1))
|
||||
digest = hashlib.sha256(out.read_bytes()).hexdigest()
|
||||
(dist / f"{NAME}.zip.sha256").write_text(f"{digest} {out.name}\n", encoding="utf-8")
|
||||
print(json.dumps({"package": str(out.relative_to(ROOT)), "files": len(manifest["files"]), "sha256": digest}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,58 @@
|
||||
"""docs/OPERATOR_COMPANY_QUESTIONS_V1.md: ONLY company-wide decisions (category C) in plain Portuguese, plus the residual counts. Read-only over the state files."""
|
||||
import gzip, json, pickle, sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]; ST = ROOT / "docs/runs/local_completion"
|
||||
DBS = ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]
|
||||
Q = json.loads((ROOT / "docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json").read_text(encoding="utf-8"))["questions"]
|
||||
summ = {n: json.loads((ST / f"{n}_dryrun_summary.json").read_text(encoding="utf-8")) for n in DBS}
|
||||
ipi_cells, entry_dist, rows_n = {}, {}, {}
|
||||
for n in DBS:
|
||||
snap = pickle.load(gzip.open(ST / f"{n}_before.pkl.gz", "rb")); cols = snap["columns"]
|
||||
ipi_cells[n] = 0; rows_n[n] = len(snap["rows"])
|
||||
for l in gzip.open(ST / f"{n}_second.ndjson.gz", "rt", encoding="utf-8"):
|
||||
r = json.loads(l)
|
||||
if r["field"] == "IPI" and r["status"] == "ALREADY_CORRECT" and r["before"] not in (None, ""):
|
||||
ipi_cells[n] += 1
|
||||
entry_dist[n] = Counter((r[cols.index("CFOP_ENTRADA")] or "").strip()[:1] or "(vazio)" for r in snap["rows"].values())
|
||||
crt = {n: summ[n]["company"]["crt"] for n in DBS}
|
||||
by = lambda key: [q for q in Q if q["KEY"] == key and q["CATEGORY"] == "C"] # noqa: E731
|
||||
regime = {"1": "Simples Nacional", "3": "Regime normal (Lucro Presumido/Real)"}
|
||||
out = ["# Perguntas de empresa — respostas necessárias ANTES da camada de IA", "",
|
||||
"Só entram decisões que valem para a **empresa inteira**. Nada aqui é preenchido automaticamente: sem resposta, as células continuam pendentes.",
|
||||
"Como responder: preencher a coluna `VALUE` da linha do mesmo QUESTION_ID em `docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.csv` e importar com `scripts/import_fiscal_context.py`.", ""]
|
||||
n_q = 0; cells_total = 0
|
||||
def block(qid, banco, pergunta, opcoes, rec, prods, cells, expl):
|
||||
global n_q, cells_total
|
||||
n_q += 1; cells_total += cells
|
||||
out.extend([f"### {qid} — {banco}", f"- **Pergunta:** {pergunta}", f"- **Opções:** {opcoes}", f"- **Recomendação:** {rec}", f"- **Produtos afetados:** {prods}", f"- **Células afetadas:** {cells}", f"- **Explicação:** {expl}", ""])
|
||||
out.append("## 1. Perfil de PIS/COFINS (por empresa)")
|
||||
for q in by("pis_cofins_profile"):
|
||||
n = q["DATABASE"]
|
||||
block(q["QUESTION_ID"], f"{n} ({regime.get(crt[n], crt[n])})", "Qual o enquadramento de PIS/COFINS desta empresa para produtos que NÃO estão em regime especial (monofásico, alíquota zero, etc.)?",
|
||||
"JSON com pis_in_cst, pis_out_cst, pis_rate, cofins_in_cst, cofins_out_cst, cofins_rate (CSTs da tabela oficial 4.3.3/4.3.4). Ex. regime não cumulativo: saída 01, alíquotas 1,65 / 7,6; cumulativo: 0,65 / 3,0",
|
||||
"nenhuma provada (depende do regime tributário real da empresa: cumulativo ou não cumulativo, dado que só o contador confirma)", q["PRODUCTS_AFFECTED"], q["CELLS"],
|
||||
"A tabela oficial da Receita já resolveu os produtos em regime especial; os demais dependem do enquadramento da empresa. " + ("Nesta base o SGV não usa estes campos na emissão (100% dos itens de NFC-e saíram sem CST de PIS/COFINS); a resposta só completa poucos produtos com valor inválido no cadastro." if summ[n]["company"].get("pis_cofins_unused_by_emission") else "O histórico de NFC-e desta base emite CST de PIS/COFINS preenchido, então o campo do produto é usado na emissão."))
|
||||
out.append("## 2. Escopo das compras (CFOP de entrada)")
|
||||
for q in by("entry_scope"):
|
||||
n = q["DATABASE"]; d = entry_dist[n]; tot = sum(d.values())
|
||||
block(q["QUESTION_ID"], f"{n}", "As compras de mercadoria desta empresa são, em regra, dentro do estado (INTRA, CFOP 1xxx) ou de outro estado (INTER, CFOP 2xxx)?",
|
||||
"INTRA | INTER", f"nenhuma provada. Só informativo: nos cadastros existentes desta base, CFOP de entrada 1xxx = {d.get('1', 0)}, 2xxx = {d.get('2', 0)}, vazio = {d.get('(vazio)', 0)} de {tot} produtos (declarações históricas, não prova de regra)",
|
||||
q["PRODUCTS_AFFECTED"], q["CELLS"], "Só é usado nos produtos cujo CFOP de entrada está vazio. Produtos que já têm CFOP de entrada válido não são alterados.")
|
||||
out.append("## 3. Alíquota padrão de ICMS")
|
||||
for q in by("icms_rate"):
|
||||
block(q["QUESTION_ID"], q["DATABASE"], "Qual a alíquota interna de ICMS padrão desta empresa (regime normal)?", "número em % (ex.: 20,5)", "nenhuma provada (a alíquota depende da mercadoria; não vale 20,5% universal)", q["PRODUCTS_AFFECTED"], q["CELLS"],
|
||||
"Um único produto da base de testes depende desta resposta. As demais empresas são Simples Nacional e não destacam ICMS: não perguntado.")
|
||||
out.append("## 4. IPI no cadastro de empresa varejista (STORE_IPI_PROFILE)")
|
||||
out.append("Evidência do próprio SGV: em TODAS as bases `CONTRIBUINTE_IPI = N`; 745 itens vendidos na base de testes de produtos com alíquota de IPI cadastrada geraram IPI = 0 (nenhum valor de IPI foi calculado), e mais de 1,6 milhão de itens vendidos nas demais bases também têm IPI 0. O campo do cadastro é, portanto, apenas referência para empresa não contribuinte. **O SGV não impõe preencher.** Decisão de política:")
|
||||
out.append("")
|
||||
for n in DBS:
|
||||
n_q += 1; cells_total += ipi_cells[n]
|
||||
out.extend([f"### QIPI-{n} — {n}", "- **Pergunta:** Guardar no cadastro dos produtos a alíquota oficial da TIPI como referência (mesmo a empresa não sendo indústria/importadora)?", "- **Opções:** `POPULATE_REFERENCE_RATE` (guardar) | `LEAVE_NOT_APPLICABLE` (deixar em branco)",
|
||||
"- **Recomendação:** nenhuma (é política da empresa; ambas as opções são válidas)", f"- **Produtos afetados:** {ipi_cells[n]}", f"- **Células afetadas:** {ipi_cells[n]} (já gravadas com o valor oficial; se responder LEAVE_NOT_APPLICABLE, serão revertidas com `scripts/rollback_field.py`)",
|
||||
"- **Explicação:** enquanto não houver resposta, nenhum novo valor de IPI é gravado. Chave: `store_ipi_profile` (escopo COMPANY).", ""])
|
||||
out.append("## 5. Origem da mercadoria — NÃO é decisão de empresa")
|
||||
out.append("A origem (0 a 8) é atributo real de cada mercadoria e o ERP não a registra em lugar nenhum (o primeiro dígito do CST é só preenchimento). **Não existe uma resposta padrão de empresa** e nenhum valor será assumido. Só responda por GRUPO/PRODUTO quando puder garantir (ex.: grupo inteiro de importados = 1). Essas perguntas continuam nos arquivos V2, fora desta lista.")
|
||||
out += ["", f"**Total desta lista: {n_q} perguntas de empresa.**"]
|
||||
(ROOT / "docs/OPERATOR_COMPANY_QUESTIONS_V1.md").write_text("\n".join(out) + "\n", encoding="utf-8")
|
||||
print(n_q, cells_total)
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Questionnaire V2: aggregates every remaining MANUAL_INPUT_REQUIRED cell (docs/runs/local_completion/*_second.ndjson.gz) into the fewest operator questions.
|
||||
Only categories C (company configuration), D (product-group/NCM configuration) and E (product fact) may become questions."""
|
||||
import csv, gzip, json, pickle, sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]; ST = ROOT / "docs" / "runs" / "local_completion"
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from central.app.context_keys import ENUMS # noqa: E402
|
||||
|
||||
DBS = ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]
|
||||
CAT = {"origin": ("E", "COMPANY"), "pis_cofins_profile": ("C", "COMPANY"), "entry_scope": ("C", "COMPANY"), "icms_rate": ("C", "COMPANY"), "cest": ("D", "NCM"), "ex_tipi": ("D", "NCM"),
|
||||
"st_withheld": ("D", "GROUP"), "st_status": ("D", "GROUP"), "icms_treatment": ("D", "GROUP"), "product_role": ("D", "GROUP"), "ncm": ("E", "PRODUCT"), "duplicate_code": ("E", "PRODUCT")}
|
||||
QUESTION = {"origin": "Qual a origem da mercadoria (0-8) dos produtos {scope}? (nenhuma evidencia no ERP; o digito do CST e apenas preenchimento)",
|
||||
"pis_cofins_profile": "Qual o perfil PIS/COFINS da empresa (CST de entrada/saida e aliquotas)? JSON com pis_in_cst, pis_out_cst, pis_rate, cofins_in_cst, cofins_out_cst, cofins_rate",
|
||||
"entry_scope": "As compras de mercadoria desta empresa sao predominantemente INTRA (dentro da UF) ou INTER (outra UF)?",
|
||||
"icms_rate": "Qual a aliquota de ICMS padrao desta empresa (regime normal)?",
|
||||
"cest": "Qual o CEST correto para o NCM {scope} (ou NONE quando o produto nao esta sujeito)?",
|
||||
"ex_tipi": "Qual o EX da TIPI aplicavel ao NCM {scope} (ou NONE)?",
|
||||
"st_withheld": "Os produtos {scope} sujeitos a ST tem o ICMS-ST ja retido na compra (Y/N)?",
|
||||
"st_status": "Os produtos {scope} estao sujeitos a ST (SUBJECT/NOT_SUBJECT)? O ERP declara ST retida mas a tabela oficial nao lista o NCM",
|
||||
"icms_treatment": "Qual o tratamento de ICMS dos produtos {scope}?",
|
||||
"product_role": "Os produtos {scope} sao revenda, producao propria, insumo ou servico?",
|
||||
"ncm": "Qual o NCM correto do produto {scope} (o atual nao existe na tabela oficial)?",
|
||||
"duplicate_code": "O CODIGO {scope} aparece em 2 linhas fisicas (violacao de PK na copia); corrigir a integridade do banco (gfix/validate) antes de classificar"}
|
||||
WHY = {"origin": "CST/CSOSN e a origem do NF-e dependem dela; nao pode ser presumida 0", "pis_cofins_profile": "PIS/COFINS fora dos regimes especiais das tabelas SPED 4.3.x", "entry_scope": "CFOP de entrada (1xxx vs 2xxx)",
|
||||
"icms_rate": "aliquota interna", "cest": "CEST oficial para o NCM tem varios candidatos ou o CEST do ERP nao consta da lista oficial", "ex_tipi": "varias linhas TIPI com aliquotas diferentes para o NCM",
|
||||
"st_withheld": "ST comprovada pela tabela BA mas retencao desconhecida", "st_status": "conflito entre declaracao do ERP e tabela oficial", "icms_treatment": "CST de ICMS nao derivavel", "product_role": "revenda/producao define CFOP e IPPT",
|
||||
"ncm": "NCM invalido bloqueia IPI/EX/CEST/ST/CFOP/CSOSN", "duplicate_code": "UPDATE por CODIGO exige rowcount 1"}
|
||||
agg = defaultdict(lambda: {"products": set(), "cells": 0, "fields": defaultdict(int), "examples": []})
|
||||
for n in DBS:
|
||||
snap = pickle.load(gzip.open(ST / f"{n}_before.pkl.gz", "rb")); cols = snap["columns"]; rows = snap["rows"]
|
||||
for l in gzip.open(ST / f"{n}_second.ndjson.gz", "rt", encoding="utf-8"):
|
||||
r = json.loads(l)
|
||||
if r["status"] != "MANUAL_INPUT_REQUIRED":
|
||||
continue
|
||||
k = r["input_key"]; cat, scope = CAT[k]; row = rows.get(r["codigo"]); g = (row[cols.index("CODGRUPO")] if row else None); ncm = (row[cols.index("CLASSIFICACAO_FISCAL")] if row else None)
|
||||
sk = {"COMPANY": "", "GROUP": g or "", "NCM": ncm or "", "PRODUCT": r["codigo"]}[scope]
|
||||
a = agg[(n, k, scope, sk)]; a["products"].add(r["row_key"]); a["cells"] += 1; a["fields"][r["field"]] += 1
|
||||
if len(a["examples"]) < 3 and row: a["examples"].append(row[cols.index("PRODUTO")])
|
||||
out = []
|
||||
for (n, k, scope, sk), a in agg.items():
|
||||
cat = CAT[k][0]
|
||||
opts = sorted(ENUMS[k]) if k in ENUMS else ("JSON" if k == "pis_cofins_profile" else "7 digits or NONE" if k == "cest" else "8 digits" if k == "ncm" else "free")
|
||||
who = "toda a empresa" if scope == "COMPANY" else f"{scope} {sk}"
|
||||
out.append({"DATABASE": n, "COMPANY": n, "SCOPE": scope, "GROUP": sk if scope == "GROUP" else "", "NCM": sk if scope == "NCM" else "", "FIELD": "|".join(sorted(a["fields"])), "KEY": k, "CATEGORY": cat,
|
||||
"QUESTION": QUESTION[k].format(scope=who), "VALID_OPTIONS": opts, "PRODUCTS_AFFECTED": len(a["products"]), "WHY_NEEDED": WHY[k], "WHAT_IT_WILL_RESOLVE": f"{a['cells']} celulas", "CELLS": a["cells"],
|
||||
"EXAMPLES": "; ".join(a["examples"])})
|
||||
out.sort(key=lambda x: -x["CELLS"])
|
||||
for i, q in enumerate(out, 1): q["QUESTION_ID"] = f"Q{i:04d}"
|
||||
(ROOT / "docs" / "LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json").write_text(json.dumps({"artifact": "LOCAL_FISCAL_CONTEXT_QUESTIONS_V2", "total_questions": len(out), "total_cells": sum(q["CELLS"] for q in out), "questions": out}, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
hdr = ["QUESTION_ID", "DATABASE", "COMPANY", "SCOPE", "GROUP", "NCM", "FIELD", "KEY", "CATEGORY", "QUESTION", "VALID_OPTIONS", "PRODUCTS_AFFECTED", "WHY_NEEDED", "WHAT_IT_WILL_RESOLVE", "CELLS", "EXAMPLES", "VALUE"]
|
||||
with open(ROOT / "docs" / "LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.csv", "w", newline="", encoding="utf-8-sig") as f:
|
||||
w = csv.DictWriter(f, fieldnames=hdr, extrasaction="ignore"); w.writeheader()
|
||||
for q in out: w.writerow({**q, "VALID_OPTIONS": q["VALID_OPTIONS"] if isinstance(q["VALID_OPTIONS"], str) else "|".join(q["VALID_OPTIONS"]), "VALUE": ""})
|
||||
from collections import Counter
|
||||
print(len(out), sum(q["CELLS"] for q in out), dict(Counter(q["CATEGORY"] for q in out)), dict(Counter(q["KEY"] for q in out)))
|
||||
print([(q["QUESTION_ID"], q["DATABASE"], q["KEY"], q["CELLS"]) for q in out[:8]])
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Parse the official SPED EFD-Contribuições tables 4.3.x (Word .doc, converted with antiword) into a versioned NCM-keyed dataset.
|
||||
Source: sped.rfb.gov.br/arquivo/download/<id> (current files, fetched directly on 2026-09-21).
|
||||
Writes docs/sources/SPED_PISCOFINS_TABLES_V3.json (per table: source id, sha256 of the .doc, version text, entries {code, description, ncm_prefixes, start, end, pis_rate, cofins_rate})."""
|
||||
import datetime, hashlib, json, re, sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]; SRC = ROOT / "sources_cache" / "cur"
|
||||
TABLES = { # sped item id -> (table, CST it defines for the SAIDA side, meaning)
|
||||
1638: ("4.3.10", ["02", "04"], "MONOFASICA_OU_PAUTA"), 1642: ("4.3.12", ["05"], "SUBSTITUICAO_TRIBUTARIA"), 1643: ("4.3.13", ["06"], "ALIQUOTA_ZERO"),
|
||||
1646: ("4.3.14", ["07"], "ISENTA"), 1651: ("4.3.15", ["08"], "SEM_INCIDENCIA"), 5786: ("4.3.11", ["03", "04"], "MONOFASICA_AD_REM")}
|
||||
NCM = re.compile(r"\d{4}(?:\.\d{2}(?:\.\d{2}(?:\.\d{2})?)?)?")
|
||||
DATE = re.compile(r"\d{2}/\d{4}|\d{2}/\d{2}/\d{4}")
|
||||
|
||||
|
||||
def parse(txt):
|
||||
rows, cur = [], None
|
||||
for line in txt.splitlines():
|
||||
if not line.startswith("|"):
|
||||
continue
|
||||
cells = [c.strip() for c in line.strip().strip("|").split("|")]
|
||||
if cells and re.fullmatch(r"\d{3}", cells[0] or ""):
|
||||
cur = {"code": cells[0], "cells": [[c] for c in cells[1:]]}; rows.append(cur)
|
||||
elif cur is not None and len(cells) > 1 and not cells[0]:
|
||||
for i, c in enumerate(cells[1:]):
|
||||
if i < len(cur["cells"]) and c:
|
||||
cur["cells"][i].append(c)
|
||||
return rows
|
||||
|
||||
|
||||
out = {"artifact": "SPED_PISCOFINS_TABLES_V3", "tables": {}}
|
||||
for sid, (tab, csts, meaning) in TABLES.items():
|
||||
raw = (SRC / f"sped_{sid}.bin").read_bytes(); txt = (SRC / f"sped_{sid}.txt").read_text(encoding="utf-8", errors="replace")
|
||||
hdr = (SRC / f"h_{sid}.txt").read_text(errors="replace"); lm = re.search(r"Last-Modified: (.*)", hdr); lm = lm.group(1).strip() if lm else None
|
||||
head_all = " ".join(txt.splitlines()[:60])
|
||||
ver = re.search(r"Vers[ãa]o (\d+(?:\.\d+)+)", head_all); eff = re.search(r"Atualizad[ao]s? em (\d{2}/\d{2}/\d{4})", head_all)
|
||||
head = " ".join(l.strip() for l in txt.splitlines()[:8] if l.strip() and not l.startswith("|"))
|
||||
entries = []
|
||||
for r in parse(txt):
|
||||
cells = [" ".join(c) for c in r["cells"]] or [""]; dcells = ["".join(c) for c in r["cells"]]
|
||||
joined_ncm = cells[1] if len(cells) > 1 else ""
|
||||
pref = sorted({m.group(0).replace(".", "") for m in NCM.finditer(joined_ncm)})
|
||||
dates = [d for c in dcells[2:] for d in DATE.findall(c)]
|
||||
rates = re.findall(r"\d+,\d+", " ".join(cells[2:4])) if tab in ("4.3.10", "4.3.12") else []
|
||||
entries.append({"code": r["code"], "description": cells[0] if cells else "", "ncm_prefixes": pref, "applies_to_all_ncm": not pref and joined_ncm.strip() in ("-", ""), "start": dates[0] if dates else None, "end": dates[1] if len(dates) > 1 else None, "rates_text": rates})
|
||||
out["tables"][tab] = {"sped_item": sid, "source_url": f"http://sped.rfb.gov.br/arquivo/download/{sid}", "download_date": datetime.date.today().isoformat(), "http_last_modified": lm, "version": ver.group(1) if ver else None, "effective_date": eff.group(1) if eff else None, "sha256_doc": hashlib.sha256(raw).hexdigest(), "version_text": head, "saida_cst": csts, "meaning": meaning,
|
||||
"fetched_via": "direct official download over http (sped.rfb.gov.br)", "entries": entries, "ncm_prefix_count": len({p for e in entries for p in e["ncm_prefixes"]})}
|
||||
print(tab, len(entries), out["tables"][tab]["ncm_prefix_count"], head[:110])
|
||||
for sid, name in ((1629, "4.3.3"), (1630, "4.3.4")):
|
||||
raw = (SRC / f"sped_{sid}.bin").read_bytes(); txt = (SRC / f"sped_{sid}.txt").read_text(encoding="utf-8", errors="replace")
|
||||
cst = {}
|
||||
for line in txt.splitlines():
|
||||
m = re.match(r"\|(\d{2})\s*\|([^|]+)\|", line)
|
||||
if m: cst[m.group(1)] = m.group(2).strip()
|
||||
out["tables"][name] = {"sped_item": sid, "sha256_doc": hashlib.sha256(raw).hexdigest(), "cst_domain": cst}
|
||||
print(name, sorted(cst))
|
||||
(ROOT / "docs" / "sources" / "SPED_PISCOFINS_TABLES_V3.json").write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Runs INSIDE the fiscal-hub container (read-only on PostgreSQL): classifies [ncm, description] pairs with the
|
||||
production Classifier through a memoizing repository wrapper. Same code path as /v1/classify, minus HTTP and the
|
||||
classification_cache write; the wrapper only memoizes pure dataset lookups (datasets are static during the run).
|
||||
usage: python /tmp/bulk_classify_in_container.py /tmp/in.json /tmp/out.json
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from functools import lru_cache
|
||||
|
||||
from central.app.classification import Classifier
|
||||
from central.app.datasets.service import DatasetService
|
||||
from central.app.db import SessionLocal
|
||||
|
||||
|
||||
class Memo:
|
||||
def __init__(self, service):
|
||||
self._s = service; self._cache = {}
|
||||
|
||||
def __getattr__(self, name):
|
||||
attr = getattr(self._s, name)
|
||||
if not callable(attr):
|
||||
return attr
|
||||
def wrapped(*a, **kw):
|
||||
key = (name, a, tuple(sorted(kw.items())))
|
||||
try:
|
||||
hash(key)
|
||||
except TypeError:
|
||||
return attr(*a, **kw)
|
||||
if key not in self._cache:
|
||||
self._cache[key] = attr(*a, **kw)
|
||||
return self._cache[key]
|
||||
return wrapped
|
||||
|
||||
|
||||
def main():
|
||||
pairs = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
out = []
|
||||
with SessionLocal() as session:
|
||||
classifier = Classifier(Memo(DatasetService(session)))
|
||||
for ncm, desc in pairs:
|
||||
out.append(classifier.classify({"current_ncm": ncm, "description": desc}))
|
||||
json.dump(out, open(sys.argv[2], "w", encoding="utf-8"), ensure_ascii=False, default=str)
|
||||
print(len(out))
|
||||
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Read-only Firebird schema/context discovery (no DML/DDL). Writes JSON to stdout."""
|
||||
import json, os, re, sys
|
||||
from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent.firebird_reader import _connect # noqa: E402
|
||||
for l in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
||||
if "=" in l and not l.startswith("#"):
|
||||
k, _, v = l.partition("="); os.environ.setdefault(k.strip(), v.strip())
|
||||
KW = re.compile(r"CFOP|CST|CSOSN|ORIGEM|PIS|COFINS|CEST|IPI|MVA|FCP|CLASSTRIB|IBS|CBS|NCM|ICMS|IVA|MARGEM|ALIQ|NATUREZA|CRT|REGIME|TRIB|PERC|BASE_", re.I)
|
||||
conn = _connect(os.environ); cur = conn.cursor()
|
||||
cur.execute("select trim(RDB$RELATION_NAME), case when RDB$VIEW_SOURCE is null then 0 else 1 end from RDB$RELATIONS where coalesce(RDB$SYSTEM_FLAG,0)=0")
|
||||
rels = cur.fetchall()
|
||||
cur.execute("select trim(rf.RDB$RELATION_NAME), trim(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, coalesce(f.RDB$CHARACTER_LENGTH,f.RDB$FIELD_LENGTH), f.RDB$FIELD_SCALE from RDB$RELATION_FIELDS rf join RDB$FIELDS f on f.RDB$FIELD_NAME=rf.RDB$FIELD_SOURCE where coalesce(rf.RDB$SYSTEM_FLAG,0)=0")
|
||||
cols = {}
|
||||
for t, c, ty, ln, sc in cur.fetchall():
|
||||
cols.setdefault(t, []).append((c, ty, ln, sc))
|
||||
out = {"tables": {}}
|
||||
for t, isview in rels:
|
||||
try:
|
||||
cur.execute(f'select count(*) from "{t}"'); n = cur.fetchone()[0]
|
||||
except Exception as e:
|
||||
n = None
|
||||
fc = [c for c in cols.get(t, []) if KW.search(c[0])]
|
||||
out["tables"][t] = {"rows": n, "view": bool(isview), "ncols": len(cols.get(t, [])), "fiscal_cols": [c[0] for c in fc]}
|
||||
cur.execute("select trim(RDB$PROCEDURE_NAME) from RDB$PROCEDURES where coalesce(RDB$SYSTEM_FLAG,0)=0"); out["procedures"] = [r[0] for r in cur.fetchall()]
|
||||
cur.execute("select trim(RDB$TRIGGER_NAME), trim(RDB$RELATION_NAME) from RDB$TRIGGERS where coalesce(RDB$SYSTEM_FLAG,0)=0"); out["triggers"] = [list(r) for r in cur.fetchall()]
|
||||
cur.execute("""select trim(rc.RDB$RELATION_NAME), trim(s.RDB$FIELD_NAME), trim(rc2.RDB$RELATION_NAME) from RDB$RELATION_CONSTRAINTS rc
|
||||
join RDB$REF_CONSTRAINTS r on r.RDB$CONSTRAINT_NAME=rc.RDB$CONSTRAINT_NAME join RDB$RELATION_CONSTRAINTS rc2 on rc2.RDB$CONSTRAINT_NAME=r.RDB$CONST_NAME_UQ
|
||||
join RDB$INDEX_SEGMENTS s on s.RDB$INDEX_NAME=rc.RDB$INDEX_NAME where rc.RDB$CONSTRAINT_TYPE='FOREIGN KEY'""")
|
||||
out["fks"] = [list(r) for r in cur.fetchall()]
|
||||
conn.close()
|
||||
print(json.dumps(out, ensure_ascii=False))
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Extract control->column bindings from the SGV executable's embedded Delphi binary DFM forms (READ-ONLY).
|
||||
|
||||
python scripts/extract_sgv_dfm_bindings.py <SGV.exe> [--needle COD_ORIGEM_MERCADORIA] [--out file.json]
|
||||
|
||||
Reads the exe bytes (never modifies it), locates every 'TPF0' form stream that contains the needle, parses the
|
||||
Delphi binary DFM and reports, for every data-aware control, its DataField/DataSource, Caption/Text-like properties
|
||||
and the chain of parent containers (group boxes, tab sheets) — i.e. the on-screen label context.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
|
||||
VA = dict(NULL=0, LIST=1, INT8=2, INT16=3, INT32=4, EXTENDED=5, STRING=6, IDENT=7, FALSE=8, TRUE=9, BINARY=10, SET=11, LSTRING=12, NIL=13,
|
||||
COLLECTION=14, SINGLE=15, CURRENCY=16, DATE=17, WSTRING=18, INT64=19, UTF8=20, DOUBLE=21)
|
||||
|
||||
|
||||
class R:
|
||||
def __init__(self, data, pos=0):
|
||||
self.d, self.p = data, pos
|
||||
|
||||
def u8(self):
|
||||
v = self.d[self.p]; self.p += 1; return v
|
||||
|
||||
def take(self, n):
|
||||
v = self.d[self.p:self.p + n]
|
||||
if len(v) < n:
|
||||
raise ValueError("eof")
|
||||
self.p += n; return v
|
||||
|
||||
def sstr(self):
|
||||
return self.take(self.u8()).decode("latin-1")
|
||||
|
||||
def value(self):
|
||||
t = self.u8()
|
||||
if t == VA["NULL"] or t == VA["NIL"]: return None
|
||||
if t == VA["LIST"]:
|
||||
out = []
|
||||
while self.d[self.p] != 0:
|
||||
out.append(self.value())
|
||||
self.p += 1; return out
|
||||
if t == VA["INT8"]: return struct.unpack("<b", self.take(1))[0]
|
||||
if t == VA["INT16"]: return struct.unpack("<h", self.take(2))[0]
|
||||
if t == VA["INT32"]: return struct.unpack("<i", self.take(4))[0]
|
||||
if t == VA["EXTENDED"]: self.take(10); return "<ext>"
|
||||
if t in (VA["STRING"], VA["IDENT"]): return self.sstr()
|
||||
if t == VA["FALSE"]: return False
|
||||
if t == VA["TRUE"]: return True
|
||||
if t == VA["BINARY"]:
|
||||
n = struct.unpack("<I", self.take(4))[0]; self.take(n); return "<binary>"
|
||||
if t == VA["SET"]:
|
||||
out = []
|
||||
while True:
|
||||
s = self.sstr()
|
||||
if not s: break
|
||||
out.append(s)
|
||||
return out
|
||||
if t == VA["LSTRING"] or t == VA["UTF8"]:
|
||||
n = struct.unpack("<I", self.take(4))[0]; return self.take(n).decode("utf-8", "replace" if t == VA["UTF8"] else "replace")
|
||||
if t == VA["WSTRING"]:
|
||||
n = struct.unpack("<I", self.take(4))[0]; return self.take(n * 2).decode("utf-16-le", "replace")
|
||||
if t == VA["SINGLE"]: self.take(4); return "<single>"
|
||||
if t == VA["CURRENCY"] or t == VA["DATE"] or t == VA["DOUBLE"] or t == VA["INT64"]: self.take(8); return "<8b>"
|
||||
if t == VA["COLLECTION"]:
|
||||
out = []
|
||||
while self.d[self.p] != 0:
|
||||
if self.d[self.p] in (2, 3, 4): # optional collection order value
|
||||
self.value()
|
||||
if self.d[self.p] == 1: # vaList marker that opens every collection item
|
||||
self.p += 1
|
||||
item = {}
|
||||
while self.d[self.p] != 0:
|
||||
key = self.sstr() # evaluate the name BEFORE the value (Python evaluates the RHS first)
|
||||
item[key] = self.value()
|
||||
self.p += 1; out.append(item)
|
||||
self.p += 1; return out
|
||||
raise ValueError(f"unknown value type {t} at {self.p}")
|
||||
|
||||
def component(self):
|
||||
first = self.u8()
|
||||
if first & 0xF0 == 0xF0: # ObjInherit / ObjFlags prefix
|
||||
if first & 2: # ffChildPos
|
||||
self.value()
|
||||
cls = self.sstr()
|
||||
else:
|
||||
self.p -= 1; cls = self.sstr()
|
||||
name = self.sstr()
|
||||
props = {}
|
||||
while self.d[self.p] != 0:
|
||||
pn = self.sstr(); props[pn] = self.value()
|
||||
self.p += 1
|
||||
kids = []
|
||||
while self.d[self.p] != 0:
|
||||
kids.append(self.component())
|
||||
self.p += 1
|
||||
return {"class": cls, "name": name, "props": props, "children": kids}
|
||||
|
||||
|
||||
def walk(node, chain, out):
|
||||
cap = node["props"].get("Caption")
|
||||
here = chain + ([f"{node['class']}:{node['name']}" + (f"[{cap}]" if isinstance(cap, str) and cap else "")] if node["class"] != "TQuery" else [])
|
||||
p = node["props"]
|
||||
if "DataField" in p or node["class"].startswith(("TLabel", "TRz", "TGroup", "TTab", "TRadio")):
|
||||
out.append({"class": node["class"], "name": node["name"], "DataField": p.get("DataField"), "DataSource": p.get("DataSource"), "Caption": cap,
|
||||
"Items": p.get("Items"), "Values": p.get("Values"), "FocusControl": p.get("FocusControl"), "Left": p.get("Left"), "Top": p.get("Top"), "path": [c for c in chain]})
|
||||
for k in node["children"]:
|
||||
walk(k, here, out)
|
||||
|
||||
|
||||
def find_forms(data, needle):
|
||||
forms = []
|
||||
for m in re.finditer(b"TPF0", data):
|
||||
start = m.start()
|
||||
try:
|
||||
r = R(data, start + 4)
|
||||
tree = r.component()
|
||||
except Exception:
|
||||
continue
|
||||
blob = data[start:r.p]
|
||||
if needle in blob:
|
||||
forms.append((start, r.p, tree))
|
||||
return forms
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("exe"); ap.add_argument("--needle", default="COD_ORIGEM_MERCADORIA"); ap.add_argument("--out")
|
||||
a = ap.parse_args(argv)
|
||||
data = open(a.exe, "rb").read()
|
||||
forms = find_forms(data, a.needle.encode())
|
||||
result = []
|
||||
for start, end, tree in forms:
|
||||
controls = []
|
||||
walk(tree, [], controls)
|
||||
result.append({"offset": start, "size": end - start, "form_class": tree["class"], "form_name": tree["name"], "controls": controls})
|
||||
text = json.dumps(result, ensure_ascii=False, indent=1, default=str)
|
||||
if a.out:
|
||||
open(a.out, "w", encoding="utf-8").write(text)
|
||||
print(f"forms containing {a.needle!r}: {[(f['form_class'], f['size'], len(f['controls'])) for f in result]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,368 @@
|
||||
"""Minimal guarded Firebird write executor for the MVP.
|
||||
|
||||
Scope is intentionally narrow: a single allowlisted table/field
|
||||
(C000025.IPI), a single authorized database file, and an optimistic
|
||||
compare-before-write. This is not a generic ORM/executor framework.
|
||||
|
||||
Per CLAUDE.md's "Firebird write policy": writes are only ever permitted
|
||||
against the disposable test database TESTESOFICIAIS.FDB. Any other path,
|
||||
or any table/field outside the allowlist, fails closed before touching
|
||||
the connection.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
AUTHORIZED_WRITE_DATABASE_PATH = Path(r"D:\PROJETOS\GTISISTEMAS\Projeto\TESTESOFICIAIS.FDB")
|
||||
|
||||
# table -> allowed writable fields. Anything not listed here is refused.
|
||||
_FIELD_ALLOWLIST: dict[str, set[str]] = {
|
||||
# C000025.ALIQ_IPI is VARCHAR(1) and unused (0/521 rows) — confirmed via
|
||||
# the vendor's own DDL scripts and live schema inspection to NOT be the
|
||||
# IPI rate field. C000025.IPI NUMERIC(15,2) is. See Mission notes.
|
||||
"C000025": {"IPI"},
|
||||
}
|
||||
# Local completion mission: every field with WRITE_ALLOWED=true in docs/FISCAL_FIELD_CONTRACT_V1.json (kept equal to
|
||||
# central.app.field_contract.WRITE_FIELDS by a test). Still C000025 only; still exact-path authorized.
|
||||
_LOCAL_COMPLETION_FIELDS = {"CLASSIFICACAO_FISCAL", "IPI", "EX_TIPI", "CEST", "SITUACAO_TRIBUTARIA", "CFOP", "CFOP_ENTRADA", "CST", "CST_ENTRADA", "CSOSN", "ALIQUOTA", "COD_ORIGEM_MERCADORIA",
|
||||
"CST_IPI", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS", "INCIDENCIA_PISCOFINS", "PISCOFINS", "IAT", "IPPT"}
|
||||
_LOCAL_ALLOWLIST_FILE = Path(__file__).resolve().parents[1] / "docs" / "LOCAL_WRITE_ALLOWLIST_V1.json"
|
||||
|
||||
_KEY_FIELD = "CODIGO"
|
||||
|
||||
|
||||
class WriteRefused(Exception):
|
||||
"""Raised whenever a guard rejects the write before it reaches the database."""
|
||||
|
||||
|
||||
def _normalized_path(path: os.PathLike[str] | str) -> str:
|
||||
return os.path.normcase(os.path.normpath(str(Path(path).resolve(strict=False))))
|
||||
|
||||
|
||||
def assert_authorized_write_path(
|
||||
path: os.PathLike[str] | str,
|
||||
authorized_path: os.PathLike[str] | str | None = None,
|
||||
) -> Path:
|
||||
"""Fail closed unless `path` is exactly the authorized write target.
|
||||
|
||||
`authorized_path` defaults to the development disposable test database
|
||||
(`AUTHORIZED_WRITE_DATABASE_PATH`) for backward compatibility with the
|
||||
dev environment. A pilot/client installation MUST pass its own
|
||||
`authorized_path` explicitly (from that installation's own config,
|
||||
e.g. `AUTHORIZED_WRITE_DATABASE_PATH` env var) — authorization is never
|
||||
inferred from the filename or extension alone, and never defaults to
|
||||
"any .FDB is fine". If a caller sets FIREBIRD_WRITES=1 in a pilot
|
||||
environment without also setting an explicit authorized_path, this
|
||||
still falls back to the dev-only path and therefore fails closed for
|
||||
any real client database, which is the safe behavior.
|
||||
"""
|
||||
candidate = Path(path)
|
||||
authorized = Path(authorized_path) if authorized_path else AUTHORIZED_WRITE_DATABASE_PATH
|
||||
if _normalized_path(candidate) != _normalized_path(authorized):
|
||||
raise WriteRefused(f"path is not the authorized write target: {candidate}")
|
||||
return authorized
|
||||
|
||||
|
||||
def assert_schema_fingerprint_authorized(observed_fingerprint: str | None, expected_fingerprint: str | None) -> None:
|
||||
"""Fail closed unless the live schema fingerprint matches what this
|
||||
installation was registered/configured to expect. Prevents writing
|
||||
against a database whose schema silently changed or was never verified."""
|
||||
if not expected_fingerprint:
|
||||
raise WriteRefused("no expected schema fingerprint configured for this installation")
|
||||
if observed_fingerprint != expected_fingerprint:
|
||||
raise WriteRefused(f"schema fingerprint mismatch: expected {expected_fingerprint!r}, observed {observed_fingerprint!r}")
|
||||
|
||||
|
||||
def assert_allowed_field(table: str, field: str) -> None:
|
||||
allowed = _FIELD_ALLOWLIST.get(table.upper())
|
||||
if not allowed or field.upper() not in allowed:
|
||||
raise WriteRefused(f"{table}.{field} is not in the write allowlist")
|
||||
|
||||
|
||||
def assert_write_eligible(proposal: dict[str, Any]) -> None:
|
||||
if not proposal.get("write_eligible"):
|
||||
raise WriteRefused(
|
||||
f"proposal for {proposal.get('product_code')}/{proposal.get('field')} "
|
||||
f"is not write_eligible (status={proposal.get('status')})"
|
||||
)
|
||||
|
||||
|
||||
def assert_schema_ok(cursor: Any, table: str, field: str, proposed_value: Any = None) -> None:
|
||||
"""Confirm the target column exists and, if it is a fixed-length text field,
|
||||
that the proposed value actually fits without truncation. Never coerce or
|
||||
truncate a value to make it fit."""
|
||||
cursor.execute(
|
||||
"SELECT f.RDB$FIELD_TYPE, f.RDB$FIELD_LENGTH FROM RDB$RELATION_FIELDS rf "
|
||||
"JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE "
|
||||
"WHERE rf.RDB$RELATION_NAME = ? AND rf.RDB$FIELD_NAME = ?",
|
||||
(table, field),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row is None:
|
||||
raise WriteRefused(f"schema guard failed: {table}.{field} not found in live schema")
|
||||
field_type, field_length = row
|
||||
# Firebird character types (CHAR=14, VARCHAR=37): reject a value that would truncate.
|
||||
if proposed_value is not None and field_type in (14, 37) and field_length is not None:
|
||||
if len(str(proposed_value)) > field_length:
|
||||
raise WriteRefused(
|
||||
f"schema guard failed: {table}.{field} is {field_length} char(s) wide, "
|
||||
f"proposed value {proposed_value!r} ({len(str(proposed_value))} chars) would truncate"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WriteAuditRecord:
|
||||
product_code: str
|
||||
field: str
|
||||
before_value: Any
|
||||
proposed_value: Any
|
||||
after_value: Any
|
||||
rowcount: int
|
||||
source_dataset: str | None
|
||||
source_version: str | None
|
||||
timestamp: str
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"product_code": self.product_code,
|
||||
"field": self.field,
|
||||
"before_value": self.before_value,
|
||||
"proposed_value": self.proposed_value,
|
||||
"after_value": self.after_value,
|
||||
"rowcount": self.rowcount,
|
||||
"source_dataset": self.source_dataset,
|
||||
"source_version": self.source_version,
|
||||
"timestamp": self.timestamp,
|
||||
}
|
||||
|
||||
|
||||
def execute_guarded_update(
|
||||
connection: Any,
|
||||
*,
|
||||
database_path: os.PathLike[str] | str,
|
||||
proposal: dict[str, Any],
|
||||
table: str = "C000025",
|
||||
key_field: str = _KEY_FIELD,
|
||||
now: str,
|
||||
authorized_path: os.PathLike[str] | str | None = None,
|
||||
) -> WriteAuditRecord:
|
||||
"""Perform one allowlisted, compare-before-write UPDATE and verify it.
|
||||
|
||||
Never commits if any guard fails. Returns an audit record with no secrets.
|
||||
`authorized_path` defaults to the dev disposable test database; a pilot
|
||||
installation must pass its own registered path explicitly.
|
||||
"""
|
||||
assert_authorized_write_path(database_path, authorized_path=authorized_path)
|
||||
assert_write_eligible(proposal)
|
||||
field = proposal["field"]
|
||||
assert_allowed_field(table, field)
|
||||
|
||||
product_code = proposal["product_code"]
|
||||
expected_before = proposal["current_value"]
|
||||
proposed_value = proposal["proposed_value"]
|
||||
if proposed_value is None:
|
||||
raise WriteRefused("refuse to write a null proposed_value")
|
||||
|
||||
cursor = connection.cursor()
|
||||
try:
|
||||
assert_schema_ok(cursor, table, field, proposed_value)
|
||||
|
||||
cursor.execute(
|
||||
f"SELECT {key_field}, {field} FROM {table} WHERE {key_field} = ?",
|
||||
(product_code,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row is None:
|
||||
raise WriteRefused(f"product {product_code} not found in {table}")
|
||||
live_before = row[1]
|
||||
if live_before != expected_before:
|
||||
raise WriteRefused(
|
||||
f"drift detected for {product_code}.{field}: expected {expected_before!r}, "
|
||||
f"found {live_before!r}"
|
||||
)
|
||||
|
||||
if expected_before is None:
|
||||
sql = f"UPDATE {table} SET {field} = ? WHERE {key_field} = ? AND {field} IS NULL"
|
||||
params = (proposed_value, product_code)
|
||||
else:
|
||||
sql = f"UPDATE {table} SET {field} = ? WHERE {key_field} = ? AND {field} = ?"
|
||||
params = (proposed_value, product_code, expected_before)
|
||||
cursor.execute(sql, params)
|
||||
rowcount = cursor.rowcount
|
||||
if rowcount != 1:
|
||||
connection.rollback()
|
||||
raise WriteRefused(f"expected rowcount 1, got {rowcount}; rolled back")
|
||||
|
||||
connection.commit()
|
||||
|
||||
cursor.execute(f"SELECT {field} FROM {table} WHERE {key_field} = ?", (product_code,))
|
||||
after_row = cursor.fetchone()
|
||||
after_value = after_row[0] if after_row else None
|
||||
if after_value != proposed_value:
|
||||
raise WriteRefused(
|
||||
f"post-write verification failed for {product_code}.{field}: "
|
||||
f"expected {proposed_value!r}, found {after_value!r}"
|
||||
)
|
||||
|
||||
return WriteAuditRecord(
|
||||
product_code=product_code,
|
||||
field=field,
|
||||
before_value=live_before,
|
||||
proposed_value=proposed_value,
|
||||
after_value=after_value,
|
||||
rowcount=rowcount,
|
||||
source_dataset=proposal.get("source_dataset"),
|
||||
source_version=proposal.get("source_version"),
|
||||
timestamp=now,
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
connection.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
finally:
|
||||
cursor.close()
|
||||
|
||||
|
||||
def execute_guarded_restore(
|
||||
connection: Any,
|
||||
*,
|
||||
database_path: os.PathLike[str] | str,
|
||||
table: str,
|
||||
field: str,
|
||||
key_field: str,
|
||||
product_code: str,
|
||||
written_value: Any,
|
||||
restore_value: Any,
|
||||
now: str,
|
||||
) -> WriteAuditRecord:
|
||||
"""Restore a field to `restore_value`, guarded the same way as the forward write."""
|
||||
assert_authorized_write_path(database_path)
|
||||
assert_allowed_field(table, field)
|
||||
|
||||
cursor = connection.cursor()
|
||||
try:
|
||||
assert_schema_ok(cursor, table, field)
|
||||
cursor.execute(f"SELECT {field} FROM {table} WHERE {key_field} = ?", (product_code,))
|
||||
row = cursor.fetchone()
|
||||
if row is None:
|
||||
raise WriteRefused(f"product {product_code} not found in {table}")
|
||||
if row[0] != written_value:
|
||||
raise WriteRefused(f"drift before restore: expected {written_value!r}, found {row[0]!r}")
|
||||
|
||||
if restore_value is None:
|
||||
sql = f"UPDATE {table} SET {field} = NULL WHERE {key_field} = ? AND {field} = ?"
|
||||
params = (product_code, written_value)
|
||||
else:
|
||||
sql = f"UPDATE {table} SET {field} = ? WHERE {key_field} = ? AND {field} = ?"
|
||||
params = (restore_value, product_code, written_value)
|
||||
cursor.execute(sql, params)
|
||||
rowcount = cursor.rowcount
|
||||
if rowcount != 1:
|
||||
connection.rollback()
|
||||
raise WriteRefused(f"expected rowcount 1 on restore, got {rowcount}; rolled back")
|
||||
connection.commit()
|
||||
|
||||
cursor.execute(f"SELECT {field} FROM {table} WHERE {key_field} = ?", (product_code,))
|
||||
after_row = cursor.fetchone()
|
||||
after_value = after_row[0] if after_row else None
|
||||
if after_value != restore_value:
|
||||
raise WriteRefused(
|
||||
f"post-restore verification failed: expected {restore_value!r}, found {after_value!r}"
|
||||
)
|
||||
|
||||
return WriteAuditRecord(
|
||||
product_code=product_code,
|
||||
field=field,
|
||||
before_value=written_value,
|
||||
proposed_value=restore_value,
|
||||
after_value=after_value,
|
||||
rowcount=rowcount,
|
||||
source_dataset=None,
|
||||
source_version=None,
|
||||
timestamp=now,
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
connection.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
finally:
|
||||
cursor.close()
|
||||
|
||||
|
||||
def load_local_allowlist(path=None):
|
||||
"""Exact canonical paths authorized for the local completion mission (never by directory/extension)."""
|
||||
import json
|
||||
doc = json.loads(Path(path or _LOCAL_ALLOWLIST_FILE).read_text(encoding="utf-8"))
|
||||
return {os.path.normcase(os.path.abspath(d["path"])) for d in doc["databases"]}
|
||||
|
||||
|
||||
def assert_local_completion_target(database_path, allowlist=None) -> None:
|
||||
allow = allowlist if allowlist is not None else load_local_allowlist()
|
||||
if os.path.normcase(os.path.abspath(str(database_path))) not in allow:
|
||||
raise WriteRefused(f"{database_path} is not in the exact local-completion write allowlist")
|
||||
|
||||
|
||||
def execute_guarded_product_update(connection, *, database_path, product_code, changes: dict, now: str, allowlist=None, table: str = "C000025", key_field: str = _KEY_FIELD, restore_before: bool = False):
|
||||
"""One transaction per product: every field is compare-before-write, ONE UPDATE, rowcount must be 1, then an independent re-read.
|
||||
changes: {field: (expected_before, proposed)}. restore_before=True (rollback of an audited write to its recorded BEFORE) additionally allows a NULL target. Refuses any field outside the local-completion field set, any null proposal,
|
||||
any path outside the exact allowlist, and any value that would truncate. Returns an audit dict (no secrets)."""
|
||||
assert_local_completion_target(database_path, allowlist)
|
||||
if table != "C000025":
|
||||
raise WriteRefused("only C000025 may be written")
|
||||
if not changes:
|
||||
raise WriteRefused("no changes")
|
||||
for field, (_, proposed) in changes.items():
|
||||
if field not in _LOCAL_COMPLETION_FIELDS:
|
||||
raise WriteRefused(f"{table}.{field} is not a contract write target")
|
||||
if proposed is None and not restore_before:
|
||||
raise WriteRefused(f"refuse to write a null proposed_value for {field}")
|
||||
cursor = connection.cursor()
|
||||
try:
|
||||
for field, (_, proposed) in changes.items():
|
||||
if proposed is not None:
|
||||
assert_schema_ok(cursor, table, field, proposed)
|
||||
fields = list(changes)
|
||||
cursor.execute(f"SELECT {', '.join(fields)} FROM {table} WHERE {key_field} = ?", (product_code,))
|
||||
row = cursor.fetchone()
|
||||
if row is None:
|
||||
raise WriteRefused(f"product {product_code} not found in {table}")
|
||||
for field, live in zip(fields, row):
|
||||
expected = changes[field][0]
|
||||
if _norm(live) != _norm(expected):
|
||||
raise WriteRefused(f"drift detected for {product_code}.{field}: expected {expected!r}, found {live!r}")
|
||||
sets = ", ".join(f"{f} = ?" for f in fields)
|
||||
cursor.execute(f"UPDATE {table} SET {sets} WHERE {key_field} = ?", tuple(changes[f][1] for f in fields) + (product_code,))
|
||||
if cursor.rowcount != 1:
|
||||
connection.rollback()
|
||||
raise WriteRefused(f"expected rowcount 1, got {cursor.rowcount}; rolled back")
|
||||
connection.commit()
|
||||
cursor.execute(f"SELECT {', '.join(fields)} FROM {table} WHERE {key_field} = ?", (product_code,))
|
||||
after = cursor.fetchone()
|
||||
bad = [f for f, a in zip(fields, after) if _norm(a) != _norm(changes[f][1])]
|
||||
if bad:
|
||||
raise WriteRefused(f"post-write verification failed for {product_code}: {bad}")
|
||||
return {"product_code": product_code, "rowcount": 1, "fields": {f: {"before": changes[f][0], "proposed": changes[f][1], "after": a} for f, a in zip(fields, after)}, "timestamp": now}
|
||||
except Exception:
|
||||
try:
|
||||
connection.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
finally:
|
||||
cursor.close()
|
||||
|
||||
|
||||
def _norm(v):
|
||||
"""Comparable form: text trimmed, numbers as float (Decimal/int/float from the driver), None stays None."""
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, (int, float)) or v.__class__.__name__ == "Decimal":
|
||||
return round(float(v), 6)
|
||||
return str(v).strip()
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Compare the cell states before/after a source refresh: gate_before/*_second.ndjson.gz vs local_completion/<stage>.ndjson.gz."""
|
||||
import gzip, json, sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]; stage = sys.argv[1] if len(sys.argv) > 1 else "dryrun"
|
||||
DBS = ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]
|
||||
trans = Counter(); ex = {}; man_b = man_a = 0
|
||||
for n in DBS:
|
||||
old = {}
|
||||
for l in gzip.open(ROOT / "docs/runs/gate_before" / f"{n}_second.ndjson.gz", "rt", encoding="utf-8"):
|
||||
r = json.loads(l); old[(r["row_key"], r["field"])] = r
|
||||
for l in gzip.open(ROOT / "docs/runs/local_completion" / f"{n}_{stage}.ndjson.gz", "rt", encoding="utf-8"):
|
||||
r = json.loads(l); o = old[(r["row_key"], r["field"])]
|
||||
man_b += o["status"] == "MANUAL_INPUT_REQUIRED"; man_a += r["status"] == "MANUAL_INPUT_REQUIRED"
|
||||
if o["status"] != r["status"] or o["proposed"] != r["proposed"]:
|
||||
k = (n, r["field"], o["status"], o["reason"], "->", r["status"], r["reason"][:40]); trans[k] += 1
|
||||
ex.setdefault(k, (r["codigo"], o["before"], r["proposed"]))
|
||||
print("MANUAL before", man_b, "after", man_a)
|
||||
for k, v in trans.most_common(30): print(v, k, ex[k])
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Generate the dry-run ERP write plan artifact for the 20-product sample.
|
||||
|
||||
Calls the running fiscal-hub /v1/classify/batch (read-only classification,
|
||||
no Firebird access, no dataset creation) and turns each decision into
|
||||
field-level write proposals via central.app.dry_run_planner. Never issues
|
||||
an ERP write.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "central"))
|
||||
|
||||
from app.dry_run_planner import build_field_proposals, mva_evidence, split_by_eligibility # noqa: E402
|
||||
|
||||
|
||||
def classify_batch(hub_url, token, products):
|
||||
payload = {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"]} for p in products]}
|
||||
request = urllib.request.Request(
|
||||
f"{hub_url}/v1/classify/batch",
|
||||
data=json.dumps(payload, ensure_ascii=True).encode(),
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request) as response:
|
||||
return json.load(response)["results"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--sample", default=str(ROOT / "docs" / "SAMPLE_PRODUCTS_V1.json"))
|
||||
parser.add_argument("--hub-url", default="http://localhost:8000")
|
||||
parser.add_argument("--token", required=True)
|
||||
parser.add_argument("--output", default=str(ROOT / "docs" / "DRY_RUN_PLAN_V1.json"))
|
||||
parser.add_argument("--artifact-name", default="DRY_RUN_PLAN_V1")
|
||||
parser.add_argument("--firebird-current", default=None, help="JSON file with fresh CODIGO/IPI/CEST read from Firebird (read-only). Omit to keep current values unknown.")
|
||||
args = parser.parse_args()
|
||||
|
||||
sample = json.loads(Path(args.sample).read_text(encoding="utf-8"))
|
||||
products = sample["sample_without_ncm"]
|
||||
decisions = classify_batch(args.hub_url, args.token, products)
|
||||
|
||||
fresh_by_code = {}
|
||||
if args.firebird_current:
|
||||
for row in json.loads(Path(args.firebird_current).read_text(encoding="utf-8")):
|
||||
fresh_by_code[row["CODIGO"]] = row
|
||||
|
||||
field_proposals, mva_records = [], []
|
||||
for product, decision in zip(products, decisions):
|
||||
code = product["CODIGO"]
|
||||
fresh = fresh_by_code.get(code)
|
||||
if fresh is not None:
|
||||
current = {"ipi": fresh.get("IPI"), "cest": fresh.get("CEST"), "ex_tipi": fresh.get("EX_TIPI")}
|
||||
else:
|
||||
current = {}
|
||||
if "CEST" in product:
|
||||
current["cest"] = product["CEST"]
|
||||
field_proposals.extend(build_field_proposals(code, decision, current))
|
||||
mva_records.append(mva_evidence(code, decision))
|
||||
|
||||
split = split_by_eligibility(field_proposals)
|
||||
ipi_rows = [p for p in field_proposals if p["field"] == "IPI"]
|
||||
ex_rows = [p for p in field_proposals if p["field"] == "EX_TIPI"]
|
||||
cest_rows = [p for p in field_proposals if p["field"] == "CEST"]
|
||||
ipi_metrics = {
|
||||
"ipi_numeric_proposals": sum(1 for p in ipi_rows if p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED", "CURRENT_UNKNOWN")),
|
||||
"ipi_non_numeric_status": sum(1 for p in ipi_rows if p["status"] == "NON_NUMERIC_IPI_STATUS"),
|
||||
"ipi_source_blank": sum(1 for p in ipi_rows if p["status"] == "SOURCE_BLANK"),
|
||||
"ipi_same_value": sum(1 for p in ipi_rows if p["status"] == "SAME_VALUE"),
|
||||
"ipi_change_required": sum(1 for p in ipi_rows if p["status"] == "CHANGE_REQUIRED"),
|
||||
"ipi_write_eligible": sum(1 for p in ipi_rows if p["write_eligible"]),
|
||||
# every emitted EX_TIPI row implies TIPI had at least one EX-specific entry for that NCM.
|
||||
"ex_tipi_db_match_products": len(ex_rows),
|
||||
"ex_tipi_concrete_proposals": sum(1 for p in ex_rows if p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED", "CURRENT_UNKNOWN")),
|
||||
"ex_tipi_candidates": sum(1 for p in ex_rows if p["status"] == "CANDIDATE"),
|
||||
"ex_tipi_write_eligible": sum(1 for p in ex_rows if p["write_eligible"]),
|
||||
}
|
||||
cest_metrics = {
|
||||
"cest_db_match_products": sum(1 for p in cest_rows if p["status"] != "NO_CEST_MAPPING_FOUND" and p["reason_code"] != "NO_CEST_MAPPING_FOUND_FOR_NCM"),
|
||||
"cest_confirmed": sum(1 for p in cest_rows if p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED", "CURRENT_UNKNOWN")),
|
||||
"cest_candidates": sum(1 for p in cest_rows if p["status"] == "CANDIDATE"),
|
||||
"cest_unresolved": sum(1 for p in cest_rows if p["status"] == "UNRESOLVED"),
|
||||
"cest_write_eligible": sum(1 for p in cest_rows if p["write_eligible"]),
|
||||
}
|
||||
|
||||
artifact = {
|
||||
"artifact": args.artifact_name,
|
||||
"firebird_read": "OK" if fresh_by_code else "BLOCKED_EXTERNAL_MISSING_CREDENTIALS",
|
||||
"firebird_writes": 0,
|
||||
"products_planned": len(products),
|
||||
"current_value_source": (
|
||||
f"live read-only SELECT from C000025 via {args.firebird_current}"
|
||||
if fresh_by_code else
|
||||
"docs/SAMPLE_PRODUCTS_V1.json (CEST) — C000025.IPI has no authorized offline source; FB_USER/FB_PASSWORD unavailable in this environment"
|
||||
),
|
||||
"field_proposals": field_proposals,
|
||||
"mva_evidence": mva_records,
|
||||
"ipi_metrics": ipi_metrics,
|
||||
"cest_metrics": cest_metrics,
|
||||
"write_eligible_count": len(split["eligible"]),
|
||||
"write_blocked_count": len(split["blocked"]),
|
||||
}
|
||||
Path(args.output).write_text(json.dumps(artifact, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(json.dumps({
|
||||
"products": len(products),
|
||||
"field_proposals": len(field_proposals),
|
||||
"write_eligible": len(split["eligible"]),
|
||||
"write_blocked": len(split["blocked"]),
|
||||
"output": args.output,
|
||||
}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Compose ERP_FISCAL_FIELD_MAP_V1, FISCAL_CONTEXT_REQUIREMENTS_V1, FISCAL_CONTEXT_COVERAGE_V1 and
|
||||
NEXT_FISCAL_FIELD_IMPLEMENTATION_V1 from the read-only discovery outputs (docs/runs/*).
|
||||
No database access."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DOCS = ROOT / "docs"
|
||||
schema = json.loads((DOCS / "runs" / "erp_schema_inventory.json").read_text(encoding="utf-8"))
|
||||
hist = json.loads((DOCS / "runs" / "erp_fiscal_history_analysis.json").read_text(encoding="utf-8"))
|
||||
extra = json.loads((DOCS / "runs" / "erp_fiscal_history_extra.json").read_text(encoding="utf-8"))
|
||||
audit = json.loads((DOCS / "runs" / "erp_c000025_audit.json").read_text(encoding="utf-8"))
|
||||
cov_v1 = json.loads((DOCS / "FISCAL_DECISION_COVERAGE_V1.json").read_text(encoding="utf-8"))
|
||||
|
||||
tables = schema["tables"]
|
||||
fiscal_tables = {t: v for t, v in tables.items() if v["fiscal_cols"] and v["rows"]}
|
||||
consistent = extra["agreement_consistent_joins_only"]
|
||||
nf = hist["nfce_join"]
|
||||
items_analyzed = nf["items_joined_to_C000025"] + hist["C000032"]["joined"]["JOINED"] + hist["C000062"]["joined"]["JOINED"]
|
||||
|
||||
VENDOR = "vendor scripts D:/PROJETOS/GTISISTEMAS/Diretorios/rel/Novo_Script_Banco.sql + atualiza/v3.5.5.NFCe/script/*"
|
||||
|
||||
|
||||
def f(table, field, dbtype, semantic, usage, conf, evidence, sources, samples, write=False):
|
||||
return {"table": table, "field": field, "db_type": dbtype, "semantic_name": semantic, "usage_class": usage, "confidence": conf,
|
||||
"evidence": evidence, "source_objects": sources, "sample_values": samples, "write_candidate": write}
|
||||
|
||||
|
||||
agree_all = nf["agreement_item_vs_product_default"]
|
||||
FIELDS = [
|
||||
f("C000025", "CFOP", "VARCHAR(4)", "CFOP padrão de saída do produto", "PRODUCT_DEFAULT", "HIGH",
|
||||
f"NFC-e item CFOP equals product default in {agree_all['CFOP']['MATCH']}/{agree_all['CFOP']['MATCH']+agree_all['CFOP']['MISMATCH']} joined items (identity-consistent: {consistent['CFOP']['MATCH']}/{consistent['CFOP']['MATCH']+consistent['CFOP']['MISMATCH']}); values are official 5xxx CFOPs; item CFOP is the emitted value. Still operation-dependent (C000030 naturezas carry 5102/5405/6102...).",
|
||||
["C000025", "NFCE_ITENS", "C000030"], ["5102", "5405", "5656"]),
|
||||
f("C000025", "CFOP_ENTRADA", "CHAR(4)", "CFOP padrão de entrada (campo existe no DDL do fornecedor)", "LEGACY_UNUSED", "MEDIUM",
|
||||
f"0/521 populated; DDL exists ({VENDOR}); company-level default lives in C000001.IND_CFOP_ENTRADA=1949", ["C000025", "C000001"], [None]),
|
||||
f("C000025", "CST", "VARCHAR(10)", "CST ICMS de saída = origem(1)+CST(2)", "PRODUCT_DEFAULT", "HIGH",
|
||||
f"vendor table C000083 (11 rows 000..090 'NACIONAL ...') defines CST as 3-digit origin+CST; NFC-e item CST matches product default in {consistent['CST']['MATCH']}/{consistent['CST']['MATCH']} identity-consistent joins",
|
||||
["C000025", "C000083", "NFCE_ITENS"], ["040", "000", "060"]),
|
||||
f("C000025", "CST_ENTRADA", "CHAR(4)", "CST padrão de entrada", "LEGACY_UNUSED", "MEDIUM", "0/521 populated; DDL exists in vendor scripts", ["C000025"], [None]),
|
||||
f("C000025", "CSOSN", "VARCHAR(60)", "CSOSN de saída (Simples Nacional)", "PRODUCT_DEFAULT", "HIGH",
|
||||
f"NFC-e item CSOSN equals product default in {consistent['CSOSN']['MATCH']}/{consistent['CSOSN']['MATCH']+consistent['CSOSN']['MISMATCH']} identity-consistent joins; company default C000004.ICMS_CSOSN=102; values 101/102/103/500 in CSOSN dataset",
|
||||
["C000025", "C000004", "NFCE_ITENS"], ["102", "500", "101"]),
|
||||
f("C000025", "GRUPO_ICMS", "VARCHAR(9)", "grupo ICMS do produto (espelha CSOSN/ST nas NFC-e)", "PRODUCT_DEFAULT", "MEDIUM",
|
||||
"487 '102', 2 '90'; NFCE_ITENS.GRUPO_ICMS carries 102/500/blank; exact semantics not documented", ["C000025", "NFCE_ITENS"], ["102", "90"]),
|
||||
f("C000025", "ORIGEM", "VARCHAR(1)", "origem (campo textual legado)", "LEGACY_UNUSED", "HIGH", "0/521 and 0/4905 (C000032.ORIGEM) populated", ["C000025", "C000032"], [None]),
|
||||
f("C000025", "COD_ORIGEM_MERCADORIA", "INTEGER", "código de origem da mercadoria (0-8)", "PRODUCT_DEFAULT", "LOW",
|
||||
f"exists in vendor DDL as INTEGER; only 1/521 populated (value 2); origin is otherwise embedded in CST digit 1; not used in any emitted NFC-e column", ["C000025"], [2]),
|
||||
f("C000025", "COD_ENTRADA_PIS", "VARCHAR(3)", "CST PIS de entrada", "LEGACY_UNUSED", "LOW", "2/521 populated with inconsistent formatting ('2','73')", ["C000025"], ["2", "73"]),
|
||||
f("C000025", "COD_SAIDA_PIS", "VARCHAR(3)", "CST PIS de saída", "LEGACY_UNUSED", "LOW",
|
||||
"3/521 populated ('01','04','2'); NFC-e items carry CST_PIS_COFINS '01' regardless, so the product field is not the source", ["C000025", "NFCE_ITENS"], ["01", "04", "2"]),
|
||||
f("C000025", "COD_ENTRADA_COFINS", "VARCHAR(3)", "CST COFINS de entrada", "LEGACY_UNUSED", "LOW", "2/521 populated", ["C000025"], ["2", "73"]),
|
||||
f("C000025", "COD_SAIDA_COFINS", "VARCHAR(3)", "CST COFINS de saída", "LEGACY_UNUSED", "LOW", "3/521 populated", ["C000025"], ["01", "04", "2"]),
|
||||
f("C000025", "ALIQ_PIS", "VARCHAR(8)", "alíquota PIS (texto)", "LEGACY_UNUSED", "LOW", "3/521 populated; NFC-e ALIQUOTA_PIS is always 0 (Simples)", ["C000025", "NFCE_ITENS"], ["0,65", "2"]),
|
||||
f("C000025", "ALIQ_COFINS", "VARCHAR(8)", "alíquota COFINS (texto)", "LEGACY_UNUSED", "LOW", "3/521 populated; NFC-e ALIQUOTA_COFINS is always 0", ["C000025", "NFCE_ITENS"], ["2", "3"]),
|
||||
f("C000025", "CST_IPI", "SMALLINT", "CST do IPI", "LEGACY_UNUSED", "LOW", "1/521 populated ('040', 3 digits although IPI CST has 2)", ["C000025"], ["040"]),
|
||||
f("C000025", "INCIDENCIA_PISCOFINS", "VARCHAR(30)", "tipo de incidência PIS/COFINS do produto", "PRODUCT_DEFAULT", "HIGH", "NENHUMA 519, MONOFÁSICA 1, SUBST.TRIBUTÁRIA 1; ties to C000025.PISCOFINS flag (N/S)", ["C000025"], ["NENHUMA", "MONOFÁSICA"]),
|
||||
f("C000025", "CEST", "VARCHAR(7)", "CEST do produto (7 dígitos sem pontuação)", "PRODUCT_DEFAULT", "HIGH",
|
||||
f"NFC-e item CEST digits equal product CEST in {consistent['CEST']['MATCH']}/{consistent['CEST']['MATCH']+consistent['CEST']['MISMATCH']} identity-consistent joins", ["C000025", "NFCE_ITENS"], ["0601100", "1708400"], write=True),
|
||||
f("C000025", "IPI", "DOUBLE PRECISION", "alíquota de IPI %", "PRODUCT_DEFAULT", "HIGH", "confirmed in earlier missions and post-write verified; 109 populated", ["C000025"], [19.5, 3.25], write=True),
|
||||
f("C000025", "EX_TIPI", "VARCHAR(3)", "código EX da TIPI", "PRODUCT_DEFAULT", "HIGH", "confirmed in earlier missions; 0 populated", ["C000025"], [None], write=True),
|
||||
f("C000025", "ALIQUOTA", "NUMERIC(15,2)", "alíquota ICMS % do produto", "UNKNOWN", "LOW",
|
||||
"product default 20.5 (492 rows) but NFC-e items use ALIQUOTA 18 (29825 rows) — product value is not what is emitted", ["C000025", "NFCE_ITENS"], [20.5, 17]),
|
||||
f("C000025", "MARGEM_AGREGADA", "NUMERIC(15,2)", "candidato a MVA/margem de ST", "LEGACY_UNUSED", "LOW",
|
||||
f"0/521 populated; C000062.MARGEM_AGREGADA is 0.0 in all 693 rows even though ICMS_SUB<>0 in {hist['C000062_mva_usage'][2]} rows (no calculation use evidenced); group-level PRODUTOS_MARGEM_AGREGADA(MVA_INDUSTRIA/MVA_DISTRIBUIDOR) has {hist['PRODUTOS_MARGEM_AGREGADA_rows']} rows and no link to C000025",
|
||||
["C000025", "C000062", "PRODUTOS_MARGEM_AGREGADA"], [None]),
|
||||
f("C000004", "CRT", "VARCHAR(1)", "código de regime tributário da empresa", "COMPANY_DEPENDENT", "HIGH", "value 1; EMISSOR.EMITENTE_CRT=1 agrees; OPTANTE_SIMPLES=S", ["C000004", "EMISSOR"], ["1"]),
|
||||
f("C000004", "UF", "VARCHAR(2)", "UF do emitente", "COMPANY_DEPENDENT", "HIGH", "BA; matches IBGE 2928703 (Bahia)", ["C000004"], ["BA"]),
|
||||
f("C000004", "ICMS_CSOSN", "VARCHAR(150)", "CSOSN padrão da empresa", "COMPANY_DEPENDENT", "MEDIUM", "value 102; equals 487/521 product defaults", ["C000004"], ["102"]),
|
||||
f("C000004", "SUBSTITUTO_TRIBUTARIO / CONTRIBUINTE_IPI / OPTANTE_SIMPLES", "VARCHAR(1)", "flags fiscais da empresa", "COMPANY_DEPENDENT", "HIGH", "N / N / S", ["C000004"], ["N", "N", "S"]),
|
||||
f("C000030", "CFOP,NATUREZA,TIPO", "VARCHAR/INTEGER", "cadastro de naturezas de operação (TIPO 1=entrada 272, 2=saída 318)", "OPERATION_DEFAULT", "HIGH",
|
||||
f"590 rows loaded by vendor script 'Script CFOP C000030.txt'; CFOP first digit distribution {hist['C000030_cfop_first_digit']}; contains 5102/5405/5656/6102/1102", ["C000030"], ["1102 Compra p/ comercialização", "5102"]),
|
||||
f("C000001", "IND_CFOP_ENTRADA/SAIDA/PERDA", "VARCHAR(5)", "CFOP padrão da empresa por finalidade", "OPERATION_DEFAULT", "HIGH", f"values {hist['C000001_cfop_defaults']} (1949/5949/5927)", ["C000001"], hist["C000001_cfop_defaults"]),
|
||||
f("NFCE_ITENS", "CFOP,CST,GRUPO_ICMS,CSOSN,NCM,CEST,CST_PIS_COFINS", "mixed", "valores efetivamente emitidos em NFC-e (34,946 itens 2018-03..2026-08)", "DOCUMENT_CALCULATED", "HIGH",
|
||||
"item-level emitted fiscal values; NCM/CEST/CSOSN/CST_PIS_COFINS populated only on 4,668 newer items (2022-06+)", ["NFCE_ITENS", "NFCE"], ["5102", "040", "102", "01"]),
|
||||
f("NFCE_ITENS", "ALIQUOTA_PIS/ALIQUOTA_COFINS/BASE_CALCULO_*", "DOUBLE/NUMERIC", "PIS/COFINS emitido", "DOCUMENT_CALCULATED", "HIGH", "always 0 on 4,668 items (Simples Nacional: no PIS/COFINS calculation in NFC-e)", ["NFCE_ITENS"], [0]),
|
||||
f("NFE_ALIQUOTA_INTERESTADUAL", "ICMS,ORIGEM,...", "mixed", "tabela de alíquotas interestaduais (729 linhas)", "OPERATION_DEFAULT", "MEDIUM", "reference table keyed by UF pair/origin; relevant only for interstate operations", ["NFE_ALIQUOTA_INTERESTADUAL"], []),
|
||||
]
|
||||
|
||||
fmap = {"artifact": "ERP_FISCAL_FIELD_MAP_V1", "database": "TESTESOFICIAIS.FDB (read-only)", "firebird_writes": 0,
|
||||
"evidence_sources": ["live RDB$ catalog (334 tables)", "value distributions", "NFCE_ITENS history join", VENDOR],
|
||||
"caveat": "57% of NFC-e items with an NCM carry a different NCM than the product currently has for the same CODPRODUTO (identity drift); history comparisons are therefore reported on identity-consistent joins.",
|
||||
"fields": FIELDS}
|
||||
(DOCS / "ERP_FISCAL_FIELD_MAP_V1.json").write_text(json.dumps(fmap, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
|
||||
# ---------------------------------------------------------------- context requirements
|
||||
AV, CFG, JOB, MISS = "AVAILABLE_FROM_ERP", "AVAILABLE_FROM_AGENT_CONFIG", "AVAILABLE_FROM_JOB", "MISSING"
|
||||
REQ = {
|
||||
"CFOP": {"official_basis": "CFOP table (IT 2023.002): direction, UF scope, ST, purpose in title/description/ind* flags", "inputs": {
|
||||
"direction (entrada/saída)": [AV, "C000030.TIPO (1/2) per natureza; NFC-e is always saída"],
|
||||
"operation_type / natureza": [AV, "C000030 (590 rows) at document level; not derivable from a product alone"],
|
||||
"origin_uf": [AV, "C000004.UF=BA"],
|
||||
"destination_uf": [JOB, "NFC-e is intra-UF by nature; NF-e needs customer UF (customer tables) — absent from a product-only job"],
|
||||
"customer_taxpayer_status": [MISS, "per-document/customer; not in product request"],
|
||||
"st_applicability": [AV, "BA_ST_MVA + CEST datasets, only 31 products MVA_RESOLVED"],
|
||||
"purpose (revenda/consumo/devolução)": [MISS, "only natureza text; no structured field"]}},
|
||||
"CSOSN": {"official_basis": "MOC CSOSN table + CRT=1", "inputs": {
|
||||
"CRT": [AV, "C000004.CRT=1, EMISSOR.EMITENTE_CRT=1"],
|
||||
"company_default_csosn": [AV, "C000004.ICMS_CSOSN=102"],
|
||||
"st_applicability": [AV, "same as CFOP (partial)"],
|
||||
"st_already_retained_by_supplier": [MISS, "requires purchase documents per product; not structured"],
|
||||
"operation_type": [MISS, "document-level"],
|
||||
"destination_context": [JOB, "document-level"]}},
|
||||
"CST_ICMS": {"official_basis": "not emitted while CRT=1 (CSOSN used)", "inputs": {
|
||||
"CRT": [AV, "C000004.CRT"], "regime_change_flag": [CFG, "only if client leaves Simples"], "operation_type": [MISS, "document-level"]}},
|
||||
"ORIGEM": {"official_basis": "MOC origin table 0-8", "inputs": {
|
||||
"objective_product_origin (import/national)": [MISS, "C000025.ORIGEM 0/521; COD_ORIGEM_MERCADORIA 1/521; ERP CST digit 1 is a config value not proof"]}},
|
||||
"PIS_COFINS": {"official_basis": "CST PIS/COFINS table + regime; rate tables 4.3.x deferred", "inputs": {
|
||||
"regime": [AV, "CRT=1 / OPTANTE_SIMPLES=S (NFC-e PIS/COFINS always 0)"],
|
||||
"product_treatment": [AV, "C000025.INCIDENCIA_PISCOFINS (519 NENHUMA, 1 MONOFÁSICA, 1 SUBST.TRIBUTÁRIA)"],
|
||||
"cst_rule_table": [MISS, "no NCM→CST PIS/COFINS rule in acquired datasets (deferred by business scope)"],
|
||||
"operation_type": [MISS, "document-level"]}},
|
||||
"FCP": {"official_basis": "FCP rate per UF (dataset) + state item list", "inputs": {
|
||||
"company_uf": [AV, "C000004.UF=BA"], "destination_uf": [JOB, "document-level"],
|
||||
"item_in_fcp_list_for_uf": [MISS, "no NCM/CEST list of FCP-subject items in datasets"],
|
||||
"operation_interstate_or_internal": [JOB, "document-level"]}},
|
||||
"RTC_CCLASSTRIB": {"official_basis": "cClassTrib table + LC 214 annexes", "inputs": {
|
||||
"ncm_to_annex_mapping": [MISS, "LC 214 annex NCM lists are not in acquired datasets"],
|
||||
"operation_nature": [MISS, "document-level"], "customer_type": [MISS, "document-level"], "CRT": [AV, "C000004.CRT"]}},
|
||||
"RTC_CBS_RATE": {"official_basis": "IT 2026.002: 2026=0.009, 2027 pending legislation", "inputs": {
|
||||
"cclasstrib_resolved": [MISS, "blocked by RTC_CCLASSTRIB"], "reference_year": [JOB, "document date"]}},
|
||||
"RTC_CCREDPRES": {"official_basis": "cCredPres official conditions", "inputs": {
|
||||
"condition_flags (ZFM, presumed credit eligibility)": [MISS, "no ERP field"]}},
|
||||
"MVA": {"official_basis": "RICMS-BA Anexo 1 item/CEST", "inputs": {
|
||||
"confirmed_cest": [AV, "only 15 CEST candidates remain, 1 historically supported"], "erp_destination_field": [MISS, "NO_PRODUCT_FIELD proven"]}},
|
||||
}
|
||||
n_av = n_cfg = n_job = n_miss = 0
|
||||
for dom in REQ.values():
|
||||
for k, (state, _) in dom["inputs"].items():
|
||||
n_av += state == AV; n_cfg += state == CFG; n_job += state == JOB; n_miss += state == MISS
|
||||
REQ_DOC = {"artifact": "FISCAL_CONTEXT_REQUIREMENTS_V1", "domains": {d: {"official_basis": v["official_basis"], "inputs": {k: {"status": s, "evidence": e} for k, (s, e) in v["inputs"].items()}} for d, v in REQ.items()},
|
||||
"summary": {"AVAILABLE_FROM_ERP": n_av, "AVAILABLE_FROM_AGENT_CONFIG": n_cfg, "AVAILABLE_FROM_JOB": n_job, "MISSING": n_miss}}
|
||||
(DOCS / "FISCAL_CONTEXT_REQUIREMENTS_V1.json").write_text(json.dumps(REQ_DOC, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
|
||||
|
||||
def ctx(d):
|
||||
st = [s for s, _ in REQ[d]["inputs"].values()]
|
||||
return {"available": sum(1 for s in st if s in (AV, CFG)), "job_dependent": st.count(JOB), "missing": st.count(MISS), "total": len(st), "complete": all(s in (AV, CFG) for s in st)}
|
||||
|
||||
|
||||
sv = hist["shadow_validation"]
|
||||
matches = sum(v.get("MATCH", 0) for k, v in sv.items() if k != "items")
|
||||
mism = sum(v.get("MISMATCH", 0) for k, v in sv.items() if k != "items")
|
||||
unres = sum(v.get("UNRESOLVED", 0) for k, v in sv.items() if k != "items")
|
||||
COV = {"artifact": "FISCAL_CONTEXT_COVERAGE_V1", "TOTAL_PRODUCTS": 521, "firebird_writes": 0,
|
||||
"ERP_TABLES_INSPECTED": len(tables), "ERP_TABLES_NON_EMPTY": sum(1 for v in tables.values() if v["rows"]),
|
||||
"ERP_CONTEXT_TABLES_FOUND": sorted(["C000004", "C000001", "C000030", "C000083", "NFE_ALIQUOTA_INTERESTADUAL", "EMISSOR"]),
|
||||
"ERP_FISCAL_TABLES_FOUND": len(fiscal_tables),
|
||||
"HISTORICAL_FISCAL_DOCUMENT_TABLES": ["NFCE", "NFCE_ITENS", "C000032", "C000061", "C000062"],
|
||||
"HISTORICAL_FISCAL_DOCUMENTS_FOUND": {"NFCE": hist["nfce_docs"]["rows"], "C000061": 50},
|
||||
"HISTORICAL_ITEMS_ANALYZED": items_analyzed,
|
||||
"CFOP_MAPPING_CONFIRMED": True, "CST_MAPPING_CONFIRMED": True, "CSOSN_MAPPING_CONFIRMED": True,
|
||||
"PIS_MAPPING_CONFIRMED": False, "COFINS_MAPPING_CONFIRMED": False, "ORIGIN_MAPPING_CONFIRMED": False, "MVA_FIELD_CONFIRMED": False,
|
||||
"CFOP_CONTEXT_AVAILABLE": ctx("CFOP"), "CSOSN_CONTEXT_AVAILABLE": ctx("CSOSN"), "CST_CONTEXT_AVAILABLE": ctx("CST_ICMS"),
|
||||
"PIS_COFINS_CONTEXT_AVAILABLE": ctx("PIS_COFINS"), "FCP_CONTEXT_AVAILABLE": ctx("FCP"), "RTC_CONTEXT_AVAILABLE": ctx("RTC_CCLASSTRIB"),
|
||||
"CONTEXT_AVAILABLE": n_av + n_cfg, "CONTEXT_JOB_DEPENDENT": n_job, "CONTEXT_MISSING": n_miss,
|
||||
"history_agreement_all_joins": agree_all, "history_agreement_identity_consistent_joins": consistent,
|
||||
"identity_drift": {"consistent_items": extra["ncm_identity_consistent_items"], "inconsistent_items": extra["ncm_identity_inconsistent_items"]},
|
||||
"cest_historically_supported": extra["cest_historically_supported"],
|
||||
"shadow_validation_engine_vs_history": {"items": sv["items"], "MATCH": matches, "MISMATCH": mism, "UNRESOLVED": unres, "by_domain": {k: v for k, v in sv.items() if k != "items"},
|
||||
"note": "NCM 'mismatch' = product's current NCM differs from the NCM emitted on that CODPRODUTO (identity drift), not an engine error; all operation-dependent domains stay UNRESOLVED by design."}}
|
||||
(DOCS / "FISCAL_CONTEXT_COVERAGE_V1.json").write_text(json.dumps(COV, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
|
||||
# ---------------------------------------------------------------- priorities
|
||||
d = cov_v1["domains"]
|
||||
def hv(field, ok):
|
||||
return ok
|
||||
ROWS = [
|
||||
# field, mapping, ctx_complete, rule, potential, hist_validation, next_action
|
||||
("IPI", "CONFIRMED", True, True, d["IPI"]["RULE_RESOLVED"], "n/a in NFC-e items (no IPI column)", "Maintain; add NT/blank policy decision."),
|
||||
("CEST", "CONFIRMED", True, True, d["CEST"]["RULE_CANDIDATE"], f"{consistent['CEST']['MATCH']}/{consistent['CEST']['MATCH']+consistent['CEST']['MISMATCH']} product==emitted; {extra['cest_historically_supported']}/15 candidates history-supported", "Use NFC-e history as EVIDENCE to narrow candidates; write only single defensible matches."),
|
||||
("EX_TIPI", "CONFIRMED", True, True, d["EX_TIPI"]["RULE_CANDIDATE"], "none", "Description-evidence rule already in engine; review queue only."),
|
||||
("NCM", "CONFIRMED", True, True, d["NCM"]["RULE_CANDIDATE"], "identity drift: 57% of NFC-e NCMs differ from current product NCM", "Investigate product-code reuse before trusting history; keep review-only."),
|
||||
("CFOP", "CONFIRMED", False, False, 521, f"{consistent['CFOP']['MATCH']}/{consistent['CFOP']['MATCH']+consistent['CFOP']['MISMATCH']} product default==emitted", "Model NFC-e sale context (saída, intra-UF, consumidor final) and define an official-table-backed rule; validate vs NFCE_ITENS."),
|
||||
("CSOSN", "CONFIRMED", False, False, 521, f"{consistent['CSOSN']['MATCH']}/{consistent['CSOSN']['MATCH']+consistent['CSOSN']['MISMATCH']} product default==emitted", "Need ST-retention context; build shadow rule (CRT=1 + ST applicability) and score against NFCE_ITENS."),
|
||||
("CST_ICMS", "CONFIRMED", False, False, 521, f"{consistent['CST']['MATCH']}/{consistent['CST']['MATCH']} product==emitted", "Not needed while CRT=1; keep as validated default."),
|
||||
("MVA", "NO_PRODUCT_FIELD", False, True, d["MVA"]["RULE_RESOLVED"], "none (MARGEM_AGREGADA unused)", "Decide with vendor whether MARGEM_AGREGADA is honoured by the emitter before any write."),
|
||||
("PIS_COFINS_CST", "UNCONFIRMED", False, False, 521, "NFC-e always '01'/blank, rates 0", "Needs vendor confirmation of which field feeds CST_PIS_COFINS (not C000025.COD_SAIDA_*)."),
|
||||
("ORIGEM", "UNCONFIRMED", False, False, 521, "CST digit 1 consistent (0)", "Vendor confirmation of COD_ORIGEM_MERCADORIA usage; obtain objective origin."),
|
||||
("FCP", "n/a (no product field)", False, False, 521, "none", "Needs FCP item list per UF; document-level only."),
|
||||
("RTC", "n/a (no ERP field)", False, False, 521, "none", "Needs NCM→LC 214 annex mapping dataset; then document-level."),
|
||||
]
|
||||
def score(r):
|
||||
return (r[1] == "CONFIRMED", r[2], r[3], r[4], "n/a" not in r[5] and r[5] != "none")
|
||||
ROWS.sort(key=score, reverse=True)
|
||||
md = ["# NEXT_FISCAL_FIELD_IMPLEMENTATION_V1", "",
|
||||
"Mechanical ordering: (1) ERP mapping confirmed, (2) required context available, (3) official rule available, (4) potential products, (5) history validation exists. Source: `docs/ERP_FISCAL_FIELD_MAP_V1.json`, `docs/FISCAL_CONTEXT_REQUIREMENTS_V1.json`, `docs/FISCAL_CONTEXT_COVERAGE_V1.json`.", "",
|
||||
"| FIELD | ERP_MAPPING | CONTEXT_AVAILABLE | RULE_AVAILABLE | POTENTIAL_PRODUCTS | HISTORICAL_VALIDATION | NEXT_ACTION |", "|---|---|---|---|---|---|---|"]
|
||||
for r in ROWS:
|
||||
md.append(f"| {r[0]} | {r[1]} | {'YES' if r[2] else 'NO'} | {'YES' if r[3] else 'NO'} | {r[4]} | {r[5]} | {r[6]} |")
|
||||
md += ["", "Caveat: NFC-e history joins by CODPRODUTO show identity drift (item NCM != product NCM on 57% of items); trust only identity-consistent joins."]
|
||||
(DOCS / "NEXT_FISCAL_FIELD_IMPLEMENTATION_V1.md").write_text("\n".join(md) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"tables": len(tables), "fiscal_tables": len(fiscal_tables), "items_analyzed": items_analyzed, "ctx": [n_av, n_cfg, n_job, n_miss], "sim": [matches, mism, unres],
|
||||
"confirmed": sum(1 for x in FIELDS if x["confidence"] in ("HIGH", "MEDIUM")), "fields": len(FIELDS)}))
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Write docs/FISCAL_FIELD_CONTRACT_V1.json from central.app.field_contract (single source of truth). No database access."""
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from central.app.field_contract import CONTRACT, CONTRACT_VERSION, RTC_CAPABILITY, WRITE_FIELDS # noqa: E402
|
||||
from central.app.context_keys import ALLOWED_KEYS # noqa: E402
|
||||
|
||||
ui = json.loads((ROOT / "docs" / "SGV_PRODUCT_FISCAL_UI_MAP_V1.json").read_text(encoding="utf-8"))
|
||||
bound = {f["field"]: f for f in ui["fields"] if f.get("field")}
|
||||
doc = {"artifact": "FISCAL_FIELD_CONTRACT_V1", "contract_version": CONTRACT_VERSION, "product_table": "C000025", "write_targets": len(WRITE_FIELDS),
|
||||
"final_states_per_product_field": ["ALREADY_CORRECT", "CHANGE_REQUIRED", "CORRECTED", "NOT_APPLICABLE", "MANUAL_INPUT_REQUIRED(<input_key>)"],
|
||||
"declared_valid_policy": "A valid ERP declaration that no proven evidence contradicts is ALREADY_CORRECT with reason DECLARED_VALID_NOT_CONTRADICTED; rule-proven results carry other reasons.",
|
||||
"operator_context_keys": sorted(ALLOWED_KEYS), "context_hierarchy": "PRODUCT > GROUP (CODGRUPO) > NCM > COMPANY (Hub table fiscal_context_entries; API /v1/context; CLI scripts/import_fiscal_context.py)",
|
||||
"unresolved_official_sources": {
|
||||
"PIS/COFINS": "No official rule table in the ACTIVE core datasets maps regime/NCM to CST PIS/COFINS or rates (SPED Tabelas 4.3.x + the Simples Nacional PIS/COFINS guidance are the missing sources; they are manual downloads and were deferred by business scope). Contract answer: company-level `pis_cofins_profile` supplied by the accountant; never assumed 0%.",
|
||||
"ICMS internal rate (regime normal)": "RICMS-BA articles on internal rates/FECP are not in the acquired datasets: `icms_rate` is an operator fact for CRT 2/3 taxed items.",
|
||||
"Monofásico incidence": "Official monophasic NCM list not acquired: INCIDENCIA_PISCOFINS keeps the ERP declaration or takes an operator `incidence`."},
|
||||
"rtc": {"status": {"RTC_CCLASSTRIB": "SUPPORTED_BY_CURRENT_ERP_SCHEMA_UI_BINDING_UNCONFIRMED", "RTC_CCREDPRES": "ERP_SCHEMA_NOT_READY", "RTC_CBS_RATES": "ERP_SCHEMA_NOT_READY"},
|
||||
"evidence": "C000025.CCLASSTRIB VARCHAR(6) exists (all NULL) in CONSTRUCAO, MMMODAS, RINCAO and UNIAO (a newer SGV schema) and is absent in TESTESOFICIAIS. The SGV executable available locally (5.0.6.124) neither binds nor mentions it and the fiscal screen shows no such control, so the binding cannot be confirmed; no cCredPres/CBS/IBS column exists in any database.",
|
||||
"write_target": False, "why_not_written": "no proven UI binding and no NCM -> LC 214 annex (cClassTrib) source in the ACTIVE datasets; a value here would be invented",
|
||||
"to_close": ["the newer SGV executable/screen that binds CCLASSTRIB (binding proof)", "official LC 214/2025 annex NCM lists to derive cClassTrib deterministically"]},
|
||||
"not_persisted_controls": {"CSOSN significado (ComboBox1)": "presentation only, not data-aware", "CODALIQUOTA": "exists in the ERP, empty, not on the fiscal screen: not a requirement without new evidence"},
|
||||
"fields": {}}
|
||||
for f in WRITE_FIELDS:
|
||||
c = dict(CONTRACT[f])
|
||||
c["UI_BINDING"] = {"classification": bound[f]["classification"] if f in bound else "CONFIRMED_STRONG", "evidence": bound[f]["evidence_basis"] if f in bound else None}
|
||||
doc["fields"][f] = c
|
||||
# EX_TIPI is not a control on the screen (audited separately) and COD_ORIGEM binding is closed by the DFM OnClick handler:
|
||||
doc["fields"]["EX_TIPI"]["UI_BINDING"] = {"classification": "CONFIRMED_STRONG", "evidence": "user-declared earlier missions; TIPI column EX_TIPI"}
|
||||
doc["fields"]["COD_ORIGEM_MERCADORIA"]["UI_BINDING"] = {"classification": "CONFIRMED_STRONG", "evidence": "DFM: RgOrigemMercadoria (TRadioGroup, OnClick=RgOrigemMercadoriaClick, no DataField) + hidden data-aware TDBEdit CodOrigemMercadoria bound to COD_ORIGEM_MERCADORIA in the same tab; the only origin-bound control of the form; values 0/1/2 observed in 4 databases"}
|
||||
(ROOT / "docs" / "FISCAL_FIELD_CONTRACT_V1.json").write_text(json.dumps(doc, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
print(len(doc["fields"]), "contracts")
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Read-only fiscal coverage matrix over all C000025 products in TESTESOFICIAIS.FDB.
|
||||
|
||||
No Firebird writes. Classifies every product via the Hub's /v1/classify/batch
|
||||
(itself read-only against Firebird — the Hub never touches Firebird at all)
|
||||
and separates DATA_AVAILABLE (a dataset exists to consult) from RULE_RESOLVED
|
||||
(a single defensible determination was reached) per the mission's own
|
||||
instruction not to conflate the two.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
|
||||
from gti_fiscal_agent.firebird_reader import read_products # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def classify_batch(hub_url, token, products, chunk_size=100):
|
||||
results = []
|
||||
for start in range(0, len(products), chunk_size):
|
||||
chunk = products[start:start + chunk_size]
|
||||
payload = {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]}
|
||||
request = urllib.request.Request(
|
||||
f"{hub_url}/v1/classify/batch",
|
||||
data=json.dumps(payload, ensure_ascii=True).encode(),
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
results.extend(json.load(response)["results"])
|
||||
return results
|
||||
|
||||
|
||||
def get_dataset_status(hub_url, token):
|
||||
request = urllib.request.Request(f"{hub_url}/v1/datasets/status", headers={"Authorization": f"Bearer {token}"})
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def main():
|
||||
token = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("AGENT_DEV_TOKEN", "")
|
||||
hub_url = os.environ.get("GTI_HUB_URL", "http://localhost:8000")
|
||||
load_dotenv(ROOT / ".env")
|
||||
|
||||
products = read_products(os.environ)
|
||||
total = len(products)
|
||||
decisions = classify_batch(hub_url, token, products)
|
||||
datasets = get_dataset_status(hub_url, token)
|
||||
active_kinds = {d["kind"] for d in datasets if d["status"] == "ACTIVE"}
|
||||
|
||||
ncm_status = Counter(d.get("status") for d in decisions)
|
||||
ncm_missing = sum(1 for p in products if not (p["CLASSIFICACAO_FISCAL"] or "").strip())
|
||||
ncm_valid = ncm_status.get("EXACT_NCM", 0)
|
||||
ncm_legacy = ncm_status.get("LEGACY_NCM", 0)
|
||||
ncm_invalid = sum(
|
||||
1 for p, d in zip(products, decisions)
|
||||
if (p["CLASSIFICACAO_FISCAL"] or "").strip() and d.get("status") not in ("EXACT_NCM", "LEGACY_NCM")
|
||||
)
|
||||
|
||||
ipi_buckets = Counter()
|
||||
cest_match = cest_confirmed = cest_candidates = 0
|
||||
mva_match = mva_resolved = mva_candidates = 0
|
||||
|
||||
for decision in decisions:
|
||||
ipi_status = decision.get("ipi_status")
|
||||
if ipi_status == "LEGACY_NCM":
|
||||
pass
|
||||
elif ipi_status == "NO_TIPI_MAPPING_FOUND":
|
||||
ipi_buckets["NO_MATCH"] += 1
|
||||
elif ipi_status == "IPI_CANDIDATES":
|
||||
ipi_buckets["AMBIGUOUS"] += 1
|
||||
elif ipi_status == "IPI_RESOLVED":
|
||||
rate_kind = (decision.get("classification", {}).get("ipi") or {}).get("rate_kind")
|
||||
ipi_buckets[{"NUMERIC": "NUMERIC", "NON_NUMERIC": "NON_NUMERIC", "BLANK": "SOURCE_BLANK"}.get(rate_kind, "NO_MATCH")] += 1
|
||||
else:
|
||||
ipi_buckets["NO_MATCH"] += 1
|
||||
|
||||
cest_status = decision.get("cest_status")
|
||||
if cest_status not in (None, "NO_CEST_MAPPING_FOUND", "LEGACY_NCM"):
|
||||
cest_match += 1
|
||||
if cest_status == "CEST_CONFIRMED":
|
||||
cest_confirmed += 1
|
||||
elif cest_status == "CEST_CANDIDATES":
|
||||
cest_candidates += 1
|
||||
|
||||
mva_status = decision.get("ba_st_mva_status")
|
||||
if mva_status not in (None, "NO_MVA_MAPPING_FOUND", "LEGACY_NCM"):
|
||||
mva_match += 1
|
||||
if mva_status == "MVA_RESOLVED":
|
||||
mva_resolved += 1
|
||||
elif mva_status == "MVA_CANDIDATES":
|
||||
mva_candidates += 1
|
||||
|
||||
coverage = {
|
||||
"artifact": "FISCAL_COVERAGE_FULL_V1",
|
||||
"firebird_writes": 0,
|
||||
"total_products": total,
|
||||
"ncm_valid": ncm_valid,
|
||||
"ncm_legacy": ncm_legacy,
|
||||
"ncm_invalid": ncm_invalid,
|
||||
"ncm_missing": ncm_missing,
|
||||
"ipi": {
|
||||
"numeric": ipi_buckets["NUMERIC"],
|
||||
"non_numeric": ipi_buckets["NON_NUMERIC"],
|
||||
"source_blank": ipi_buckets["SOURCE_BLANK"],
|
||||
"ambiguous": ipi_buckets["AMBIGUOUS"],
|
||||
"no_match": ipi_buckets["NO_MATCH"],
|
||||
},
|
||||
"cest": {"match": cest_match, "confirmed": cest_confirmed, "candidates": cest_candidates},
|
||||
"ba_st_mva": {"match": mva_match, "resolved": mva_resolved, "candidates": mva_candidates},
|
||||
"pis_cofins": {
|
||||
"data_available": True, # CST_PIS/CST_COFINS domain tables exist (fiscal_code_entries), no product-level linkage yet
|
||||
"match": 0,
|
||||
"unresolved": total,
|
||||
"note": "CST_PIS/CST_COFINS code domains are ACTIVE and consultable, but no NCM-linked determination table (SPED Tabela 4.3.10-4.3.17) is available to resolve a specific product's PIS/COFINS situation yet. DATA_AVAILABLE=true, RULE_RESOLVED=0 for all products.",
|
||||
},
|
||||
"cfop": {"dataset_available": False, "note": "CFOP full table FAILED to acquire this session (portal-gated); MOC7 only references CFOP, does not enumerate it. See docs/SOURCE_REGISTRY_V2.md."},
|
||||
"cst_icms": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 11},
|
||||
"csosn": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 10},
|
||||
"origin": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 9},
|
||||
"fcp": {"dataset_available": False, "note": "No single canonical national table exists; FAILED (see docs/SOURCE_REGISTRY_V2.md)."},
|
||||
"icms_monophasic": {"match": 0, "dataset_available": False},
|
||||
"anp": {"match": 0, "dataset_available": False},
|
||||
"ncm_utrib": {"dataset_available": False},
|
||||
"rtc": {
|
||||
"cclasstrib_match": 0, "ccredpres_match": 0, "cbs_rate_match": 0, "biofuel_index_available": False,
|
||||
"note": "All RTC (Reforma Tributária) tables FAILED to acquire — 2025/26-era portal documents, not present in MOC7 or locally.",
|
||||
},
|
||||
"active_dataset_kinds": sorted(active_kinds),
|
||||
}
|
||||
|
||||
output_path = ROOT / "docs" / "FISCAL_COVERAGE_FULL_V1.json"
|
||||
output_path.write_text(json.dumps(coverage, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(json.dumps({k: v for k, v in coverage.items() if k not in ("pis_cofins", "rtc")}, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,167 @@
|
||||
"""Read-only fiscal coverage matrix (V2) over all C000025 products in
|
||||
TESTESOFICIAIS.FDB, after importing CFOP/FCP/ANP/NCM_UTRIB/RTC_BIOFUEL_INDEX/
|
||||
RTC_CCREDPRES from sources/raw/bootstrap_v1/.
|
||||
|
||||
No Firebird writes. Separates DATA_AVAILABLE (a dataset exists) from
|
||||
MATCH_FOUND (this product's own field — NCM or CFOP — actually matches an
|
||||
official row) from RULE_RESOLVED (a fiscal determination was made) — the
|
||||
mission's own required distinction. Never invents a rule to inflate
|
||||
RULE_RESOLVED.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
|
||||
from gti_fiscal_agent.firebird_reader import read_products # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def http_get(url, token):
|
||||
request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def classify_batch(hub_url, token, products, chunk_size=100):
|
||||
results = []
|
||||
for start in range(0, len(products), chunk_size):
|
||||
chunk = products[start:start + chunk_size]
|
||||
payload = {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]}
|
||||
request = urllib.request.Request(
|
||||
f"{hub_url}/v1/classify/batch",
|
||||
data=json.dumps(payload, ensure_ascii=True).encode(),
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
results.extend(json.load(response)["results"])
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
token = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("AGENT_DEV_TOKEN", "")
|
||||
hub_url = os.environ.get("GTI_HUB_URL", "http://localhost:8000")
|
||||
load_dotenv(ROOT / ".env")
|
||||
|
||||
products = read_products(os.environ)
|
||||
total = len(products)
|
||||
decisions = classify_batch(hub_url, token, products)
|
||||
datasets = http_get(f"{hub_url}/v1/datasets/status", token)
|
||||
active_kinds = {d["kind"] for d in datasets if d["status"] == "ACTIVE"}
|
||||
record_counts = {d["kind"]: d["record_count"] for d in datasets if d["status"] == "ACTIVE"}
|
||||
|
||||
ncm_status = Counter(d.get("status") for d in decisions)
|
||||
ncm_missing = sum(1 for p in products if not (p["CLASSIFICACAO_FISCAL"] or "").strip())
|
||||
ncm_valid = ncm_status.get("EXACT_NCM", 0)
|
||||
ncm_legacy = ncm_status.get("LEGACY_NCM", 0)
|
||||
ncm_invalid = sum(
|
||||
1 for p, d in zip(products, decisions)
|
||||
if (p["CLASSIFICACAO_FISCAL"] or "").strip() and d.get("status") not in ("EXACT_NCM", "LEGACY_NCM")
|
||||
)
|
||||
|
||||
ipi_buckets = Counter()
|
||||
cest_match = cest_confirmed = cest_candidates = 0
|
||||
mva_match = mva_resolved = mva_candidates = 0
|
||||
ncm_utrib_match = 0
|
||||
cfop_match = 0
|
||||
cfop_seen = {}
|
||||
ncm_utrib_seen = {}
|
||||
|
||||
for product, decision in zip(products, decisions):
|
||||
ipi_status = decision.get("ipi_status")
|
||||
if ipi_status == "LEGACY_NCM":
|
||||
pass
|
||||
elif ipi_status == "NO_TIPI_MAPPING_FOUND":
|
||||
ipi_buckets["NO_MATCH"] += 1
|
||||
elif ipi_status == "IPI_CANDIDATES":
|
||||
ipi_buckets["AMBIGUOUS"] += 1
|
||||
elif ipi_status == "IPI_RESOLVED":
|
||||
rate_kind = (decision.get("classification", {}).get("ipi") or {}).get("rate_kind")
|
||||
ipi_buckets[{"NUMERIC": "NUMERIC", "NON_NUMERIC": "NON_NUMERIC", "BLANK": "SOURCE_BLANK"}.get(rate_kind, "NO_MATCH")] += 1
|
||||
else:
|
||||
ipi_buckets["NO_MATCH"] += 1
|
||||
|
||||
cest_status = decision.get("cest_status")
|
||||
if cest_status not in (None, "NO_CEST_MAPPING_FOUND", "LEGACY_NCM"):
|
||||
cest_match += 1
|
||||
if cest_status == "CEST_CONFIRMED":
|
||||
cest_confirmed += 1
|
||||
elif cest_status == "CEST_CANDIDATES":
|
||||
cest_candidates += 1
|
||||
|
||||
mva_status = decision.get("ba_st_mva_status")
|
||||
if mva_status not in (None, "NO_MVA_MAPPING_FOUND", "LEGACY_NCM"):
|
||||
mva_match += 1
|
||||
if mva_status == "MVA_RESOLVED":
|
||||
mva_resolved += 1
|
||||
elif mva_status == "MVA_CANDIDATES":
|
||||
mva_candidates += 1
|
||||
|
||||
matched_ncm = decision.get("classification", {}).get("ncm")
|
||||
if matched_ncm:
|
||||
if matched_ncm not in ncm_utrib_seen:
|
||||
ncm_utrib_seen[matched_ncm] = http_get(f"{hub_url}/v1/reference/ncm_utrib/{matched_ncm}", token)["found"]
|
||||
if ncm_utrib_seen[matched_ncm]:
|
||||
ncm_utrib_match += 1
|
||||
|
||||
cfop_value = (product.get("CFOP") or "").strip()
|
||||
if cfop_value:
|
||||
if cfop_value not in cfop_seen:
|
||||
cfop_seen[cfop_value] = http_get(f"{hub_url}/v1/reference/cfop/{cfop_value}", token)["found"]
|
||||
if cfop_seen[cfop_value]:
|
||||
cfop_match += 1
|
||||
|
||||
coverage = {
|
||||
"artifact": "FISCAL_COVERAGE_FULL_V2",
|
||||
"firebird_writes": 0,
|
||||
"total_products": total,
|
||||
"ncm_valid": ncm_valid, "ncm_legacy": ncm_legacy, "ncm_invalid": ncm_invalid, "ncm_missing": ncm_missing,
|
||||
"ipi": {
|
||||
"numeric": ipi_buckets["NUMERIC"], "non_numeric": ipi_buckets["NON_NUMERIC"],
|
||||
"source_blank": ipi_buckets["SOURCE_BLANK"], "ambiguous": ipi_buckets["AMBIGUOUS"], "no_match": ipi_buckets["NO_MATCH"],
|
||||
},
|
||||
"cest": {"match": cest_match, "confirmed": cest_confirmed, "candidates": cest_candidates},
|
||||
"ba_st_mva": {"match": mva_match, "resolved": mva_resolved, "candidates": mva_candidates},
|
||||
"pis_cofins": {
|
||||
"data_available": True, "match_found": 0, "rule_resolved": 0, "rule_unresolved": total,
|
||||
"note": "CST_PIS/CST_COFINS code domain is ACTIVE (66 codes) but no NCM-linked SPED Tabela 4.3.10-4.3.17 determination table was available to acquire (MISSING_BOOTSTRAP_FILE) — no per-product PIS/COFINS rule can be resolved.",
|
||||
},
|
||||
"cfop": {"dataset_available": "CFOP" in active_kinds, "record_count": record_counts.get("CFOP", 0), "match_found": cfop_match, "note": "match = this product's own C000025.CFOP value exists in the official table"},
|
||||
"cst_icms": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 11},
|
||||
"csosn": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 10},
|
||||
"origin": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 9},
|
||||
"fcp": {"dataset_available": "FCP" in active_kinds, "record_count": record_counts.get("FCP", 0), "note": "per-UF rate table; not linked to a product-level UF field in C000025, so match_found is not meaningful per-product"},
|
||||
"icms_monophasic": {"match_found": 0, "dataset_available": False, "status": "MISSING_BOOTSTRAP_FILE"},
|
||||
"anp": {"dataset_available": "ANP" in active_kinds, "record_count": record_counts.get("ANP", 0), "match_found": 0, "note": "C000025 has no ANP/fuel product code field to join on; dataset exists but is not applicable to this ERP's product catalog"},
|
||||
"ncm_utrib": {"dataset_available": "NCM_UTRIB" in active_kinds, "record_count": record_counts.get("NCM_UTRIB", 0), "match_found": ncm_utrib_match},
|
||||
"rtc": {
|
||||
"cclasstrib_match": 0, "cclasstrib_available": False, "cclasstrib_status": "MISSING_BOOTSTRAP_FILE",
|
||||
"ccredpres_match": 0, "ccredpres_available": "RTC_CCREDPRES" in active_kinds, "ccredpres_record_count": record_counts.get("RTC_CCREDPRES", 0),
|
||||
"cbs_rate_match": 0, "cbs_rate_available": False, "cbs_rate_status": "MISSING_BOOTSTRAP_FILE",
|
||||
"biofuel_match": 0, "biofuel_available": "RTC_BIOFUEL_INDEX" in active_kinds, "biofuel_record_count": record_counts.get("RTC_BIOFUEL_INDEX", 0),
|
||||
"note": "cCredPres and biofuel index datasets are ACTIVE but have no join key present in C000025 (no cProdANP/cCredPres field on this ERP's grocery/restaurant catalog), so match_found=0 is a data-shape fact, not a gap.",
|
||||
},
|
||||
"active_dataset_kinds": sorted(active_kinds),
|
||||
}
|
||||
|
||||
output_path = ROOT / "docs" / "FISCAL_COVERAGE_FULL_V2.json"
|
||||
output_path.write_text(json.dumps(coverage, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(json.dumps({k: v for k, v in coverage.items() if k not in ("pis_cofins", "rtc")}, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Read-only fiscal coverage matrix (V3) over all C000025 products in
|
||||
TESTESOFICIAIS.FDB, after importing CFOP/FCP/ANP/NCM_UTRIB/RTC_BIOFUEL_INDEX/
|
||||
RTC_CCREDPRES from sources/raw/bootstrap_v1/.
|
||||
|
||||
No Firebird writes. Separates DATA_AVAILABLE (a dataset exists) from
|
||||
MATCH_FOUND (this product's own field — NCM or CFOP — actually matches an
|
||||
official row) from RULE_RESOLVED (a fiscal determination was made) — the
|
||||
mission's own required distinction. Never invents a rule to inflate
|
||||
RULE_RESOLVED.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
|
||||
from gti_fiscal_agent.firebird_reader import read_products # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def http_get(url, token):
|
||||
request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def classify_batch(hub_url, token, products, chunk_size=100):
|
||||
results = []
|
||||
for start in range(0, len(products), chunk_size):
|
||||
chunk = products[start:start + chunk_size]
|
||||
payload = {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]}
|
||||
request = urllib.request.Request(
|
||||
f"{hub_url}/v1/classify/batch",
|
||||
data=json.dumps(payload, ensure_ascii=True).encode(),
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
results.extend(json.load(response)["results"])
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
token = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("AGENT_DEV_TOKEN", "")
|
||||
hub_url = os.environ.get("GTI_HUB_URL", "http://localhost:8000")
|
||||
load_dotenv(ROOT / ".env")
|
||||
|
||||
products = read_products(os.environ)
|
||||
total = len(products)
|
||||
decisions = classify_batch(hub_url, token, products)
|
||||
datasets = http_get(f"{hub_url}/v1/datasets/status", token)
|
||||
active_kinds = {d["kind"] for d in datasets if d["status"] == "ACTIVE"}
|
||||
record_counts = {d["kind"]: d["record_count"] for d in datasets if d["status"] == "ACTIVE"}
|
||||
|
||||
ncm_status = Counter(d.get("status") for d in decisions)
|
||||
ncm_missing = sum(1 for p in products if not (p["CLASSIFICACAO_FISCAL"] or "").strip())
|
||||
ncm_valid = ncm_status.get("EXACT_NCM", 0)
|
||||
ncm_legacy = ncm_status.get("LEGACY_NCM", 0)
|
||||
ncm_invalid = sum(
|
||||
1 for p, d in zip(products, decisions)
|
||||
if (p["CLASSIFICACAO_FISCAL"] or "").strip() and d.get("status") not in ("EXACT_NCM", "LEGACY_NCM")
|
||||
)
|
||||
|
||||
ipi_buckets = Counter()
|
||||
cest_match = cest_confirmed = cest_candidates = 0
|
||||
mva_match = mva_resolved = mva_candidates = 0
|
||||
ncm_utrib_match = 0
|
||||
cfop_match = 0
|
||||
cfop_seen = {}
|
||||
ncm_utrib_seen = {}
|
||||
|
||||
for product, decision in zip(products, decisions):
|
||||
ipi_status = decision.get("ipi_status")
|
||||
if ipi_status == "LEGACY_NCM":
|
||||
pass
|
||||
elif ipi_status == "NO_TIPI_MAPPING_FOUND":
|
||||
ipi_buckets["NO_MATCH"] += 1
|
||||
elif ipi_status == "IPI_CANDIDATES":
|
||||
ipi_buckets["AMBIGUOUS"] += 1
|
||||
elif ipi_status == "IPI_RESOLVED":
|
||||
rate_kind = (decision.get("classification", {}).get("ipi") or {}).get("rate_kind")
|
||||
ipi_buckets[{"NUMERIC": "NUMERIC", "NON_NUMERIC": "NON_NUMERIC", "BLANK": "SOURCE_BLANK"}.get(rate_kind, "NO_MATCH")] += 1
|
||||
else:
|
||||
ipi_buckets["NO_MATCH"] += 1
|
||||
|
||||
cest_status = decision.get("cest_status")
|
||||
if cest_status not in (None, "NO_CEST_MAPPING_FOUND", "LEGACY_NCM"):
|
||||
cest_match += 1
|
||||
if cest_status == "CEST_CONFIRMED":
|
||||
cest_confirmed += 1
|
||||
elif cest_status == "CEST_CANDIDATES":
|
||||
cest_candidates += 1
|
||||
|
||||
mva_status = decision.get("ba_st_mva_status")
|
||||
if mva_status not in (None, "NO_MVA_MAPPING_FOUND", "LEGACY_NCM"):
|
||||
mva_match += 1
|
||||
if mva_status == "MVA_RESOLVED":
|
||||
mva_resolved += 1
|
||||
elif mva_status == "MVA_CANDIDATES":
|
||||
mva_candidates += 1
|
||||
|
||||
matched_ncm = decision.get("classification", {}).get("ncm")
|
||||
if matched_ncm:
|
||||
if matched_ncm not in ncm_utrib_seen:
|
||||
ncm_utrib_seen[matched_ncm] = http_get(f"{hub_url}/v1/reference/ncm_utrib/{matched_ncm}", token)["found"]
|
||||
if ncm_utrib_seen[matched_ncm]:
|
||||
ncm_utrib_match += 1
|
||||
|
||||
cfop_value = (product.get("CFOP") or "").strip()
|
||||
if cfop_value:
|
||||
if cfop_value not in cfop_seen:
|
||||
cfop_seen[cfop_value] = http_get(f"{hub_url}/v1/reference/cfop/{cfop_value}", token)["found"]
|
||||
if cfop_seen[cfop_value]:
|
||||
cfop_match += 1
|
||||
|
||||
cbs_2026 = http_get(f"{hub_url}/v1/reference/rtc_cbs_rate/2026", token)
|
||||
coverage = {
|
||||
"artifact": "FISCAL_COVERAGE_FULL_V3",
|
||||
"firebird_writes": 0,
|
||||
"total_products": total,
|
||||
"ncm_valid": ncm_valid, "ncm_legacy": ncm_legacy, "ncm_invalid": ncm_invalid, "ncm_missing": ncm_missing,
|
||||
"ipi": {
|
||||
"numeric": ipi_buckets["NUMERIC"], "non_numeric": ipi_buckets["NON_NUMERIC"],
|
||||
"source_blank": ipi_buckets["SOURCE_BLANK"], "ambiguous": ipi_buckets["AMBIGUOUS"], "no_match": ipi_buckets["NO_MATCH"],
|
||||
},
|
||||
"cest": {"match": cest_match, "confirmed": cest_confirmed, "candidates": cest_candidates},
|
||||
"ba_st_mva": {"match": mva_match, "resolved": mva_resolved, "candidates": mva_candidates},
|
||||
"pis_cofins_4_3_10_17": "DEFERRED_BY_BUSINESS_SCOPE",
|
||||
"pis_cofins": {
|
||||
"data_available": True, "match_found": 0, "rule_resolved": 0, "rule_unresolved": total,
|
||||
"note": "CST_PIS/CST_COFINS code domain is ACTIVE (66 codes) but no NCM-linked SPED Tabela 4.3.10-4.3.17 determination table was available to acquire (MISSING_BOOTSTRAP_FILE) — no per-product PIS/COFINS rule can be resolved.",
|
||||
},
|
||||
"cfop": {"dataset_available": "CFOP" in active_kinds, "record_count": record_counts.get("CFOP", 0), "match_found": cfop_match, "note": "match = this product's own C000025.CFOP value exists in the official table"},
|
||||
"cst_icms": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 11},
|
||||
"csosn": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 10},
|
||||
"origin": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 9},
|
||||
"fcp": {"dataset_available": "FCP" in active_kinds, "record_count": record_counts.get("FCP", 0), "note": "per-UF rate table; not linked to a product-level UF field in C000025, so match_found is not meaningful per-product"},
|
||||
"icms_monophasic": {"match_found": 0, "dataset_available": False, "status": "OUT_OF_SCOPE_MVP"},
|
||||
"anp": {"scope": "ACTIVE_REFERENCE_ONLY", "dataset_available": "ANP" in active_kinds, "record_count": record_counts.get("ANP", 0), "match_found": 0, "note": "C000025 has no ANP/fuel product code field to join on; dataset exists but is not applicable to this ERP's product catalog"},
|
||||
"ncm_utrib": {"dataset_available": "NCM_UTRIB" in active_kinds, "record_count": record_counts.get("NCM_UTRIB", 0), "match_found": ncm_utrib_match},
|
||||
"rtc": {
|
||||
"cclasstrib_available": "RTC_CCLASSTRIB" in active_kinds, "cclasstrib_record_count": record_counts.get("RTC_CCLASSTRIB", 0), "cclasstrib_match": 0,
|
||||
"cbs_rate_available": "RTC_CBS_RATES" in active_kinds, "cbs_rate_record_count": record_counts.get("RTC_CBS_RATES", 0),
|
||||
"cbs_rate_2026": cbs_2026,
|
||||
"ccredpres_available": "RTC_CCREDPRES" in active_kinds, "ccredpres_record_count": record_counts.get("RTC_CCREDPRES", 0),
|
||||
"biofuel_available": "RTC_BIOFUEL_INDEX" in active_kinds, "biofuel_scope": "ACTIVE_REFERENCE_ONLY",
|
||||
"note": "cClassTrib/cCredPres are DATA_AVAILABLE only: C000025 carries no cClassTrib/cCredPres field to join on, so match_found=0 and rule_resolved=0 are data-shape facts, not gaps. No rule was invented.",
|
||||
},
|
||||
"core_domains_active": {d: (k in active_kinds) for d, k in {"NCM": "NCM", "TIPI_IPI": "TIPI_IPI", "CEST": "CEST", "BA_ST_MVA": "BA_ST_MVA", "CFOP": "CFOP", "ORIGIN": "NFE_MOC_TABLES", "CST_ICMS": "NFE_MOC_TABLES", "CSOSN": "NFE_MOC_TABLES", "CST_PIS": "NFE_MOC_TABLES", "CST_COFINS": "NFE_MOC_TABLES", "FCP": "FCP", "RTC_CCLASSTRIB": "RTC_CCLASSTRIB", "RTC_CCREDPRES": "RTC_CCREDPRES", "RTC_CBS_RATES": "RTC_CBS_RATES"}.items()},
|
||||
"active_dataset_kinds": sorted(active_kinds),
|
||||
}
|
||||
|
||||
output_path = ROOT / "docs" / "FISCAL_COVERAGE_FULL_V3.json"
|
||||
output_path.write_text(json.dumps(coverage, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(json.dumps({k: v for k, v in coverage.items() if k not in ("pis_cofins", "rtc")}, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Compose docs/MULTI_DATABASE_FINDINGS_V1.md from the read-only audit + coverage artifacts. No database access."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
D = ROOT / "docs"
|
||||
audit = json.loads((D / "MULTI_DATABASE_SCHEMA_AUDIT_V1.json").read_text(encoding="utf-8"))
|
||||
cov = json.loads((D / "MULTI_DATABASE_FISCAL_COVERAGE_V1.json").read_text(encoding="utf-8"))
|
||||
compat = json.loads((D / "MULTI_DATABASE_FIELD_COMPATIBILITY_V1.json").read_text(encoding="utf-8"))
|
||||
raw = {p.stem: json.loads(p.read_text(encoding="utf-8")) for p in (D / "runs" / "multi_db").glob("*.json") if not p.stem.endswith("_coverage")}
|
||||
A = {a["file"]: a for a in audit["databases"]}
|
||||
C = {c["file"]: c for c in cov["databases"]}
|
||||
|
||||
|
||||
def seg(f):
|
||||
return A[f]["segment_inferred"]["segment"]
|
||||
|
||||
|
||||
def pct(a, b):
|
||||
return f"{100 * a / b:.1f}%" if b else "n/a"
|
||||
|
||||
|
||||
lines = ["# MULTI_DATABASE_FINDINGS_V1", "",
|
||||
"Read-only audit of the copies in `Banco/` (no Firebird writes, no new fiscal rules, no new datasets). Segment = inferred from table population and the company name stored in C000004, never from the file name.", "",
|
||||
"## Databases", "", "| file | segment (evidence) | products | company UF/CRT (C000004) | EMISSOR.EMITENTE_CRT | schema class |", "|---|---|---|---|---|---|"]
|
||||
for f, a in A.items():
|
||||
co = a["company"]
|
||||
crt = f"{co.get('uf')}/{co.get('crt')}" if co.get("status") == "OK" else f"UNAVAILABLE ({co.get('rows')} company rows)"
|
||||
lines.append(f"| {f} | {seg(f)} | {a['product_count']} | {crt} | {co.get('emissor_crt')} | {a['schema_class']} |")
|
||||
lines += ["", "Company-context findings: C000004.CRT disagrees with EMISSOR.EMITENTE_CRT in 3 of the 4 copies, OPTANTE_SIMPLES=S coexists with CRT=3 (UNIAO) and OPTANTE_SIMPLES=N with CRT=2 (MMMODAS); RINCAO's C000004 holds 3 company rows (fail-closed: context ambiguous). The Agent must not guess: it needs an explicit company selection and a decided CRT source of truth.", "",
|
||||
"## Fiscal coverage (Fiscal Engine V1, read-only)", "", "| metric | " + " | ".join(C) + " |", "|---|" + "---|" * len(C)]
|
||||
keys = ["TOTAL_PRODUCTS", "NCM_VALID", "NCM_LEGACY", "NCM_INVALID", "NCM_MISSING", "IPI_NUMERIC", "IPI_NON_NUMERIC", "IPI_SOURCE_BLANK", "IPI_AMBIGUOUS", "IPI_NO_MATCH", "IPI_ALREADY_CORRECT", "IPI_CHANGE_REQUIRED",
|
||||
"IPI_WRITE_ELIGIBLE", "CEST_CONFIRMED", "CEST_CANDIDATES", "CFOP_RESOLVED", "CFOP_CONTEXT_REQUIRED", "CSOSN_RESOLVED", "CSOSN_CANDIDATES", "CSOSN_CONTEXT_REQUIRED", "ST_CONFIRMED", "ST_CANDIDATE", "ST_UNKNOWN",
|
||||
"ST_NOT_APPLICABLE", "PRODUCT_ROLE_KNOWN", "PRODUCT_ROLE_UNKNOWN", "ST_WITHHOLDING_CONFIRMED", "ST_WITHHOLDING_NOT_CONFIRMED", "ST_WITHHOLDING_UNKNOWN"]
|
||||
for k in keys:
|
||||
lines.append(f"| {k} | " + " | ".join(str(C[f]["TOTAL_PRODUCTS"] if k == "TOTAL_PRODUCTS" else C[f]["metrics"].get(k, 0)) for f in C) + " |")
|
||||
lines += ["", "Guards: ST/MVA dataset is BA-only and CSOSN logic needs CRT=1; when the company context does not satisfy them the engine reports context-required instead of applying a rule (`guards_applied` in the coverage artifact).", "",
|
||||
"## Segment comparison", "", "| indicator | " + " | ".join(f"{f} ({seg(f)})" for f in C) + " |", "|---|" + "---|" * len(C)]
|
||||
q = lambda f, k: C[f]["quality"][k] # noqa: E731
|
||||
n = lambda f: C[f]["TOTAL_PRODUCTS"] # noqa: E731
|
||||
rows = [("NCM valid share", lambda f: pct(C[f]["metrics"].get("NCM_VALID", 0), n(f))), ("CEST filled", lambda f: pct(q(f, "cest_filled"), n(f))), ("CSOSN filled", lambda f: pct(q(f, "csosn_filled"), n(f))),
|
||||
("CFOP distinct values", lambda f: q(f, "cfop_distinct")), ("CFOP top", lambda f: q(f, "cfop_top")[:3]), ("SITUACAO_TRIBUTARIA", lambda f: q(f, "situacao_tributaria")),
|
||||
("ST_CANDIDATE+CONFIRMED (BA dataset)", lambda f: C[f]["metrics"].get("ST_CANDIDATE", 0) + C[f]["metrics"].get("ST_CONFIRMED", 0)), ("purchase items C000088", lambda f: q(f, "purchase_items_C000088")),
|
||||
("products with purchase evidence", lambda f: q(f, "products_with_purchase_evidence")), ("recipes / composed", lambda f: f"{q(f, 'recipes')} / {q(f, 'composed_products')}"),
|
||||
("role known share", lambda f: pct(C[f]["metrics"].get("PRODUCT_ROLE_KNOWN", 0), n(f))), ("IPI numeric share", lambda f: pct(C[f]["metrics"].get("IPI_NUMERIC", 0), n(f))), ("IPI already stored", lambda f: pct(q(f, "ipi_filled"), n(f)))]
|
||||
for name, fn in rows:
|
||||
lines.append(f"| {name} | " + " | ".join(str(fn(f)) for f in C) + " |")
|
||||
|
||||
# mechanical attribution of gaps
|
||||
def all_zero(k):
|
||||
return all(C[f]["metrics"].get(k, 0) == 0 for f in C)
|
||||
|
||||
|
||||
erp_wide = [k for k in ("CEST_CONFIRMED", "CSOSN_RESOLVED", "ST_CONFIRMED", "ST_WITHHOLDING_CONFIRMED", "IPI_WRITE_ELIGIBLE") if all_zero(k)]
|
||||
lines += ["", "## Where the gaps come from", "",
|
||||
f"- **ERP-wide (0 in every database):** {', '.join(erp_wide) or 'none'}. These are limits of the SGV data model / current rules, not of one client.",
|
||||
"- **Client/segment:** CEST filling varies from near-empty (apparel) to mostly filled (market/restaurant/construction) — segment-shaped (goods under ST agreements), but each segment has ONE database here, so client vs segment cannot be separated.",
|
||||
"- **Company context (client-specific and ERP-wide):** CRT/regime data is inconsistent inside the same ERP (C000004 vs EMISSOR vs OPTANTE_SIMPLES); it is a data-quality/config issue every client needs to settle.",
|
||||
"- **No auto-repair (oficina) database was provided** (`Banco/` holds construction, apparel, restaurant and market copies); auto-repair conclusions are NOT drawn.", "",
|
||||
"## GUI map validation (21 write targets + origin)", "",
|
||||
f"All {compat['write_targets_count']} write-target columns exist in all databases ({len(compat['write_targets_present_in_all'])} present everywhere); the only type variation is {json.dumps(compat['variations'], ensure_ascii=False)}. "
|
||||
"Value patterns repeat across databases (CST 3-digit origin+CST, CSOSN 102/500/101, SITUACAO_TRIBUTARIA T, IAT/IPPT/INCIDENCIA constants, PIS/COFINS columns almost empty), which supports the DFM-derived map without changing it.", ""]
|
||||
org = compat["fields"]["COD_ORIGEM_MERCADORIA"]
|
||||
lines += ["Origin (`COD_ORIGEM_MERCADORIA`): populated counts per database " + json.dumps(org["population_per_db"]) + "; observed values " + json.dumps({k: v for k, v in org["distribution_top_per_db"].items()}, ensure_ascii=False) +
|
||||
". Values stay inside the 0/1/2 domain, but so few rows are populated that this is weak correlation: the field stays **PROBABLE** until a UI diff on the radio group.", "",
|
||||
"## Architecture answer", "",
|
||||
"**CAN_USE_ONE_GENERIC_AGENT=true.** All four databases expose C000004 and C000025 with every write-target column; differences are optional tables, one column length (CEST VARCHAR(10) vs (7)) and company-context quality. Required capabilities: "
|
||||
"`supports_ncm`, `supports_ipi` (only IPI is written today), `supports_cest`, C000025+C000004 present, an unambiguous (or explicitly configured) company context, and the write executor's live column-length/type check. "
|
||||
"Detected-but-optional: `supports_csosn/cfop/cst/pis/cofins/origin/iat/incidence/ippt`, and evidence tables (C000088, C000100, RECEITA_INGREDIENTES, NFCE_ITENS) which only enrich validation. "
|
||||
"No per-client Agent is needed; per-client configuration is: company selector (C000004.CODIGO), CRT source of truth, and the authorized write path/fingerprint (pilot two-layer authorization).", ""]
|
||||
(D / "MULTI_DATABASE_FINDINGS_V1.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
print("ok", erp_wide)
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Build docs/SGV_PRODUCT_FISCAL_UI_MAP_V1.json from the SGV form definition (Delphi DFM inside SGV.exe) — READ-ONLY.
|
||||
|
||||
Evidence hierarchy per UI control:
|
||||
1. DFM binding: data-aware control on form Tfrmproduto whose DataSource is dsproduto2 (dataset qrproduto =
|
||||
'select * from c000025') and whose DataField names the column; the on-screen caption/position matches the label.
|
||||
2. Column type/domain compatibility (live schema + live values).
|
||||
3. Optional operator UI diff (docs/SGV_PRODUCT_FISCAL_UI_DIFF_SESSION_V1.json), which can only confirm/upgrade.
|
||||
Classes: CONFIRMED_STRONG | PROBABLE | AMBIGUOUS | NO_MATCH.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent.firebird_reader import _connect # noqa: E402
|
||||
|
||||
for line in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
DFM_JSON = ROOT / "docs" / "runs" / "sgv_product_form_bindings.json"
|
||||
SESSION = ROOT / "docs" / "SGV_PRODUCT_FISCAL_UI_DIFF_SESSION_V1.json"
|
||||
dfm = next(f for f in json.loads(DFM_JSON.read_text(encoding="utf-8")) if f["form_class"] == "Tfrmproduto")
|
||||
bound = {c["DataField"]: c for c in dfm["controls"] if c.get("DataField") and c.get("DataSource") == "dsproduto2"}
|
||||
by_label = {}
|
||||
if SESSION.exists():
|
||||
for r in json.loads(SESSION.read_text(encoding="utf-8")).get("results", []):
|
||||
by_label[r["ui_label"]] = r
|
||||
|
||||
# ui_label, screen area, column (None = no column), direction, semantic, expected control name, notes
|
||||
UI = [
|
||||
("NCM", "Aba Estoque > Posição do Estoque/Informações", "CLASSIFICACAO_FISCAL", "n/a", "NCM canônico (8 dígitos); C000025.NCM is not on the screen", "DBEdit14", "NCM lookup control; column NCM (C000025.NCM) has no control."),
|
||||
("CFOP (Nota Fiscal de Entrada)", "Config. Impostos > Nota Fiscal de Entrada", "CFOP_ENTRADA", "entrada", "CFOP padrão de entrada", "DBEdit72", "Column is empty (0/521) but the screen writes it."),
|
||||
("CST de ICMS (Nota Fiscal de Entrada)", "Config. Impostos > Nota Fiscal de Entrada", "CST_ENTRADA", "entrada", "CST ICMS de entrada", "DBEdit73", "Column is empty (0/521) but the screen writes it."),
|
||||
("CFOP (Nota Fiscal de Saída)", "Config. Impostos > Nota Fiscal de Saída GRUPO DE ICMS", "CFOP", "saída", "CFOP padrão de saída", "DbEdit71", ""),
|
||||
("CST de ICMS (Nota Fiscal de Saída)", "Config. Impostos > Nota Fiscal de Saída GRUPO DE ICMS", "CST", "saída", "CST ICMS 3 dígitos (origem+CST)", "dbedit13", "CRT=1 emission uses CSOSN."),
|
||||
("Aliq. ICMS", "Config. Impostos > Nota Fiscal de Saída GRUPO DE ICMS", "ALIQUOTA", "saída", "alíquota ICMS %", "ealiq", "Binding proven; emitted NFC-e values differ from this default (see earlier missions)."),
|
||||
("CEST", "Config. Impostos > Nota Fiscal de Saída GRUPO DE ICMS", "CEST", "n/a", "CEST (7 dígitos)", "DBEdit74", ""),
|
||||
("Seletor ao lado do CEST (T/I/N/S)", "Config. Impostos > Nota Fiscal de Saída GRUPO DE ICMS", "SITUACAO_TRIBUTARIA", "saída", "situação tributária do produto: T=Tributado, I=Isento, N=Não Tributado, S=Substituição Tributária", "wwDBComboBox2", "Hint of the control lists the domain; 'S' is the ERP's own ST flag."),
|
||||
("Cofins Entrada", "Config. Impostos > CST PIS/COFINS", "COD_ENTRADA_COFINS", "entrada", "CST COFINS de entrada", "RzDBButtonEdit3", ""),
|
||||
("Cofins Saída", "Config. Impostos > CST PIS/COFINS", "COD_SAIDA_COFINS", "saída", "CST COFINS de saída", "RzDBButtonEdit1", ""),
|
||||
("Alíquota (Cofins)", "Config. Impostos > CST PIS/COFINS", "ALIQ_COFINS", "saída", "alíquota COFINS (texto)", "RzDBEdit4", ""),
|
||||
("Pis Entrada", "Config. Impostos > CST PIS/COFINS", "COD_ENTRADA_PIS", "entrada", "CST PIS de entrada", "RzDBButtonEdit5", ""),
|
||||
("Pis Saída", "Config. Impostos > CST PIS/COFINS", "COD_SAIDA_PIS", "saída", "CST PIS de saída", "RzDBButtonEdit2", ""),
|
||||
("Alíquota (Pis)", "Config. Impostos > CST PIS/COFINS", "ALIQ_PIS", "saída", "alíquota PIS (texto)", "RzDBEdit3", ""),
|
||||
("CSOSN Classificação", "Config. Impostos > CSOSN", "CSOSN", "saída", "CSOSN (código)", "DBEdit75", ""),
|
||||
("CSOSN Informações - Significados", "Config. Impostos > CSOSN", None, "n/a", "descrição do CSOSN (apoio visual)", "ComboBox1", "Plain TComboBox, not data-aware: presentation helper, no column."),
|
||||
("Alíquota IPI %", "Config. Impostos > IPI", "IPI", "saída", "alíquota IPI %", "RzDBEdit2", "Cálculo Preço tab has other IPI/PIS/COFINS controls bound to C000026 (price calculation), not fiscal defaults."),
|
||||
("CST IPI", "Config. Impostos > IPI", "CST_IPI", "saída", "CST do IPI", "RzDBEdit1", ""),
|
||||
("Código da origem da Mercadoria (0/1/2)", "Config. Impostos", "COD_ORIGEM_MERCADORIA", "n/a", "origem da mercadoria (0 Nacional, 1 Estrangeira-Importação Direta, 2 Estrangeira-Mercado Interno)", "CodOrigemMercadoria",
|
||||
"Radio group RgOrigemMercadoria is not data-aware; a hidden data-aware edit bound to COD_ORIGEM_MERCADORIA sits beside it. The radio->edit copy lives in event code."),
|
||||
("IAT", "Config. Impostos", "IAT", "n/a", "arredondamento/truncamento (A/T)", "wwDBComboBox3", ""),
|
||||
("Incidência", "Config. Impostos", "INCIDENCIA_PISCOFINS", "n/a", "tipo de incidência PIS/COFINS", "wwDBComboBox1", ""),
|
||||
("PIS/COFINS (combo)", "Config. Impostos", "PISCOFINS", "n/a", "flag PIS/COFINS (N/S)", "ecombo_piscofins", ""),
|
||||
("IPPT", "Config. Impostos", "IPPT", "n/a", "produção própria/terceiros (P/T)", "wwDBComboBox4", ""),
|
||||
]
|
||||
|
||||
cur = _connect(os.environ).cursor()
|
||||
cur.execute("select trim(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, coalesce(f.RDB$CHARACTER_LENGTH,f.RDB$FIELD_LENGTH), f.RDB$FIELD_SCALE from RDB$RELATION_FIELDS rf join RDB$FIELDS f on f.RDB$FIELD_NAME=rf.RDB$FIELD_SOURCE where rf.RDB$RELATION_NAME='C000025'")
|
||||
TN = {7: "SMALLINT", 8: "INTEGER", 14: "CHAR", 37: "VARCHAR", 27: "DOUBLE PRECISION", 16: "NUMERIC", 10: "FLOAT", 35: "TIMESTAMP", 12: "DATE"}
|
||||
meta = {}
|
||||
for n, t, ln, sc in cur.fetchall():
|
||||
meta[n] = f"{TN.get(t, t)}({ln})" if t in (14, 37) else (f"NUMERIC(*,{-sc})" if t == 16 and sc else TN.get(t, str(t)))
|
||||
|
||||
|
||||
def stats(col):
|
||||
cur.execute(f'select "{col}", count(*) from C000025 group by "{col}" order by 2 desc')
|
||||
vals = [(None if v is None else str(v).strip(), n) for v, n in cur.fetchall()]
|
||||
return sum(n for v, n in vals if v not in (None, "")), [v for v, _ in vals[:5]]
|
||||
|
||||
|
||||
entries = []
|
||||
for label, area, col, direction, sem, ctrl, notes in UI:
|
||||
diff = by_label.get(label)
|
||||
if col is None:
|
||||
entries.append({"ui_label": label, "screen_area": area, "table": "C000025", "field": None, "db_type": None, "direction": direction, "semantic": sem, "before": None, "after": None,
|
||||
"confirmed_by_ui_diff": False, "classification": "NO_MATCH", "confidence": "HIGH", "binding_type": "NOT_DATA_AWARE", "evidence_basis": f"DFM: {ctrl} is not data-aware",
|
||||
"write_target": False, "current_population": None, "sample_values": [], "notes": notes})
|
||||
continue
|
||||
c = bound.get(col)
|
||||
dfm_ok = bool(c) and c["name"] == ctrl and col in meta
|
||||
pop, samples = stats(col) if col in meta else (None, [])
|
||||
if not dfm_ok:
|
||||
klass, binding, basis = "NO_MATCH", "NO_DFM_BINDING", f"expected control {ctrl} bound to {col} not found in DFM"
|
||||
elif ctrl == "CodOrigemMercadoria":
|
||||
klass, binding, basis = "CONFIRMED_STRONG", "DFM_RADIO_ONCLICK_HIDDEN_EDIT", "DFM: RgOrigemMercadoria (TRadioGroup, no DataField, OnClick=RgOrigemMercadoriaClick) + hidden data-aware TDBEdit CodOrigemMercadoria bound to COD_ORIGEM_MERCADORIA in the same tab, the only origin-bound control of the form; the handler and both symbols exist in the executable; values 0/1/2 observed in 5 databases"
|
||||
else:
|
||||
klass, binding = "CONFIRMED_STRONG", "DFM_DATAFIELD"
|
||||
basis = f"DFM: {c['class']} {c['name']} DataField={col} DataSource=dsproduto2 (qrproduto = select * from c000025)"
|
||||
ui_diff = bool(diff and diff.get("binding_type") == "DIRECT_BINDING" and col in diff.get("db_fields_changed", []))
|
||||
if ui_diff:
|
||||
klass, binding = "CONFIRMED_STRONG", "DFM_DATAFIELD+UI_DIFF"
|
||||
entries.append({"ui_label": label, "screen_area": area, "table": "C000025", "field": col, "db_type": meta.get(col), "direction": direction, "semantic": sem, "before": None, "after": None,
|
||||
"confirmed_by_ui_diff": ui_diff, "classification": klass, "confidence": "HIGH" if klass == "CONFIRMED_STRONG" else "MEDIUM", "binding_type": binding, "evidence_basis": basis,
|
||||
"dfm_control": {"class": c["class"], "name": c["name"], "left": c.get("Left"), "top": c.get("Top"), "container_path": [p.split(":")[1] for p in c["path"][-3:]]} if c else None,
|
||||
"write_target": klass == "CONFIRMED_STRONG", "current_population": pop, "sample_values": samples, "notes": notes})
|
||||
cur.connection.close()
|
||||
|
||||
not_on_screen = [f for f in ("ORIGEM", "GRUPO_ICMS", "MARGEM_AGREGADA", "CFOP_CUPOM", "NCM") if f not in bound]
|
||||
cnt = lambda k: sum(1 for e in entries if e["classification"] == k) # noqa: E731
|
||||
doc = {"artifact": "SGV_PRODUCT_FISCAL_UI_MAP_V1", "product_table": "C000025", "company_table": "C000004", "company_fields": {"CRT": "C000004.CRT", "UF": "C000004.UF"},
|
||||
"evidence_source": "Delphi DFM of form Tfrmproduto extracted from SGV_5.0.6.124.exe (docs/runs/sgv_product_form_bindings.json); dsproduto2 -> qrproduto = 'select * from c000025'",
|
||||
"architecture_rule": "GTI Fiscal writes ONLY the canonical product record (C000025). Transactional/historical tables are evidence only.",
|
||||
"transactional_table_writes_allowed": False,
|
||||
"forbidden_write_tables": ["NFCE", "NFCE_ITENS", "C000032", "C000061", "C000062", "C000088", "INVENTARIO", "C000026 (price calculation)", "any historical/document/item table"],
|
||||
"columns_with_no_screen_control": [f for f in not_on_screen],
|
||||
"summary": {"ui_fields_total": len(entries), "CONFIRMED_STRONG": cnt("CONFIRMED_STRONG"), "PROBABLE": cnt("PROBABLE"), "AMBIGUOUS": cnt("AMBIGUOUS"), "NO_MATCH": cnt("NO_MATCH"),
|
||||
"write_target_fields": [e["field"] for e in entries if e["write_target"]],
|
||||
"fields_requiring_manual_ui_diff": [e["ui_label"] for e in entries if e["classification"] in ("PROBABLE", "AMBIGUOUS")],
|
||||
"unidentified_controls": ["Panel1.ealiquota -> CODALIQUOTA (data-aware, not visible in the supplied screenshots; column empty 0/521)"]},
|
||||
"fields": entries}
|
||||
(ROOT / "docs" / "SGV_PRODUCT_FISCAL_UI_MAP_V1.json").write_text(json.dumps(doc, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
print(json.dumps(doc["summary"], ensure_ascii=False))
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Import operator fiscal context (CSV or JSON) into the Hub — the answers to docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V1.csv.
|
||||
|
||||
python scripts/import_fiscal_context.py <file.csv|file.json> --token <AGENT_TOKEN> [--hub http://localhost:8000] [--dry-run]
|
||||
|
||||
CSV columns (the questionnaire CSV works as-is): company_key, scope, scope_key, key, value, reason (rows with an empty `value` are skipped).
|
||||
JSON: {"company_key": "...", "entries": [{"scope","scope_key","key","value","reason"}]} or a list of rows that carry company_key.
|
||||
Entries are validated locally (same rules as the Hub) and sent per company; the Hub applies each batch all-or-nothing.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from central.app.context_keys import validate_entry # noqa: E402
|
||||
|
||||
|
||||
def load_rows(path):
|
||||
p = Path(path)
|
||||
if p.suffix.lower() == ".json":
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
if isinstance(data, dict):
|
||||
return [{"company_key": data.get("company_key"), **e} for e in data.get("entries", [])]
|
||||
return data
|
||||
with p.open(newline="", encoding="utf-8-sig") as f:
|
||||
return list(csv.DictReader(f))
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("file"); ap.add_argument("--token", required=True); ap.add_argument("--hub", default="http://localhost:8000"); ap.add_argument("--dry-run", action="store_true")
|
||||
a = ap.parse_args(argv)
|
||||
by_company, errors, skipped = defaultdict(list), [], 0
|
||||
for i, row in enumerate(load_rows(a.file), start=1):
|
||||
if not str(row.get("value") or "").strip():
|
||||
skipped += 1
|
||||
continue
|
||||
entry, err = validate_entry(row)
|
||||
if err:
|
||||
errors.append((i, err))
|
||||
else:
|
||||
by_company[str(row.get("company_key") or "").strip()].append(entry)
|
||||
if errors:
|
||||
for i, e in errors[:30]:
|
||||
print(f"row {i}: {e}")
|
||||
raise SystemExit(f"{len(errors)} invalid row(s); nothing was sent")
|
||||
total = 0
|
||||
for company, entries in by_company.items():
|
||||
if not company:
|
||||
raise SystemExit("company_key is required on every row")
|
||||
total += len(entries)
|
||||
if a.dry_run:
|
||||
continue
|
||||
req = urllib.request.Request(f"{a.hub}/v1/context/import", data=json.dumps({"company_key": company, "entries": entries}).encode(), method="POST",
|
||||
headers={"Authorization": f"Bearer {a.token}", "Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
print(company, json.load(r))
|
||||
except urllib.error.HTTPError as e:
|
||||
raise SystemExit(f"Hub refused {company}: {e.read().decode()[:300]}")
|
||||
print(f"{'validated' if a.dry_run else 'imported'} {total} entries ({skipped} unanswered rows skipped)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Read-only: per-column stats for a table. usage: inspect_erp_table.py TABLE [regex-of-cols]"""
|
||||
import os, re, sys
|
||||
from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent.firebird_reader import _connect
|
||||
for l in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
||||
if "=" in l and not l.startswith("#"):
|
||||
k, _, v = l.partition("="); os.environ.setdefault(k.strip(), v.strip())
|
||||
t = sys.argv[1]; rx = re.compile(sys.argv[2], re.I) if len(sys.argv) > 2 else None
|
||||
conn = _connect(os.environ); cur = conn.cursor()
|
||||
cur.execute("select trim(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, coalesce(f.RDB$CHARACTER_LENGTH,f.RDB$FIELD_LENGTH) from RDB$RELATION_FIELDS rf join RDB$FIELDS f on f.RDB$FIELD_NAME=rf.RDB$FIELD_SOURCE where rf.RDB$RELATION_NAME=? order by rf.RDB$FIELD_POSITION", (t,))
|
||||
for name, ty, ln in cur.fetchall():
|
||||
if rx and not rx.search(name): continue
|
||||
if ty in (261,): # blob
|
||||
print(f"{name} BLOB"); continue
|
||||
cur.execute(f'select "{name}", count(*) from "{t}" group by "{name}" order by 2 desc rows 1 to 400')
|
||||
rows = cur.fetchall()
|
||||
nn = sum(c for v, c in rows if v is not None)
|
||||
top = [(str(v).strip()[:24] if v is not None else None, c) for v, c in rows[:5]]
|
||||
print(f"{name} t{ty}/{ln} nn={nn} distinct{'>=' if len(rows)>=400 else '='}{len(rows)} {top}")
|
||||
conn.close()
|
||||
@@ -0,0 +1,397 @@
|
||||
"""GTI Fiscal LOCAL COMPLETION V1 runner: snapshot -> dry-run -> guarded apply -> verify -> second pass -> reports/questions.
|
||||
|
||||
python scripts/local_completion.py <stage> [--db NAME ...]
|
||||
stages: snapshot | dryrun | apply | verify | second | report | questions | all
|
||||
|
||||
Databases are ONLY those in docs/LOCAL_WRITE_ALLOWLIST_V1.json (exact canonical paths). Writes go through
|
||||
scripts/firebird_write_executor.execute_guarded_product_update (path allowlist, schema/type/length guard, compare-before-write,
|
||||
one UPDATE per product, rowcount, independent re-read). Nothing is written by any stage except `apply` (and `second`, which must write 0).
|
||||
State lives in docs/runs/local_completion/ (git-ignored bulk files).
|
||||
"""
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
import fdb # noqa: E402
|
||||
import run_fiscal_decision_v1 as base # noqa: E402
|
||||
from central.app.field_contract import ALREADY, CHANGE, CONTRACT, CONTRACT_VERSION, CORRECTED, MANUAL, NA, WRITE_FIELDS, Context, resolve_product # noqa: E402
|
||||
from gti_fiscal_agent import company_context as cc # noqa: E402
|
||||
from gti_fiscal_agent.coverage import build_refs # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import read_evidence # noqa: E402
|
||||
|
||||
STATE = ROOT / "docs" / "runs" / "local_completion"
|
||||
HUB = "http://localhost:8000"
|
||||
# Per-database company configuration (operator decisions recorded in docs/AGENT_COMPANY_CONTEXT_RULES_V1.md): the three databases with a CRT conflict use the
|
||||
# emission authority EMISSOR; the company is always the C000004 row 000001 (the one whose CNPJ equals the EMISSOR CNPJ).
|
||||
COMPANY_CFG = {"CONSTRUCAO": {"company_code": "000001"}, "MMMODAS": {"company_code": "000001", "crt_source": "EMISSOR"}, "RINCAO": {"company_code": "000001", "crt_source": "EMISSOR"},
|
||||
"UNIAO": {"company_code": "000001", "crt_source": "EMISSOR"}, "TESTESOFICIAIS": {}}
|
||||
BASE_COLS = ["CODIGO", "PRODUTO", "CODGRUPO", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO"]
|
||||
s_ = lambda v: None if v is None else str(v).strip() # noqa: E731
|
||||
|
||||
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", ROOT / "scripts" / "firebird_write_executor.py")
|
||||
mod = importlib.util.module_from_spec(spec); sys.modules[spec.name] = mod; spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def allowlist():
|
||||
doc = json.loads((ROOT / "docs" / "LOCAL_WRITE_ALLOWLIST_V1.json").read_text(encoding="utf-8"))
|
||||
return {d["name"]: d["path"] for d in doc["databases"]}
|
||||
|
||||
|
||||
def connect(path, rw=False):
|
||||
kw = dict(database=path, user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"))
|
||||
if not rw:
|
||||
kw["isolation_level"] = fdb.ISOLATION_LEVEL_READ_COMMITED_RO
|
||||
return fdb.connect(**kw)
|
||||
|
||||
|
||||
def env_for(path):
|
||||
return {"FIREBIRD_DATABASE_PATH": path, "FB_USER": os.environ["FB_USER"], "FB_PASSWORD": os.environ["FB_PASSWORD"], "FIREBIRD_CLIENT_LIBRARY": os.environ.get("FIREBIRD_CLIENT_LIBRARY")}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------------------------ snapshot
|
||||
def snapshot(name, path):
|
||||
conn = connect(path); cur = conn.cursor()
|
||||
cur.execute("SELECT TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE WHERE rf.RDB$RELATION_NAME = 'C000025' ORDER BY rf.RDB$FIELD_POSITION")
|
||||
cols = [c for c, t in cur.fetchall() if t != 261] # blobs (images) excluded
|
||||
cur.execute(f"SELECT {', '.join(cols)} FROM C000025 ORDER BY CODIGO")
|
||||
rows = {s_(r[cols.index("CODIGO")]): tuple(r) for r in cur.fetchall()}
|
||||
side = {}
|
||||
for t in ("C000004", "EMISSOR"):
|
||||
try:
|
||||
cur.execute(f"SELECT * FROM {t}")
|
||||
names = [d[0] for d in cur.description]
|
||||
data = [tuple(v.read() if hasattr(v, "read") else v for v in r) for r in cur.fetchall()]
|
||||
side[t] = {"columns": names, "rows": data, "sha256": hashlib.sha256(repr(data).encode("utf-8", "replace")).hexdigest()}
|
||||
except Exception as e: # noqa: BLE001
|
||||
side[t] = {"error": type(e).__name__}
|
||||
company = cc.resolve(cc.collect_evidence(cur), cc.config_from_env({k.upper(): v for k, v in _cfg_env(name).items()}))
|
||||
conn.close()
|
||||
snap = {"name": name, "path": path, "taken": datetime.now(timezone.utc).isoformat(timespec="seconds"), "columns": cols, "rows": rows, "side_tables": side,
|
||||
"product_count": len(rows), "products_logical_sha256": hashlib.sha256(repr(sorted(rows.items())).encode("utf-8", "replace")).hexdigest(),
|
||||
"company": {k: company.get(k) for k in ("status", "classification", "company_code", "company_cnpj", "company_uf", "company_crt", "source_of_truth", "confidence", "warnings", "reasons")}}
|
||||
STATE.mkdir(parents=True, exist_ok=True)
|
||||
with gzip.open(STATE / f"{name}_before.pkl.gz", "wb") as f:
|
||||
pickle.dump(snap, f)
|
||||
return snap
|
||||
|
||||
|
||||
def _cfg_env(name):
|
||||
cfg = COMPANY_CFG.get(name, {})
|
||||
out = {}
|
||||
if cfg.get("company_code"): out["COMPANY_CODE"] = cfg["company_code"]
|
||||
if cfg.get("crt_source"): out["COMPANY_CRT_SOURCE"] = cfg["crt_source"]
|
||||
return out
|
||||
|
||||
|
||||
def load_snapshot(name):
|
||||
with gzip.open(STATE / f"{name}_before.pkl.gz", "rb") as f:
|
||||
return pickle.load(f)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------------------------ analysis
|
||||
def classify_pairs(pairs):
|
||||
pairs = sorted(set(pairs))
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fin, fout = Path(tmp) / "in.json", Path(tmp) / "out.json"
|
||||
fin.write_text(json.dumps(pairs, ensure_ascii=False), encoding="utf-8")
|
||||
subprocess.run(["docker", "cp", str(ROOT / "scripts" / "bulk_classify_in_container.py"), "projeto-fiscal-hub-1:/tmp/bulk.py"], check=True)
|
||||
subprocess.run(["docker", "cp", str(fin), "projeto-fiscal-hub-1:/tmp/in.json"], check=True)
|
||||
subprocess.run(["docker", "exec", "projeto-fiscal-hub-1", "sh", "-c", "cd /app && python /tmp/bulk.py /tmp/in.json /tmp/out.json"], check=True, capture_output=True)
|
||||
subprocess.run(["docker", "cp", "projeto-fiscal-hub-1:/tmp/out.json", str(fout)], check=True)
|
||||
return dict(zip(pairs, json.loads(fout.read_text(encoding="utf-8"))))
|
||||
|
||||
|
||||
def hub_get(path, token):
|
||||
return base.http(HUB + path, token)
|
||||
|
||||
|
||||
def analyze(name, path, token, snap=None, extra_context=None):
|
||||
"""Read the database (read-only) and resolve every product/field. Returns the full analysis."""
|
||||
conn = connect(path); cur = conn.cursor()
|
||||
cur.execute("SELECT TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME = 'C000025'")
|
||||
have = {r[0] for r in cur.fetchall()}
|
||||
cols = [c for c in BASE_COLS + WRITE_FIELDS if c in have]
|
||||
cur.execute(f"SELECT {', '.join(cols)} FROM C000025 ORDER BY CODIGO")
|
||||
products = []
|
||||
for r in cur.fetchall():
|
||||
p = {c: None for c in BASE_COLS + WRITE_FIELDS}
|
||||
p.update(zip(cols, r))
|
||||
for k in p:
|
||||
if isinstance(p[k], str):
|
||||
p[k] = p[k].strip()
|
||||
products.append(p)
|
||||
company = cc.resolve(cc.collect_evidence(cur), cc.config_from_env(_cfg_env(name)))
|
||||
ev_rows = cc.collect_evidence(cur)
|
||||
sel = next((c for c in ev_rows["c000004"] if c["code"] == company["company_code"]), {})
|
||||
groups = {}
|
||||
if "C000017" in {r[0] for r in _tables(cur)}:
|
||||
cur.execute("SELECT TRIM(CODIGO), TRIM(GRUPO) FROM C000017")
|
||||
groups = dict(cur.fetchall())
|
||||
contrib = None
|
||||
if "CONTRIBUINTE_IPI" in _cols(cur, "C000004"):
|
||||
cur.execute("SELECT CODIGO, CONTRIBUINTE_IPI FROM C000004")
|
||||
contrib = {s_(a): s_(b) for a, b in cur.fetchall()}.get(company["company_code"])
|
||||
conn.close()
|
||||
if company["status"] != "PASS":
|
||||
raise RuntimeError(f"{name}: company context {company['classification']}: {company['reasons']}")
|
||||
comp = {"uf": company["company_uf"], "crt": company["company_crt"], "contribuinte_ipi": contrib, "company_code": company["company_code"], "company_key": company["company_cnpj_fingerprint"]}
|
||||
# ERP-emission evidence: the SGV NFC-e history shows whether the PIS/COFINS cadastral fields are consumed at emission (issued CST_PIS_COFINS non-blank share)
|
||||
try:
|
||||
_c = connect(path); _cur = _c.cursor()
|
||||
_cur.execute("SELECT COUNT(*), COUNT(NULLIF(TRIM(CST_PIS_COFINS), '')) FROM NFCE_ITENS")
|
||||
tot, nonblank = _cur.fetchone(); _c.close()
|
||||
comp["pis_cofins_emission_items"] = tot; comp["pis_cofins_emission_nonblank"] = nonblank
|
||||
comp["pis_cofins_unused_by_emission"] = bool(tot >= 1000 and nonblank / tot < 0.001)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
comp["pis_cofins_unused_by_emission"] = False; comp["pis_cofins_evidence_error"] = str(exc)[:120]
|
||||
evidence = read_evidence(env_for(path), company["company_code"])
|
||||
refs = build_refs(lambda p: hub_get(p, token), products, comp["uf"])
|
||||
entries = list(extra_context or [])
|
||||
try:
|
||||
entries += hub_get(f"/v1/context?company_key={comp['company_key']}", token)["entries"]
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
ctx = Context(entries)
|
||||
manual = {}
|
||||
for p in products:
|
||||
v, _ = ctx.get("ncm", s_(p["CODIGO"]))
|
||||
if v:
|
||||
manual[s_(p["CODIGO"])] = v
|
||||
pairs = [(s_(p["CLASSIFICACAO_FISCAL"]) or "", s_(p["PRODUTO"]) or "") for p in products] + [(v, s_(p["PRODUTO"]) or "") for p in products for v in [manual.get(s_(p["CODIGO"]))] if v]
|
||||
dec = classify_pairs(pairs)
|
||||
results = {}
|
||||
dup_codes = {c: n for c, n in Counter(s_(p["CODIGO"]) for p in products).items() if n > 1}
|
||||
seen = Counter()
|
||||
for p in products:
|
||||
code = s_(p["CODIGO"]); desc = s_(p["PRODUTO"]) or ""
|
||||
seen[code] += 1
|
||||
rk = code if seen[code] == 1 else f"{code}#{seen[code]}"
|
||||
p["_rk"] = rk
|
||||
m = manual.get(code)
|
||||
md = dec[(m, desc)] if m else None
|
||||
valid_manual = bool(md and md.get("status") == "EXACT_NCM")
|
||||
d = md if valid_manual else dec[(s_(p["CLASSIFICACAO_FISCAL"]) or "", desc)]
|
||||
results[rk] = resolve_product(p, d, comp, refs, evidence, ctx, manual_ncm=(m, valid_manual) if m else None)
|
||||
if dup_codes.get(code, 1) > 1: # data-integrity defect (PK violation in the copy): every physical row that shares the code is blocked, all 23 cells stay accounted
|
||||
for f in WRITE_FIELDS:
|
||||
r = results[rk][f]
|
||||
r.update(state=MANUAL, reason="DUPLICATE_KEY_BLOCKED", input_key="duplicate_code", input_scope="PRODUCT", write_eligible=False, proposed=None)
|
||||
results[rk]["_ds"] = {k: (v or "")[:8] for k, v in (d.get("dataset_versions") or {}).items() if k in ("ncm", "tipi_ipi", "cest", "ba_st_mva", "cfop", "nfe_moc_tables")}
|
||||
return {"name": name, "path": path, "products": products, "company": comp, "company_result": company, "results": results, "groups": groups, "context_entries": len(entries), "engine": {"contract": CONTRACT_VERSION}}
|
||||
|
||||
|
||||
def _tables(cur):
|
||||
cur.execute("SELECT TRIM(RDB$RELATION_NAME) FROM RDB$RELATIONS WHERE COALESCE(RDB$SYSTEM_FLAG,0)=0")
|
||||
return cur.fetchall()
|
||||
|
||||
|
||||
def _cols(cur, table):
|
||||
cur.execute("SELECT TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME = ?", (table,))
|
||||
return {r[0] for r in cur.fetchall()}
|
||||
|
||||
|
||||
def summarize(an):
|
||||
per = {f: Counter() for f in WRITE_FIELDS}
|
||||
reasons = {f: Counter() for f in WRITE_FIELDS}
|
||||
manual_keys = Counter()
|
||||
for code, res in an["results"].items():
|
||||
for f in WRITE_FIELDS:
|
||||
r = res[f]
|
||||
per[f][r["state"]] += 1
|
||||
reasons[f][(r["state"], r["reason"])] += 1
|
||||
if r["state"] == MANUAL:
|
||||
manual_keys[(f, r["input_key"])] += 1
|
||||
roles = Counter(res["_facts"]["role"] for res in an["results"].values())
|
||||
return {"name": an["name"], "total_products": len(an["products"]), "company": an["company"], "context_entries": an["context_entries"],
|
||||
"fields": {f: dict(per[f]) for f in WRITE_FIELDS}, "reasons": {f: {f"{s}|{r}": n for (s, r), n in reasons[f].most_common(12)} for f in WRITE_FIELDS},
|
||||
"manual_inputs": {f"{f}|{k}": n for (f, k), n in manual_keys.most_common()}, "roles": dict(roles),
|
||||
"change_required_cells": sum(per[f][CHANGE] for f in WRITE_FIELDS), "write_eligible_cells": sum(1 for res in an["results"].values() for f in WRITE_FIELDS if res[f]["write_eligible"])}
|
||||
|
||||
|
||||
def write_ndjson(name, an, suffix):
|
||||
with gzip.open(STATE / f"{name}_{suffix}.ndjson.gz", "wt", encoding="utf-8") as f:
|
||||
for p in an["products"]:
|
||||
code = s_(p["CODIGO"]); res = an["results"][p["_rk"]]
|
||||
for fld in WRITE_FIELDS:
|
||||
r = res[fld]
|
||||
f.write(json.dumps({"db": an["name"], "codigo": code, "row_key": p["_rk"], "field": fld, "before": r["current"], "proposed": r["proposed"], "status": r["state"], "reason": r["reason"], "input_key": r["input_key"],
|
||||
"provenance": f"{CONTRACT_VERSION}|" + ",".join(f"{k}:{v}" for k, v in res["_ds"].items()) + f"|role={res['_facts']['role']}", "write_eligible": r["write_eligible"]}, ensure_ascii=False, default=str) + "\n")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------------------------ apply / verify
|
||||
def apply(name, path, an, snap):
|
||||
ex = load_executor()
|
||||
ex.assert_local_completion_target(path)
|
||||
now_company = cc.resolve(cc.collect_evidence(_ro_cursor(path)), cc.config_from_env(_cfg_env(name)))
|
||||
if now_company["status"] != "PASS" or (now_company["company_uf"], now_company["company_crt"]) != (an["company"]["uf"], an["company"]["crt"]):
|
||||
raise RuntimeError("company-context guard failed before writing")
|
||||
cols = snap["columns"]
|
||||
conn = connect(path, rw=True)
|
||||
audit, failed, attempted, succeeded = [], [], 0, 0
|
||||
try:
|
||||
for p in an["products"]:
|
||||
code = s_(p["CODIGO"]); res = an["results"][code]
|
||||
changes = {}
|
||||
for f in WRITE_FIELDS:
|
||||
r = res[f]
|
||||
if r["write_eligible"]:
|
||||
changes[f] = (snap["rows"][code][cols.index(f)], r["proposed"])
|
||||
if not changes:
|
||||
continue
|
||||
attempted += len(changes)
|
||||
try:
|
||||
rec = ex.execute_guarded_product_update(conn, database_path=path, product_code=code, changes=changes, now=datetime.now(timezone.utc).isoformat(timespec="seconds"))
|
||||
audit.append({"db": name, **rec, "outcome": "SUCCEEDED"})
|
||||
succeeded += len(changes)
|
||||
except ex.WriteRefused as e:
|
||||
failed.append({"db": name, "product_code": code, "fields": list(changes), "error": str(e)[:200]})
|
||||
finally:
|
||||
conn.close()
|
||||
with gzip.open(STATE / f"{name}_write_audit.ndjson.gz", "wt", encoding="utf-8") as f:
|
||||
for a in audit + [{**x, "outcome": "FAILED"} for x in failed]:
|
||||
f.write(json.dumps(a, ensure_ascii=False, default=str) + "\n")
|
||||
return {"name": name, "cells_attempted": attempted, "cells_succeeded": succeeded, "cells_failed": sum(len(x["fields"]) for x in failed), "products_updated": len(audit), "failures": failed[:20]}
|
||||
|
||||
|
||||
def _ro_cursor(path):
|
||||
return connect(path).cursor()
|
||||
|
||||
|
||||
def verify(name, path, an, snap):
|
||||
conn = connect(path); cur = conn.cursor()
|
||||
cols = snap["columns"]
|
||||
cur.execute(f"SELECT {', '.join(cols)} FROM C000025 ORDER BY CODIGO")
|
||||
after = {s_(r[cols.index("CODIGO")]): tuple(r) for r in cur.fetchall()}
|
||||
conn.close()
|
||||
norm = lambda v: None if v is None else (round(float(v), 6) if isinstance(v, (int, float)) or v.__class__.__name__ == "Decimal" else str(v).strip()) # noqa: E731
|
||||
planned = verified = bad_planned = unplanned_changes = 0
|
||||
problems = []
|
||||
for p in an["products"]:
|
||||
code = s_(p["CODIGO"]); before = snap["rows"][code]; aft = after.get(code)
|
||||
if aft is None:
|
||||
problems.append((code, "MISSING_AFTER")); continue
|
||||
exp = {f: an["results"][code][f]["proposed"] for f in WRITE_FIELDS if an["results"][code][f]["write_eligible"]}
|
||||
for i, c in enumerate(cols):
|
||||
if c in exp:
|
||||
planned += 1
|
||||
if norm(aft[i]) == norm(exp[c]):
|
||||
verified += 1
|
||||
else:
|
||||
bad_planned += 1; problems.append((code, c, "AFTER!=PROPOSED", aft[i], exp[c]))
|
||||
elif norm(aft[i]) != norm(before[i]):
|
||||
unplanned_changes += 1; problems.append((code, c, "UNPLANNED_CHANGE", before[i], aft[i]))
|
||||
return {"name": name, "products_after": len(after), "products_before": len(snap["rows"]), "planned_cells": planned, "post_write_verified": verified, "planned_mismatch": bad_planned,
|
||||
"unplanned_changes": unplanned_changes, "problems": problems[:20]}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------------------------ questions
|
||||
def questions(analyses):
|
||||
groups = defaultdict(lambda: {"products": [], "fields": Counter(), "choices": None, "examples": []})
|
||||
for an in analyses:
|
||||
by_code = {s_(p["CODIGO"]): p for p in an["products"]}
|
||||
for code, res in an["results"].items():
|
||||
p = by_code[code]
|
||||
for f in WRITE_FIELDS:
|
||||
r = res[f]
|
||||
if r["state"] != MANUAL:
|
||||
continue
|
||||
key, scope = r["input_key"], r["input_scope"]
|
||||
if key in ("ncm",):
|
||||
gk = (an["name"], key, "PRODUCT", code, "")
|
||||
elif key in ("cest", "ex_tipi"):
|
||||
ncm = "".join(c for c in (p["CLASSIFICACAO_FISCAL"] or "") if c.isdigit())
|
||||
gk = (an["name"], key, "NCM", ncm, "")
|
||||
elif scope == "COMPANY" or key in ("entry_scope", "rounding", "icms_rate", "credit_rate", "pis_cofins_profile", "crt"):
|
||||
gk = (an["name"], key, "COMPANY", "", "")
|
||||
else:
|
||||
grp = s_(p["CODGRUPO"]) or "(sem grupo)"
|
||||
gk = (an["name"], key, "GROUP", grp, an["groups"].get(grp, ""))
|
||||
g = groups[gk]
|
||||
g["products"].append(code); g["fields"][f] += 1
|
||||
if r.get("choices") and g["choices"] is None:
|
||||
g["choices"] = r["choices"]
|
||||
if len(g["examples"]) < 3:
|
||||
g["examples"].append(f"{code} {s_(p['PRODUTO'])[:40]}")
|
||||
from central.app.context_keys import ENUMS
|
||||
out = []
|
||||
for (db, key, scope, skey, gname), g in sorted(groups.items()):
|
||||
allowed = sorted(ENUMS[key]) if key in ENUMS else ("JSON {pis_in_cst,pis_out_cst,pis_rate,cofins_in_cst,cofins_out_cst,cofins_rate}" if key == "pis_cofins_profile" else "free text (see key)")
|
||||
out.append({"database": db, "company_key": next(a["company"]["company_key"] for a in analyses if a["name"] == db), "key": key, "scope": scope, "scope_key": skey, "group_name": gname,
|
||||
"products_affected": len(set(g["products"])), "fields_affected": dict(g["fields"]), "allowed_values": allowed, "candidate_choices": g["choices"], "examples": g["examples"], "answer": ""})
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("stage"); ap.add_argument("--db", nargs="*")
|
||||
a = ap.parse_args()
|
||||
base.load_dotenv(ROOT / ".env")
|
||||
token = os.environ["AGENT_DEV_TOKEN"]
|
||||
allow = allowlist()
|
||||
names = a.db or list(allow)
|
||||
for n in names:
|
||||
if n not in allow:
|
||||
raise SystemExit(f"{n} is not in the exact write allowlist")
|
||||
STATE.mkdir(parents=True, exist_ok=True)
|
||||
analyses = []
|
||||
for name in names:
|
||||
path = allow[name]
|
||||
print(f"== {name} :: {a.stage}", flush=True)
|
||||
if a.stage in ("snapshot", "all"):
|
||||
snap = snapshot(name, path)
|
||||
print(f" snapshot: {snap['product_count']} products, company {snap['company'].get('classification')}", flush=True)
|
||||
if a.stage in ("dryrun", "apply", "verify", "second", "report", "questions", "all"):
|
||||
snap = load_snapshot(name)
|
||||
if a.stage in ("dryrun", "all", "apply"):
|
||||
an = analyze(name, path, token, snap)
|
||||
(STATE / f"{name}_dryrun_summary.json").write_text(json.dumps(summarize(an), ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
write_ndjson(name, an, "dryrun")
|
||||
print(" dry-run:", json.dumps({k: summarize(an)[k] for k in ("total_products", "change_required_cells", "write_eligible_cells")}), flush=True)
|
||||
if a.stage in ("apply", "all"):
|
||||
res = apply(name, path, an, snap)
|
||||
(STATE / f"{name}_apply_summary.json").write_text(json.dumps(res, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
print(" apply:", json.dumps({k: res[k] for k in ("cells_attempted", "cells_succeeded", "cells_failed", "products_updated")}), flush=True)
|
||||
v = verify(name, path, an, snap)
|
||||
(STATE / f"{name}_verify_summary.json").write_text(json.dumps(v, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
print(" verify:", json.dumps({k: v[k] for k in ("planned_cells", "post_write_verified", "planned_mismatch", "unplanned_changes")}), flush=True)
|
||||
if a.stage in ("second", "all"):
|
||||
an2 = analyze(name, path, token, snap)
|
||||
s2 = summarize(an2)
|
||||
second = {"name": name, "SECOND_PASS_CHANGE_REQUIRED": s2["change_required_cells"], "SECOND_PASS_WRITES": 0, "fields": s2["fields"]}
|
||||
if s2["write_eligible_cells"]:
|
||||
second["SECOND_PASS_WRITES"] = apply(name, path, an2, snap)["cells_succeeded"]
|
||||
(STATE / f"{name}_second_pass.json").write_text(json.dumps(second, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
write_ndjson(name, an2, "second")
|
||||
print(" second pass:", second["SECOND_PASS_CHANGE_REQUIRED"], second["SECOND_PASS_WRITES"], flush=True)
|
||||
if a.stage in ("questions",):
|
||||
analyses.append(analyze(name, path, token, snap))
|
||||
if a.stage == "questions":
|
||||
q = questions(analyses)
|
||||
(ROOT / "docs" / "LOCAL_FISCAL_CONTEXT_QUESTIONS_V1.json").write_text(json.dumps({"artifact": "LOCAL_FISCAL_CONTEXT_QUESTIONS_V1", "total_questions": len(q), "questions": q}, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
import csv
|
||||
with open(ROOT / "docs" / "LOCAL_FISCAL_CONTEXT_QUESTIONS_V1.csv", "w", newline="", encoding="utf-8-sig") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["database", "company_key", "scope", "scope_key", "key", "group_name", "products_affected", "fields_affected", "allowed_values", "examples", "value", "reason"])
|
||||
for x in q:
|
||||
w.writerow([x["database"], x["company_key"], x["scope"], x["scope_key"], x["key"], x["group_name"], x["products_affected"], json.dumps(x["fields_affected"], ensure_ascii=False),
|
||||
x["allowed_values"] if isinstance(x["allowed_values"], str) else "|".join(x["allowed_values"]), " ; ".join(x["examples"]), "", ""])
|
||||
print("questions:", len(q))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Compose the LOCAL COMPLETION V1 artifacts from docs/runs/local_completion/* (no database access):
|
||||
docs/LOCAL_FULL_FISCAL_DRY_RUN_V1.json, docs/LOCAL_FULL_FISCAL_WRITE_AUDIT_V1.json, docs/LOCAL_DATABASE_VALIDATION_REPORT_V1.md, docs/local_completion_totals.json
|
||||
"""
|
||||
import gzip
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from central.app.field_contract import WRITE_FIELDS # noqa: E402
|
||||
|
||||
ST = ROOT / "docs" / "runs" / "local_completion"
|
||||
NAMES = [d["name"] for d in json.loads((ROOT / "docs" / "LOCAL_WRITE_ALLOWLIST_V1.json").read_text(encoding="utf-8"))["databases"]]
|
||||
load = lambda n, suffix: json.loads((ST / f"{n}_{suffix}.json").read_text(encoding="utf-8")) if (ST / f"{n}_{suffix}.json").exists() else None # noqa: E731
|
||||
|
||||
dry, audit_doc, per_db = {}, {}, {}
|
||||
totals = Counter()
|
||||
report = ["# LOCAL_DATABASE_VALIDATION_REPORT_V1", "",
|
||||
"Local completion mission over the five authorized disposable copies. States per (product, field): ALREADY_CORRECT (incl. `DECLARED_VALID_NOT_CONTRADICTED`), CORRECTED (written and independently re-read), NOT_APPLICABLE, MANUAL_INPUT_REQUIRED(<input>), FAILED (write refused/failed). No cell is left in a generic state.", ""]
|
||||
for n in NAMES:
|
||||
d, ap, vf, sp = load(n, "dryrun_summary"), load(n, "apply_summary"), load(n, "verify_summary"), load(n, "second_pass")
|
||||
if not d:
|
||||
continue
|
||||
failed_by_field = Counter()
|
||||
ok_by_field = Counter()
|
||||
p = ST / f"{n}_write_audit.ndjson.gz"
|
||||
if p.exists():
|
||||
with gzip.open(p, "rt", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
a = json.loads(line)
|
||||
if a.get("outcome") == "SUCCEEDED":
|
||||
for fld in a["fields"]:
|
||||
ok_by_field[fld] += 1
|
||||
else:
|
||||
for fld in a.get("fields", []):
|
||||
failed_by_field[fld] += 1
|
||||
rows = {}
|
||||
for f in WRITE_FIELDS:
|
||||
c = d["fields"][f]
|
||||
corrected = ok_by_field.get(f, 0)
|
||||
failed = failed_by_field.get(f, 0)
|
||||
rows[f] = {"ALREADY_CORRECT": c.get("ALREADY_CORRECT", 0), "CORRECTED": corrected, "NOT_APPLICABLE": c.get("NOT_APPLICABLE", 0), "MANUAL_INPUT_REQUIRED": c.get("MANUAL_INPUT_REQUIRED", 0), "FAILED": failed,
|
||||
"CHANGE_REQUIRED_planned": c.get("CHANGE_REQUIRED", 0)}
|
||||
per_db[n] = {"total_products": d["total_products"], "fields": rows, "company": d["company"], "manual_inputs": d["manual_inputs"], "roles": d["roles"], "apply": ap, "verify": vf, "second_pass": sp}
|
||||
for f in WRITE_FIELDS:
|
||||
for k in ("ALREADY_CORRECT", "CORRECTED", "NOT_APPLICABLE", "MANUAL_INPUT_REQUIRED", "FAILED", "CHANGE_REQUIRED_planned"):
|
||||
totals[k] += rows[f][k]
|
||||
totals[f"{f}:{k}"] += rows[f][k]
|
||||
dry[n] = {"total_products": d["total_products"], "company": d["company"], "states_before_writes": d["fields"], "top_reasons": d["reasons"], "manual_inputs": d["manual_inputs"], "roles": d["roles"],
|
||||
"detail_file": f"docs/runs/local_completion/{n}_dryrun.ndjson.gz (git-ignored; one JSON line per product/field: BEFORE, PROPOSED, STATUS, REASON, PROVENANCE, WRITE_ELIGIBLE)"}
|
||||
audit_doc[n] = {"path": "exact allowlist path", "apply": ap, "verify": vf, "second_pass": {k: sp[k] for k in ("SECOND_PASS_CHANGE_REQUIRED", "SECOND_PASS_WRITES")} if sp else None,
|
||||
"cells_by_field": {f: {"attempted": rows[f]["CORRECTED"] + rows[f]["FAILED"], "succeeded": rows[f]["CORRECTED"], "failed": rows[f]["FAILED"]} for f in WRITE_FIELDS if rows[f]["CORRECTED"] + rows[f]["FAILED"]},
|
||||
"detail_file": f"docs/runs/local_completion/{n}_write_audit.ndjson.gz"}
|
||||
report += [f"## {n}", f"Produtos: {d['total_products']} | empresa: UF {d['company']['uf']} / CRT {d['company']['crt']} (código {d['company']['company_code']})", ""]
|
||||
if ap:
|
||||
report.append(f"Escrita: {ap['cells_succeeded']} células corrigidas em {ap['products_updated']} produtos ({ap['cells_failed']} falharam); verificação: {vf['post_write_verified']}/{vf['planned_cells']} conferem, {vf['unplanned_changes']} mudanças não planejadas. Segunda passagem: CHANGE_REQUIRED={sp['SECOND_PASS_CHANGE_REQUIRED']}, writes={sp['SECOND_PASS_WRITES']}." if sp else f"Escrita: {ap['cells_succeeded']} células corrigidas; verificação {vf['post_write_verified']}/{vf['planned_cells']}.")
|
||||
report += ["", "| campo | ALREADY_CORRECT | CORRECTED | NOT_APPLICABLE | MANUAL_INPUT_REQUIRED | FAILED |", "|---|---|---|---|---|---|"]
|
||||
for f in WRITE_FIELDS:
|
||||
r = rows[f]
|
||||
report.append(f"| {f} | {r['ALREADY_CORRECT']} | {r['CORRECTED']} | {r['NOT_APPLICABLE']} | {r['MANUAL_INPUT_REQUIRED']} | {r['FAILED']} |")
|
||||
report += ["", f"Células fiscais corrigidas: **{sum(rows[f]['CORRECTED'] for f in WRITE_FIELDS)}**. Entradas manuais pedidas (campo|chave): " + ", ".join(f"{k}={v}" for k, v in list(d["manual_inputs"].items())[:8]) + " ...", ""]
|
||||
|
||||
(ROOT / "docs" / "LOCAL_FULL_FISCAL_DRY_RUN_V1.json").write_text(json.dumps({"artifact": "LOCAL_FULL_FISCAL_DRY_RUN_V1", "contract": "field-contract-v1", "databases": dry}, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
(ROOT / "docs" / "LOCAL_FULL_FISCAL_WRITE_AUDIT_V1.json").write_text(json.dumps({"artifact": "LOCAL_FULL_FISCAL_WRITE_AUDIT_V1", "databases": audit_doc}, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
(ROOT / "docs" / "LOCAL_DATABASE_VALIDATION_REPORT_V1.md").write_text("\n".join(report) + "\n", encoding="utf-8")
|
||||
(ROOT / "docs" / "runs" / "local_completion" / "totals.json").write_text(json.dumps({"totals": dict(totals), "per_db": per_db}, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
print(json.dumps({k: v for k, v in totals.items() if ":" not in k}))
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Assisted single-session mapping of the SGV "Cadastro de Produto" fiscal screen (READ-ONLY tool).
|
||||
|
||||
python scripts/map_product_ui_fields.py --product <CODIGO>
|
||||
|
||||
For each UI field the tool shows CURRENT and SUGGESTED_TEMP, waits for ENTER while the operator changes ONLY that
|
||||
field in the SGV and saves, re-reads the whole C000025 row, diffs it against the previous baseline and records
|
||||
which column(s) changed. The tool never writes to Firebird and never touches the GUI. State is persisted after every
|
||||
step (docs/SGV_PRODUCT_FISCAL_UI_DIFF_SESSION_V1.json) so the session can be resumed.
|
||||
|
||||
Binding classes: DIRECT_BINDING (one non-technical column changed) | DERIVED/COUPLED_FIELD (several changed;
|
||||
ALL recorded, none chosen) | AUTOMATIC_METADATA (only timestamp-like technical columns) | AMBIGUOUS (nothing changed
|
||||
or several changed without the expected one).
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src")); sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
SESSION = ROOT / "docs" / "SGV_PRODUCT_FISCAL_UI_DIFF_SESSION_V1.json"
|
||||
# Columns the ERP rewrites by itself on every save. Only these are ignored, and they are still listed in the record.
|
||||
TECHNICAL = {"ULTIMA_ALTERACAO", "DATA_ALTERACAO", "DATA_ULTIMA_ALTERACAO", "USUARIO_ALTERACAO"}
|
||||
|
||||
# ui_label, hypothesised column (never trusted), candidate temporary values (first one different from CURRENT wins)
|
||||
SEQUENCE = [
|
||||
("NCM", "CLASSIFICACAO_FISCAL", ["22030000", "19022000"]),
|
||||
("CFOP entrada", "CFOP_ENTRADA", ["1102", "1949"]),
|
||||
("CFOP saída", "CFOP", ["5405", "5102", "5949"]),
|
||||
("CST ICMS entrada", "CST_ENTRADA", ["040", "000", "060"]),
|
||||
("CST ICMS saída", "CST", ["041", "000", "060"]),
|
||||
("Alíquota ICMS", "ALIQUOTA", ["12", "18", "17"]),
|
||||
("CEST", "CEST", ["0601100", "1708400"]),
|
||||
("CSOSN", "CSOSN", ["103", "500", "102"]),
|
||||
("PIS entrada", "COD_ENTRADA_PIS", ["73", "98", "50"]),
|
||||
("PIS saída", "COD_SAIDA_PIS", ["04", "01", "49"]),
|
||||
("Alíquota PIS", "ALIQ_PIS", ["0,65", "1,65"]),
|
||||
("COFINS entrada", "COD_ENTRADA_COFINS", ["73", "98", "50"]),
|
||||
("COFINS saída", "COD_SAIDA_COFINS", ["04", "01", "49"]),
|
||||
("Alíquota COFINS", "ALIQ_COFINS", ["3", "7,6"]),
|
||||
("IPI", "IPI", ["5", "10", "15"]),
|
||||
("CST IPI", "CST_IPI", ["50", "00", "99"]),
|
||||
("EX TIPI", "EX_TIPI", ["1", "2"]),
|
||||
("Origem", "ORIGEM", ["1", "2", "0"]),
|
||||
("IAT", "IAT", ["T", "A"]),
|
||||
("Incidência PIS/COFINS", "INCIDENCIA_PISCOFINS", ["MONOFÁSICA", "SUBST.TRIBUTÁRIA", "NENHUMA"]),
|
||||
("IPPT", "IPPT", ["P", "T"]),
|
||||
]
|
||||
|
||||
|
||||
def _same(a, b):
|
||||
if a is None or b is None:
|
||||
return a is None and b is None
|
||||
try:
|
||||
return float(str(a).replace(",", ".")) == float(str(b).replace(",", "."))
|
||||
except ValueError:
|
||||
return str(a).strip().upper() == str(b).strip().upper()
|
||||
|
||||
|
||||
def suggest_temp(current, candidates):
|
||||
"""First candidate that differs from the current value (never the same value, never blank)."""
|
||||
for c in candidates:
|
||||
if not _same(current, c):
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def classify(changes, expected_column):
|
||||
"""changes: [{'FIELD','BEFORE','AFTER','technical'}]. Never picks a column when several changed."""
|
||||
real = [c["FIELD"] for c in changes if not c["technical"]]
|
||||
tech = [c["FIELD"] for c in changes if c["technical"]]
|
||||
if not real:
|
||||
return "AUTOMATIC_METADATA" if tech else "AMBIGUOUS", "no fiscal column changed" if not tech else "only technical timestamp columns changed"
|
||||
if len(real) == 1:
|
||||
return "DIRECT_BINDING", "exactly one column changed" + ("" if real[0] == expected_column else f" (differs from hypothesis {expected_column})")
|
||||
if expected_column in real:
|
||||
return "DERIVED/COUPLED_FIELD", f"{len(real)} columns changed together; none selected"
|
||||
return "AMBIGUOUS", f"{len(real)} columns changed and none is the hypothesis {expected_column}"
|
||||
|
||||
|
||||
def compute_diff(before, after):
|
||||
return [{"FIELD": f, "BEFORE": before.get(f), "AFTER": after.get(f), "technical": f in TECHNICAL}
|
||||
for f in sorted(set(before) | set(after)) if before.get(f) != after.get(f)]
|
||||
|
||||
|
||||
def new_session(product):
|
||||
return {"artifact": "SGV_PRODUCT_FISCAL_UI_DIFF_SESSION_V1", "database": "TESTESOFICIAIS.FDB", "product": product, "started": datetime.now().isoformat(timespec="seconds"),
|
||||
"initial_snapshot": None, "baseline": None, "results": [], "firebird_writes": 0}
|
||||
|
||||
|
||||
def save(session):
|
||||
SESSION.write_text(json.dumps(session, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
|
||||
|
||||
def load_env():
|
||||
for line in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def run_step(session, ui_label, column, candidates, read_row, ask):
|
||||
baseline = session["baseline"]
|
||||
current = baseline.get(column) if column else None
|
||||
temp = suggest_temp(current, candidates) if candidates else None
|
||||
print(f"\nUI_FIELD={ui_label}\nCURRENT={current}\nSUGGESTED_TEMP={temp}\nHYPOTHESIS_COLUMN={column} (not trusted)")
|
||||
answer = ask("Change ONLY this field in the SGV and save, then press ENTER [s=skip (not on screen), q=quit]: ").strip().lower()
|
||||
if answer == "q":
|
||||
return "quit"
|
||||
if answer == "s":
|
||||
session["results"].append({"ui_label": ui_label, "hypothesis_column": column, "status": "NOT_ON_SCREEN", "at": datetime.now().isoformat(timespec="seconds")})
|
||||
save(session)
|
||||
return "skipped"
|
||||
after = read_row()
|
||||
changes = compute_diff(baseline, after)
|
||||
binding, why = classify(changes, column)
|
||||
rec = {"ui_label": ui_label, "hypothesis_column": column, "current": current, "suggested_temp": temp, "status": "TESTED", "binding_type": binding, "reason": why,
|
||||
"db_fields_changed": [c["FIELD"] for c in changes if not c["technical"]], "technical_fields_changed": [c["FIELD"] for c in changes if c["technical"]],
|
||||
"changes": changes, "at": datetime.now().isoformat(timespec="seconds")}
|
||||
session["results"].append(rec)
|
||||
session["baseline"] = after # new state is the baseline for the next field
|
||||
save(session)
|
||||
print(f"UI_FIELD={ui_label}\nDB_FIELDS_CHANGED={rec['db_fields_changed']}\nBEFORE={ {c['FIELD']: c['BEFORE'] for c in changes} }\nAFTER={ {c['FIELD']: c['AFTER'] for c in changes} }\nBINDING={binding} ({why})")
|
||||
return "tested"
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--product", required=True)
|
||||
ap.add_argument("--restart", action="store_true", help="discard a previous session file for this product")
|
||||
a = ap.parse_args(argv)
|
||||
load_env()
|
||||
from sgv_product_snapshot import read_row # read-only single-row reader (read-only isolation)
|
||||
from gti_fiscal_agent.firebird_reader import parse_company_context, _connect
|
||||
path = os.environ.get("FIREBIRD_DATABASE_PATH", "")
|
||||
if Path(path).name.upper() != "TESTESOFICIAIS.FDB":
|
||||
raise SystemExit("this experiment is only for the disposable TESTESOFICIAIS.FDB (FIREBIRD_DATABASE_PATH)")
|
||||
reader = lambda: read_row(os.environ, a.product) # noqa: E731
|
||||
|
||||
if SESSION.exists() and not a.restart and json.loads(SESSION.read_text(encoding="utf-8")).get("product") == a.product:
|
||||
session = json.loads(SESSION.read_text(encoding="utf-8"))
|
||||
print(f"Resuming session ({len(session['results'])} results already recorded).")
|
||||
session["baseline"] = reader() # honour any manual change since the last step
|
||||
else:
|
||||
session = new_session(a.product)
|
||||
session["initial_snapshot"] = session["baseline"] = reader()
|
||||
save(session)
|
||||
conn = _connect(os.environ)
|
||||
try:
|
||||
cur = conn.cursor(); cur.execute("SELECT UF, CRT FROM C000004")
|
||||
session["company_context_read_from_db"] = parse_company_context(cur.fetchall())
|
||||
finally:
|
||||
conn.close()
|
||||
done = {r["ui_label"] for r in session["results"]}
|
||||
for ui_label, column, candidates in SEQUENCE:
|
||||
if ui_label in done:
|
||||
continue
|
||||
if run_step(session, ui_label, column, candidates, reader, input) == "quit":
|
||||
break
|
||||
else:
|
||||
while True:
|
||||
extra = input("\nAny OTHER fiscal field visible on the screen? Type its label (or ENTER to finish): ").strip()
|
||||
if not extra:
|
||||
break
|
||||
if run_step(session, extra, None, [], reader, input) == "quit":
|
||||
break
|
||||
session["finished"] = datetime.now().isoformat(timespec="seconds")
|
||||
save(session)
|
||||
changed = {}
|
||||
for r in session["results"]:
|
||||
for c in r.get("changes", []):
|
||||
if not c["technical"]:
|
||||
changed.setdefault(c["FIELD"], session["initial_snapshot"].get(c["FIELD"]))
|
||||
print("\nSession saved:", SESSION)
|
||||
if changed:
|
||||
print("RESTORE_ORIGINAL_VALUES (set these back in the SGV):")
|
||||
for f, v in changed.items():
|
||||
print(f" {f} = {v}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Manually set C000025.IPI back to NULL for one test product, to simulate ERP
|
||||
drift for the incremental auto-apply test. Guarded the same way as any other
|
||||
write here (path + allowlist + schema), executed once, by hand, outside the
|
||||
normal Agent flow — this is test-fixture setup, not something the Agent does.
|
||||
"""
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def load_dotenv(path):
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ[key.strip()] = value.strip()
|
||||
|
||||
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", ROOT / "scripts" / "firebird_write_executor.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def main():
|
||||
code = sys.argv[1] if len(sys.argv) > 1 else "000026"
|
||||
load_dotenv(ROOT / ".env")
|
||||
executor = load_executor()
|
||||
database_path = os.environ["FIREBIRD_DATABASE_PATH"]
|
||||
executor.assert_authorized_write_path(database_path)
|
||||
executor.assert_allowed_field("C000025", "IPI")
|
||||
|
||||
import fdb
|
||||
conn = fdb.connect(
|
||||
database=database_path,
|
||||
user=os.environ["FB_USER"],
|
||||
password=os.environ["FB_PASSWORD"],
|
||||
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
executor.assert_schema_ok(cur, "C000025", "IPI", None)
|
||||
cur.execute("SELECT IPI FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
row = cur.fetchone()
|
||||
before = row[0] if row else None
|
||||
cur.execute("UPDATE C000025 SET IPI = NULL WHERE CODIGO = ? AND IPI = ?", (code, before))
|
||||
rowcount = cur.rowcount
|
||||
if rowcount != 1:
|
||||
conn.rollback()
|
||||
raise SystemExit(f"expected rowcount 1, got {rowcount}")
|
||||
conn.commit()
|
||||
cur.execute("SELECT IPI FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
after = cur.fetchone()[0]
|
||||
print(json.dumps({"product_code": code, "before": before, "after": after, "rowcount": rowcount}, indent=2))
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Read-only operational profile of emitted documents (trusted corpus + document keys). No writes."""
|
||||
import json, os, sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent.firebird_reader import _connect
|
||||
for l in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
||||
if "=" in l and not l.startswith("#"):
|
||||
k, _, v = l.partition("="); os.environ.setdefault(k.strip(), v.strip())
|
||||
s = lambda v: None if v is None else str(v).strip()
|
||||
trusted = json.loads((ROOT / "docs" / "runs" / "trusted_historical_items.json").read_text(encoding="utf-8"))
|
||||
cur = _connect(os.environ).cursor()
|
||||
cur.execute("select CFOP, NATUREZA, TIPO from C000030"); nat = {}
|
||||
for cfop, n, t in cur.fetchall(): nat.setdefault(s(cfop), []).append((s(n), t))
|
||||
cur.execute("select CHAVE from NFCE where CHAVE is not null and trim(CHAVE) <> ''"); keys = [s(r[0]) for r in cur.fetchall()]
|
||||
model = Counter(k[20:22] for k in keys if len(k) == 44); uf = Counter(k[:2] for k in keys if len(k) == 44)
|
||||
cur.execute("select CFOP, count(*) from NFCE_ITENS group by CFOP"); all_cfop = {s(a): b for a, b in cur.fetchall()}
|
||||
cur.execute("select CFOP, CST, CSOSN, count(*) from C000062 group by CFOP, CST, CSOSN"); nf_rows = [(s(a), s(b), s(c), n) for a, b, c, n in cur.fetchall()]
|
||||
cf = Counter(t["cfop"] for t in trusted); cs = Counter(t["csosn"] for t in trusted); x = Counter((t["cfop"], t["csosn"], t["cst"]) for t in trusted)
|
||||
cest_by_cfop = Counter((t["cfop"], bool((t["cest"] or "").strip())) for t in trusted)
|
||||
def tipo(c):
|
||||
ts = {t for _, t in nat.get(c, [])}; return {"in_C000030": c in nat, "tipo_values": sorted(str(t) for t in ts), "meaning": {"1": "entrada", "2": "saída"}.get(str(next(iter(ts))) if len(ts) == 1 else "", "ambiguous/absent")}
|
||||
doc = {"artifact": "ERP_OPERATION_PROFILES_V1", "firebird_writes": 0, "trusted_items": len(trusted),
|
||||
"nfce_document_keys": {"analyzed": len(keys), "model_positions_20_22": dict(model), "uf_code_positions_0_2": dict(uf),
|
||||
"reading": "chave de acesso: cUF (29=BA) + modelo (65=NFC-e, official: consumidor final, operação interna) — proven for the keyed subset"},
|
||||
"trusted_cfop": dict(cf), "trusted_csosn": dict(cs), "trusted_cfop_csosn_cst": {"|".join(map(str, k)): v for k, v in x.most_common()},
|
||||
"trusted_cest_present_by_cfop": {f"{k[0]}|cest_present={k[1]}": v for k, v in cest_by_cfop.items()},
|
||||
"cfop_natureza_C000030": {c: tipo(c) for c in sorted(x for x in (set(cf) | set(all_cfop) | {r[0] for r in nf_rows}) if x)},
|
||||
"all_nfce_items_cfop": all_cfop, "C000062_cfop_cst_csosn": [{"cfop": a, "cst": b, "csosn": c, "n": n} for a, b, c, n in nf_rows],
|
||||
"context_stability": {
|
||||
"direction": {"value": "saída", "stable": True, "evidence": "100% of NFC-e item CFOPs start with 5/6/7 (saída); C000025.CFOP is a saída default", "available_to_agent": True},
|
||||
"company_uf": {"value": "BA", "stable": True, "evidence": "C000004.UF + chave cUF=29", "available_to_agent": True},
|
||||
"destination_uf_nfce": {"value": "BA", "stable": True, "evidence": "NFC-e (mod 65) is intra-state consumer sale", "available_to_agent": "derivable (document model), not per product"},
|
||||
"consumer_final": {"value": True, "stable": True, "evidence": "NFC-e model 65", "available_to_agent": "derivable (document model)"},
|
||||
"customer_taxpayer_status": {"stable": False, "evidence": "not present on NFC-e items; NF-e items (C000062) have no customer join in this analysis", "available_to_agent": False},
|
||||
"st_retention_by_supplier": {"stable": False, "evidence": "no structured field", "available_to_agent": False}},
|
||||
"note": "Frequency is descriptive only; no rule is derived from it."}
|
||||
(ROOT / "docs" / "ERP_OPERATION_PROFILES_V1.json").write_text(json.dumps(doc, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
print(json.dumps({"model": doc["nfce_document_keys"], "cfop": cf, "csosn": cs, "x": x.most_common(6)}, ensure_ascii=True))
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Zero-write proof for a Firebird database (READ-ONLY tool).
|
||||
|
||||
python scripts/prove_zero_write.py snapshot <FDB> <out.json> # before the run
|
||||
python scripts/prove_zero_write.py compare <before.json> <after.json>
|
||||
|
||||
A snapshot records: (1) database-level DML counters (MON$RECORD_STATS inserts/updates/deletes), (2) a SHA-256 over every row/column of
|
||||
C000025 (and C000004), (3) row counts of C000025/C000004/C000030/NFCE/NFCE_ITENS, (4) the file SHA-256 and size. `compare` reports whether
|
||||
the DML counters, table digests and counts are identical. Only SELECT statements are executed. The file SHA-256 is informational only: any attach
|
||||
(even an idle read-only one) touches header/page metadata, so it is not part of the verdict.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def load_env():
|
||||
f = ROOT / ".env"
|
||||
if f.exists():
|
||||
for line in f.read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def table_digest(cur, table):
|
||||
cur.execute(f"SELECT * FROM {table} ORDER BY 1")
|
||||
h = hashlib.sha256()
|
||||
n = 0
|
||||
for row in cur.fetchall():
|
||||
h.update(repr([v.read() if hasattr(v, "read") else v for v in row]).encode("utf-8", "replace"))
|
||||
n += 1
|
||||
return {"rows": n, "sha256": h.hexdigest()}
|
||||
|
||||
|
||||
def snapshot(path):
|
||||
import fdb
|
||||
conn = fdb.connect(database=path, user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
stats = None
|
||||
try:
|
||||
cur.execute("SELECT r.MON$RECORD_INSERTS, r.MON$RECORD_UPDATES, r.MON$RECORD_DELETES FROM MON$RECORD_STATS r JOIN MON$DATABASE d ON d.MON$STAT_ID = r.MON$STAT_ID")
|
||||
row = cur.fetchone()
|
||||
stats = {"inserts": row[0], "updates": row[1], "deletes": row[2]}
|
||||
except Exception as e: # noqa: BLE001
|
||||
stats = {"unavailable": type(e).__name__}
|
||||
digests = {t: table_digest(cur, t) for t in ("C000025", "C000004")}
|
||||
counts = {}
|
||||
for t in ("C000030", "NFCE", "NFCE_ITENS", "C000061", "C000032"):
|
||||
try:
|
||||
cur.execute(f"SELECT COUNT(*) FROM {t}")
|
||||
counts[t] = cur.fetchone()[0]
|
||||
except Exception: # noqa: BLE001
|
||||
counts[t] = None
|
||||
finally:
|
||||
conn.close()
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(8 * 1024 * 1024), b""):
|
||||
h.update(block)
|
||||
return {"database": Path(path).name, "dml_counters": stats, "table_digests": digests, "row_counts": counts, "file_sha256": h.hexdigest(), "file_size": Path(path).stat().st_size}
|
||||
|
||||
|
||||
def compare(before, after):
|
||||
res = {
|
||||
"dml_counters_identical": before["dml_counters"] == after["dml_counters"],
|
||||
"table_digests_identical": before["table_digests"] == after["table_digests"],
|
||||
"row_counts_identical": before["row_counts"] == after["row_counts"],
|
||||
"file_sha256_identical": before["file_sha256"] == after["file_sha256"],
|
||||
"dml_counters": {"before": before["dml_counters"], "after": after["dml_counters"]},
|
||||
}
|
||||
res["ZERO_WRITE_PROVEN"] = res["table_digests_identical"] and res["row_counts_identical"] and res["dml_counters_identical"]
|
||||
return res
|
||||
|
||||
|
||||
def main(argv):
|
||||
load_env()
|
||||
if argv[0] == "snapshot":
|
||||
Path(argv[2]).write_text(json.dumps(snapshot(argv[1]), indent=1), encoding="utf-8")
|
||||
print("snapshot written")
|
||||
elif argv[0] == "compare":
|
||||
r = compare(json.loads(Path(argv[1]).read_text(encoding="utf-8")), json.loads(Path(argv[2]).read_text(encoding="utf-8")))
|
||||
print(json.dumps(r, indent=1))
|
||||
raise SystemExit(0 if r["ZERO_WRITE_PROVEN"] else 1)
|
||||
else:
|
||||
raise SystemExit(__doc__)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Read-only NCM historical reconciliation over NFCE_ITENS vs C000025.
|
||||
|
||||
Explains every NFC-e item whose emitted NCM differs from the product's current NCM and builds the
|
||||
TRUSTED_HISTORICAL_ITEMS corpus (identity-proven items outside systemic bad-join windows).
|
||||
Outputs: docs/NCM_HISTORICAL_RECONCILIATION_V1.json, docs/runs/trusted_historical_items.json
|
||||
"""
|
||||
import json, os, sys, urllib.request
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent.firebird_reader import _connect # noqa: E402
|
||||
|
||||
for l in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
||||
if "=" in l and not l.startswith("#"):
|
||||
k, _, v = l.partition("="); os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
WINDOW_RATE = 0.5 # month is a systemic bad-join window when >= 50% of its NCM-bearing items disagree
|
||||
s = lambda v: None if v is None else str(v).strip()
|
||||
dg = lambda v: "".join(ch for ch in str(v or "") if ch.isdigit())
|
||||
|
||||
|
||||
def ncm_status(tok, ncms):
|
||||
out = {}
|
||||
ncms = sorted(n for n in ncms if n)
|
||||
for i in range(0, len(ncms), 100):
|
||||
chunk = ncms[i:i + 100]
|
||||
req = urllib.request.Request("http://localhost:8000/v1/classify/batch", data=json.dumps({"products": [{"current_ncm": n, "description": ""} for n in chunk]}).encode(),
|
||||
headers={"Content-Type": "application/json", "Authorization": "Bearer " + tok})
|
||||
for n, r in zip(chunk, json.load(urllib.request.urlopen(req, timeout=600))["results"]):
|
||||
out[n] = r.get("status")
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
tok = [l.split("=", 1)[1].strip() for l in (ROOT / ".env").read_text(encoding="utf-8").splitlines() if l.startswith("AGENT_DEV_TOKEN=")][0]
|
||||
cur = _connect(os.environ).cursor()
|
||||
cur.execute("select CODIGO, PRODUTO, CLASSIFICACAO_FISCAL, DATA_CADASTRO, ULTIMA_ALTERACAO, PRECOVENDA from C000025")
|
||||
P = {s(r[0]): {"desc": s(r[1]), "ncm": s(r[2]) or "", "cad": r[3], "alt": r[4], "price": r[5]} for r in cur.fetchall()}
|
||||
by_ncm = defaultdict(list)
|
||||
for k, p in P.items():
|
||||
by_ncm[p["ncm"]].append((k, p["price"]))
|
||||
cur.execute("select ID, CODPRODUTO, NCM, DATA, VALOR, CFOP, CSOSN, CST, CEST, GRUPO_ICMS, CST_PIS_COFINS, NUMERO_NOTA from NFCE_ITENS where NCM is not null")
|
||||
items = [dict(zip(("id", "cod", "ncm", "date", "val", "cfop", "csosn", "cst", "cest", "gicms", "pis", "nota"), r)) for r in cur.fetchall()]
|
||||
for it in items:
|
||||
it["cod"] = s(it["cod"]); it["ncm"] = s(it["ncm"]) or ""
|
||||
joined = [it for it in items if it["cod"] in P]
|
||||
orphans = [it for it in items if it["cod"] not in P]
|
||||
|
||||
month = lambda it: str(it["date"])[:7]
|
||||
mstat = defaultdict(lambda: [0, 0])
|
||||
for it in joined:
|
||||
mstat[month(it)][0 if it["ncm"] == P[it["cod"]]["ncm"] else 1] += 1
|
||||
windows = {m for m, (ok, bad) in mstat.items() if bad and bad / (ok + bad) >= WINDOW_RATE}
|
||||
|
||||
status = ncm_status(tok, {it["ncm"] for it in joined} | {p["ncm"] for p in P.values()})
|
||||
# per-product timeline outside systemic windows for change detection
|
||||
timeline = defaultdict(list)
|
||||
for it in joined:
|
||||
if month(it) not in windows:
|
||||
timeline[it["cod"]].append((it["date"], it["ncm"]))
|
||||
|
||||
def changed_after(cod, cur_ncm):
|
||||
seq = sorted(timeline[cod])
|
||||
vals = [n for _, n in seq]
|
||||
if cur_ncm not in vals or vals[0] == cur_ncm:
|
||||
return False
|
||||
first_cur = vals.index(cur_ncm)
|
||||
return all(v != cur_ncm for v in vals[:first_cur]) and all(v == cur_ncm for v in vals[first_cur:])
|
||||
|
||||
cls = Counter(); reasons = Counter(); by_month = Counter(); examples = defaultdict(list); trusted = []
|
||||
for it in joined:
|
||||
p = P[it["cod"]]
|
||||
if it["ncm"] == p["ncm"]:
|
||||
if month(it) not in windows and len(dg(it["ncm"])) == 8:
|
||||
trusted.append(it)
|
||||
continue
|
||||
if dg(it["ncm"]).lstrip("0") == dg(p["ncm"]).lstrip("0") and dg(it["ncm"]):
|
||||
c, r = "NORMALIZATION_BUG", "DIGITS_EQUAL_AFTER_NORMALIZATION"
|
||||
elif status.get(it["ncm"]) == "LEGACY_NCM" or status.get(p["ncm"]) == "LEGACY_NCM":
|
||||
c, r = "LEGACY_NCM", "ITEM_OR_PRODUCT_NCM_IS_LEGACY"
|
||||
elif month(it) in windows:
|
||||
c, r = "BAD_JOIN", f"SYSTEMIC_WINDOW_{month(it)}"
|
||||
elif any(k != it["cod"] and pr is not None and it["val"] is not None and abs(float(pr) - float(it["val"])) < 0.005 for k, pr in by_ncm.get(it["ncm"], [])):
|
||||
c, r = "BAD_JOIN", "ALT_PRODUCT_WITH_SAME_NCM_AND_PRICE"
|
||||
elif p["alt"] is not None and str(p["alt"])[:10] > str(it["date"])[:10] and changed_after(it["cod"], p["ncm"]):
|
||||
c, r = "PRODUCT_CHANGED_AFTER_DOCUMENT", "MONOTONIC_SWITCH_TO_CURRENT_NCM_AND_PRODUCT_MODIFIED_LATER"
|
||||
elif status.get(it["ncm"]) == "EXACT_NCM":
|
||||
vals = [n for _, n in timeline[it["cod"]]]
|
||||
if vals and vals.count(p["ncm"]) / len(vals) > 0.5:
|
||||
c, r = "TRUE_HISTORICAL_DIFFERENCE", "ISOLATED_DIFFERENCE_PRODUCT_OTHERWISE_CONSISTENT_UNPROVEN_CAUSE"
|
||||
else:
|
||||
c, r = "UNKNOWN", "NO_CONSISTENT_PRODUCT_HISTORY"
|
||||
else:
|
||||
c, r = "UNKNOWN", "ITEM_NCM_NOT_VALID_IN_NCM_DATASET"
|
||||
cls[c] += 1; reasons[(c, r)] += 1; by_month[month(it)] += 1
|
||||
if len(examples[c]) < 4:
|
||||
examples[c].append({"nfce_item_id": it["id"], "product": it["cod"], "product_desc": p["desc"], "product_ncm": p["ncm"], "item_ncm": it["ncm"], "date": str(it["date"])[:10], "reason": r})
|
||||
|
||||
n_mis = sum(cls.values())
|
||||
trusted_ids = [t["id"] for t in trusted]
|
||||
doc = {"artifact": "NCM_HISTORICAL_RECONCILIATION_V1", "firebird_writes": 0,
|
||||
"items_with_ncm": len(items), "items_joined_to_product": len(joined), "orphan_items_no_product": len(orphans),
|
||||
"mismatches_total": n_mis, "mismatches_in_initial_sample_5000": 1671,
|
||||
"classification": dict(cls), "reasons": {f"{c}|{r}": n for (c, r), n in reasons.most_common()},
|
||||
"systemic_bad_join_windows": {m: {"matches": mstat[m][0], "mismatches": mstat[m][1]} for m in sorted(windows)},
|
||||
"window_rule": f"month with >= {int(WINDOW_RATE*100)}% NCM disagreement is treated as a systemic bad-join window",
|
||||
"mismatches_by_month": dict(sorted(by_month.items())), "examples": examples,
|
||||
"technical_bugs_found": {"normalization": cls["NORMALIZATION_BUG"], "note": "0 => no code bug; the mismatches are data-identity problems (CODPRODUTO does not reliably identify the product in 2022-12 and a few other windows)."},
|
||||
"trusted_corpus": {"definition": "NFC-e item with NCM present, product exists, item NCM == product NCM (identity proof), 8-digit NCM, month not a systemic window",
|
||||
"TRUSTED_HISTORICAL_ITEMS": len(trusted), "excluded_matches_in_windows": sum(mstat[m][0] for m in windows),
|
||||
"items_without_ncm_not_trusted": 34946 - len(items)}}
|
||||
(ROOT / "docs" / "NCM_HISTORICAL_RECONCILIATION_V1.json").write_text(json.dumps(doc, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
(ROOT / "docs" / "runs" / "trusted_historical_items.json").write_text(json.dumps([{k: (str(v) if k == "date" else v) for k, v in t.items()} for t in trusted], ensure_ascii=False, default=str), encoding="utf-8")
|
||||
print(json.dumps({k: doc[k] for k in ("items_with_ncm", "items_joined_to_product", "orphan_items_no_product", "mismatches_total", "classification", "systemic_bad_join_windows")}, ensure_ascii=True), len(trusted))
|
||||
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Write release/RELEASE_MANIFEST.json: one coherent record of what RC1 is made of (no secrets).
|
||||
|
||||
python scripts/release_manifest.py <AGENT_TOKEN> [HUB_URL]
|
||||
|
||||
Registers together: release version, agent/hub versions, fiscal engine + decision versions, engine module versions, ACTIVE dataset versions
|
||||
(kind, provider, sha256, record_count) read from the running Hub, alembic head (from the migration files) and the applied revision.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
|
||||
def get(url, token):
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return json.load(r)
|
||||
|
||||
|
||||
def main():
|
||||
token = sys.argv[1]
|
||||
hub = sys.argv[2] if len(sys.argv) > 2 else "http://localhost:8000"
|
||||
from central.app.classification import DECISION_VERSION, ENGINE_VERSION
|
||||
from central.app.fiscal_decision import DECISION_ENGINE_VERSION
|
||||
from central.app.icms_context import ICMS_CONTEXT_ENGINE_VERSION
|
||||
from central.app.product_defaults import PRODUCT_DEFAULTS_ENGINE_VERSION
|
||||
version = (ROOT / "release" / "VERSION").read_text(encoding="utf-8").strip()
|
||||
heads = sorted(p.name.split("_")[0] for p in (ROOT / "central" / "alembic" / "versions").glob("00*.py"))
|
||||
try:
|
||||
applied = subprocess.run(["docker", "exec", "projeto-fiscal-hub-1", "sh", "-c", "cd /app && alembic -c central/alembic.ini current"], capture_output=True, text=True, timeout=60).stdout
|
||||
applied = re.findall(r"^(\d{4})", applied, re.M)[-1:] or ["unknown"]
|
||||
except Exception: # noqa: BLE001
|
||||
applied = ["unknown"]
|
||||
datasets = sorted(({"kind": d["kind"], "sha256": d["sha256"][:16], "record_count": d["record_count"]} for d in get(f"{hub}/v1/datasets/status", token) if d["status"] == "ACTIVE"), key=lambda x: x["kind"])
|
||||
manifest = {"artifact": "RELEASE_MANIFEST", "GTI_FISCAL_VERSION": version, "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
"hub": get(f"{hub}/version", token), "agent_version": version,
|
||||
"fiscal_engine": {"ENGINE_VERSION": ENGINE_VERSION, "DECISION_VERSION": DECISION_VERSION, "DECISION_ENGINE_VERSION": DECISION_ENGINE_VERSION,
|
||||
"PRODUCT_DEFAULTS_ENGINE_VERSION": PRODUCT_DEFAULTS_ENGINE_VERSION, "ICMS_CONTEXT_ENGINE_VERSION": ICMS_CONTEXT_ENGINE_VERSION},
|
||||
"migrations": {"alembic_head_in_repo": heads[-1], "applied_revision": applied[0], "count": len(heads)},
|
||||
"datasets_active": datasets, "note": "A release bump does not change fiscal semantics; engine/decision versions are versioned independently."}
|
||||
(ROOT / "release" / "RELEASE_MANIFEST.json").write_text(json.dumps(manifest, indent=1), encoding="utf-8")
|
||||
print(json.dumps({"version": version, "engine": ENGINE_VERSION, "migrations": manifest["migrations"], "datasets": len(datasets)}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Guarded rollback of one field to the BEFORE values recorded in the local-completion write audit.
|
||||
usage: python scripts/rollback_field.py FIELD [--db NAME ...] Writes a rollback audit next to the original audit and verifies 100% against the BEFORE snapshot."""
|
||||
import argparse, gzip, json, pickle, sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from dotenv import load_dotenv
|
||||
import local_completion as lc
|
||||
|
||||
ap = argparse.ArgumentParser(); ap.add_argument("field"); ap.add_argument("--db", nargs="*"); ap.add_argument("--plan", action="store_true", help="only the codes listed in rollback_plan.json"); a = ap.parse_args()
|
||||
plan = json.loads((Path(__file__).resolve().parents[1] / "docs" / "runs" / "local_completion" / "rollback_plan.json").read_text()) if a.plan else None
|
||||
load_dotenv(lc.ROOT / ".env")
|
||||
ex = lc.load_executor(); al = lc.allowlist()
|
||||
out = {}
|
||||
for name in a.db or list(al):
|
||||
path = al[name]; todo = []
|
||||
for line in gzip.open(lc.STATE / f"{name}_write_audit.ndjson.gz", "rt", encoding="utf-8"):
|
||||
r = json.loads(line)
|
||||
if r.get("outcome") == "SUCCEEDED" and a.field in r["fields"] and (plan is None or r["product_code"] in plan.get(name, {}).get(a.field, [])):
|
||||
todo.append((r["product_code"], r["fields"][a.field]["before"], r["fields"][a.field]["after"]))
|
||||
conn = lc.connect(path, rw=True); ok = fail = 0; audit = []
|
||||
for code, before, after in todo:
|
||||
try:
|
||||
res = ex.execute_guarded_product_update(conn, database_path=path, product_code=code, changes={a.field: (after, before)}, now=datetime.now(timezone.utc).isoformat(), restore_before=True)
|
||||
audit.append({**res, "outcome": "ROLLED_BACK"}); ok += 1
|
||||
except Exception as e: # noqa: BLE001
|
||||
audit.append({"product_code": code, "outcome": "FAILED", "error": str(e)}); fail += 1
|
||||
conn.close()
|
||||
with gzip.open(lc.STATE / f"{name}_rollback_{a.field}{"_plan" if plan else ""}.ndjson.gz", "wt", encoding="utf-8") as f:
|
||||
for r in audit: f.write(json.dumps(r, default=str) + "\n")
|
||||
# independent verification against the BEFORE snapshot
|
||||
snap = pickle.load(gzip.open(lc.STATE / f"{name}_before.pkl.gz", "rb")); cols = snap["columns"]; i = cols.index(a.field)
|
||||
c2 = lc.connect(path); cur = c2.cursor(); cur.execute(f"select CODIGO, {a.field} from C000025"); live = {}
|
||||
for k, v in cur.fetchall(): live.setdefault(k, []).append(v)
|
||||
n = lambda v: None if v is None else str(v).strip() # noqa: E731
|
||||
mism = sum(1 for k, r in snap["rows"].items() if (plan is None or k in plan.get(name, {}).get(a.field, [])) and k in live and any(n(v) != n(r[i]) for v in live[k]))
|
||||
out[name] = {"rollback_planned": len(todo), "rolled_back": ok, "failed": fail, "column_mismatch_vs_before_snapshot": mism, "rows_snapshot": len(snap["rows"])}
|
||||
print(name, out[name], flush=True)
|
||||
json.dump(out, open(lc.STATE / f"rollback_{a.field}{"_plan" if plan else ""}_summary.json", "w"), indent=1)
|
||||
@@ -0,0 +1,661 @@
|
||||
"""Run the narrowly scoped, read-only Mission 00 Firebird probe."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(PROJECT_ROOT / "src"))
|
||||
|
||||
from gti_discovery.firebird_connection import (
|
||||
AUTHORIZED_DATABASE_PATH,
|
||||
_FirebirdAdapter,
|
||||
_ADAPTER_CREATION_TOKEN,
|
||||
_create_firebird_adapter,
|
||||
connect_authorized_read_only,
|
||||
get_database_identity,
|
||||
validate_authorized_database_path,
|
||||
validate_authorized_database_identity,
|
||||
)
|
||||
from gti_discovery.collector import _safe_sample_rows, collect_metadata
|
||||
from gti_discovery.fingerprint import (
|
||||
SCHEMA_CHANGED,
|
||||
SCHEMA_SUPPORTED,
|
||||
SCHEMA_UNKNOWN,
|
||||
build_fingerprint,
|
||||
classify_schema,
|
||||
)
|
||||
from gti_discovery.mapping import build_schema_mapping, identify_product_candidates
|
||||
from gti_discovery.provenance import (
|
||||
ProvenanceError,
|
||||
validate_fdb_provenance,
|
||||
validate_imported_fdb_provenance,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_TARGET = AUTHORIZED_DATABASE_PATH
|
||||
CLIENT_LIBRARY = Path(r"C:\Program Files\Firebird\Firebird_2_5\bin\fbclient.dll")
|
||||
_BLOCKER_CATEGORIES = {
|
||||
ValueError: "target_rejected",
|
||||
ImportError: "driver_unavailable",
|
||||
FileNotFoundError: "client_library_unavailable",
|
||||
PermissionError: "connection_failed",
|
||||
RuntimeError: "connection_failed",
|
||||
}
|
||||
|
||||
|
||||
class _FdbDriver:
|
||||
version = "unknown"
|
||||
|
||||
def __init__(self, fdb_module: Any):
|
||||
self._fdb = fdb_module
|
||||
self.version = str(getattr(fdb_module, "__version__", "unknown"))
|
||||
self._last_connection = None
|
||||
|
||||
def connect(self, **options):
|
||||
if set(options) - {"database", "read_only", "client_library"}:
|
||||
raise RuntimeError("unexpected connection settings")
|
||||
if options.pop("read_only", None) is not True:
|
||||
raise RuntimeError("read-only mode is required")
|
||||
client_library = options.pop("client_library", None)
|
||||
user = os.environ.get("FB_USER")
|
||||
password = os.environ.get("FB_PASSWORD")
|
||||
if not user or not password:
|
||||
raise RuntimeError("Firebird credentials are unavailable in FB_USER/FB_PASSWORD")
|
||||
isolation_level = _read_only_isolation_level(self._fdb)
|
||||
raw = self._fdb.connect(
|
||||
database=options["database"],
|
||||
user=user,
|
||||
password=password,
|
||||
fb_library_name=client_library,
|
||||
isolation_level=isolation_level,
|
||||
)
|
||||
self._last_connection = raw
|
||||
observed = getattr(raw, "default_tpb", None)
|
||||
if observed != isolation_level:
|
||||
close = getattr(raw, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
raise RuntimeError("driver did not establish the fdb read-only TPB")
|
||||
evidence = {
|
||||
"accepted": True,
|
||||
"driver_api": "fdb.Connection.default_tpb",
|
||||
"configured_tpb": bytes(isolation_level),
|
||||
"observed_default_tpb": bytes(observed),
|
||||
}
|
||||
return _create_firebird_adapter(raw, evidence, _ADAPTER_CREATION_TOKEN)
|
||||
|
||||
def facts(self) -> dict[str, Any]:
|
||||
return {name: _fact(self._last_connection, name) for name in (
|
||||
"engine_version", "database_sql_dialect", "ods_version",
|
||||
"ods_minor_version", "charset", "page_size",
|
||||
)}
|
||||
|
||||
def compatibility_evidence(self, client_version: str) -> dict[str, Any]:
|
||||
isolation_level = _read_only_isolation_level(self._fdb)
|
||||
return {
|
||||
"fdb_version": self.version,
|
||||
"client_version": client_version,
|
||||
"firebird_version": ".".join(str(client_version).split(".")[:3]),
|
||||
"read_only_isolation": _read_only_isolation_level(self._fdb) is not None,
|
||||
"read_only_tpb": bytes(isolation_level),
|
||||
"read_only_api": "fdb.Connection.default_tpb",
|
||||
}
|
||||
|
||||
|
||||
def _read_only_isolation_level(fdb: Any) -> Any:
|
||||
"""Use fdb 2.0.4's actual read-only TPB constant (including its spelling)."""
|
||||
value = getattr(fdb, "ISOLATION_LEVEL_READ_COMMITED_RO", None)
|
||||
if isinstance(value, bytes) and value == b"\x03\x08\x06\x0f\x11":
|
||||
return value
|
||||
raise RuntimeError("fdb read-only isolation API is unavailable")
|
||||
|
||||
|
||||
def _fact(connection: Any, name: str) -> Any:
|
||||
try:
|
||||
value = getattr(connection, name)
|
||||
except Exception:
|
||||
return None
|
||||
if isinstance(value, (str, int, float, bool)) or value is None:
|
||||
return value
|
||||
return str(value)
|
||||
|
||||
|
||||
def _read_file_version(path: Path) -> str | None:
|
||||
"""Read Windows file-version metadata without loading the client DLL."""
|
||||
if sys.platform != "win32":
|
||||
return None
|
||||
try:
|
||||
version = ctypes.windll.version
|
||||
size = version.GetFileVersionInfoSizeW(str(path), None)
|
||||
if not size:
|
||||
return None
|
||||
buffer = ctypes.create_string_buffer(size)
|
||||
if not version.GetFileVersionInfoW(str(path), 0, size, buffer):
|
||||
return None
|
||||
pointer = ctypes.c_void_p()
|
||||
length = ctypes.c_uint()
|
||||
if not version.VerQueryValueW(buffer, "\\", ctypes.byref(pointer), ctypes.byref(length)):
|
||||
return None
|
||||
class FixedFileInfo(ctypes.Structure):
|
||||
_fields_ = [("signature", ctypes.c_uint32), ("struct_version", ctypes.c_uint32),
|
||||
("file_version_ms", ctypes.c_uint32), ("file_version_ls", ctypes.c_uint32)]
|
||||
info = ctypes.cast(pointer, ctypes.POINTER(FixedFileInfo)).contents
|
||||
return ".".join((str(info.file_version_ms >> 16), str(info.file_version_ms & 0xffff),
|
||||
str(info.file_version_ls >> 16), str(info.file_version_ls & 0xffff)))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _client_metadata(path: Path) -> dict[str, Any]:
|
||||
exists = path.is_file()
|
||||
return {"path": str(path), "exists": exists, "version": _read_file_version(path) if exists else None}
|
||||
|
||||
|
||||
def _read_client_architecture(path: Path) -> str:
|
||||
"""Read PE architecture metadata without loading or executing the DLL."""
|
||||
try:
|
||||
with path.open("rb") as handle:
|
||||
if handle.read(2) != b"MZ":
|
||||
return "UNKNOWN"
|
||||
handle.seek(0x3C)
|
||||
offset = struct.unpack("<I", handle.read(4))[0]
|
||||
handle.seek(offset)
|
||||
if handle.read(4) != b"PE\x00\x00":
|
||||
return "UNKNOWN"
|
||||
machine = struct.unpack("<H", handle.read(2))[0]
|
||||
except (OSError, struct.error):
|
||||
return "UNKNOWN"
|
||||
return {0x014C: "x86", 0x8664: "x64", 0xAA64: "arm64"}.get(machine, "UNKNOWN")
|
||||
|
||||
|
||||
def _python_architecture() -> str:
|
||||
return "x64" if struct.calcsize("P") == 8 else "x86" if struct.calcsize("P") == 4 else "UNKNOWN"
|
||||
|
||||
|
||||
def validate_firebird_runtime(*, fdb_module: Any = None) -> dict[str, Any]:
|
||||
"""Return deterministic dependency evidence; never infer unknown facts."""
|
||||
try:
|
||||
provenance = validate_fdb_provenance()
|
||||
except ProvenanceError as error:
|
||||
return {"status": "UNKNOWN", "blocker": "compatibility_unproven", "provenance": error.evidence}
|
||||
client = _client_metadata(CLIENT_LIBRARY)
|
||||
client = dict(client)
|
||||
client["architecture"] = client.get("architecture", _read_client_architecture(CLIENT_LIBRARY))
|
||||
distribution_version = None
|
||||
try:
|
||||
distribution_version = importlib.metadata.version("fdb")
|
||||
except (importlib.metadata.PackageNotFoundError, ImportError):
|
||||
pass
|
||||
fdb_version = getattr(fdb_module, "__version__", distribution_version)
|
||||
python_architecture = _python_architecture()
|
||||
read_only_api = (
|
||||
fdb_module is not None
|
||||
and isinstance(getattr(fdb_module, "ISOLATION_LEVEL_READ_COMMITED_RO", None), bytes)
|
||||
and getattr(fdb_module, "ISOLATION_LEVEL_READ_COMMITED_RO") == b"\x03\x08\x06\x0f\x11"
|
||||
)
|
||||
version_match = fdb_version == "2.0.4" and distribution_version == "2.0.4"
|
||||
architecture_match = client["architecture"] in {"x86", "x64", "arm64"} and (
|
||||
(python_architecture == "x64" and client["architecture"] == "x64")
|
||||
or (python_architecture == "x86" and client["architecture"] == "x86")
|
||||
)
|
||||
valid = (
|
||||
provenance.get("status") == "VALID"
|
||||
and client["exists"] is True
|
||||
and client["version"] == "2.5.9.27139"
|
||||
and client["architecture"] != "UNKNOWN"
|
||||
and architecture_match
|
||||
and version_match
|
||||
and fdb_module is not None
|
||||
and read_only_api
|
||||
)
|
||||
compatibility = {
|
||||
"status": "VALID" if valid else "UNKNOWN",
|
||||
"python_client_architecture_match": architecture_match,
|
||||
"fdb_client_version_match": version_match,
|
||||
"read_only_api": read_only_api,
|
||||
}
|
||||
return {
|
||||
"status": "VALID" if valid else "UNKNOWN",
|
||||
"python_runtime": {"version": platform.python_version(), "architecture": python_architecture},
|
||||
"fdb": {"version": fdb_version, "distribution_version": distribution_version},
|
||||
"client": client,
|
||||
"provenance": provenance,
|
||||
"compatibility": compatibility,
|
||||
**({} if valid else {"blocker": "compatibility_unproven"}),
|
||||
}
|
||||
|
||||
|
||||
def _blocker_category(error: Exception) -> str:
|
||||
if type(error).__name__ == "DatabaseError" and (
|
||||
"-902" in str(error) or "user name and password are not defined" in str(error).lower()
|
||||
):
|
||||
return "credentials_rejected"
|
||||
if isinstance(error, ValueError) and "read-only" in str(error).lower():
|
||||
return "read_only_unproven"
|
||||
if isinstance(error, RuntimeError) and "credentials" in str(error).lower():
|
||||
return "credentials_unavailable"
|
||||
if isinstance(error, RuntimeError) and "compatib" in str(error).lower():
|
||||
return "compatibility_unproven"
|
||||
if isinstance(error, RuntimeError) and "provenance" in str(error).lower():
|
||||
return "compatibility_unproven"
|
||||
return next((category for error_type, category in _BLOCKER_CATEGORIES.items()
|
||||
if isinstance(error, error_type)), "unexpected_failure")
|
||||
|
||||
|
||||
def _firebird_259(version: Any) -> bool:
|
||||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)", str(version or ""))
|
||||
return bool(match and tuple(int(item) for item in match.groups()) == (2, 5, 9))
|
||||
|
||||
|
||||
def _validate_runtime_compatibility(driver: Any) -> dict[str, Any]:
|
||||
"""Prove the installed 2.5.9 client and usable driver before connect."""
|
||||
client = _client_metadata(CLIENT_LIBRARY)
|
||||
if not client["exists"] or not _firebird_259(client["version"]):
|
||||
raise RuntimeError("Firebird 2.5.9 client compatibility is unproven")
|
||||
if type(driver) is not _FdbDriver:
|
||||
raise RuntimeError("Firebird driver/client compatibility is unproven")
|
||||
try:
|
||||
import fdb as installed_fdb
|
||||
installed_version = importlib.metadata.version("fdb")
|
||||
except (ImportError, importlib.metadata.PackageNotFoundError) as error:
|
||||
raise RuntimeError("Firebird driver/client compatibility is unproven") from error
|
||||
if (
|
||||
driver._fdb is not installed_fdb
|
||||
or installed_version != "2.0.4"
|
||||
or getattr(installed_fdb, "__version__", None) != "2.0.4"
|
||||
or _read_only_isolation_level(installed_fdb) != b"\x03\x08\x06\x0f\x11"
|
||||
):
|
||||
raise RuntimeError("Firebird driver/client compatibility is unproven")
|
||||
return client
|
||||
|
||||
|
||||
def _validate_preimport_compatibility() -> dict[str, Any]:
|
||||
provenance = validate_fdb_provenance()
|
||||
client = _client_metadata(CLIENT_LIBRARY)
|
||||
client = dict(client)
|
||||
client["architecture"] = client.get("architecture", _read_client_architecture(CLIENT_LIBRARY))
|
||||
try:
|
||||
fdb_version = importlib.metadata.version("fdb")
|
||||
except importlib.metadata.PackageNotFoundError as error:
|
||||
raise RuntimeError("fdb 2.0.4 compatibility is unproven") from error
|
||||
if (
|
||||
not client["exists"]
|
||||
or not _firebird_259(client["version"])
|
||||
or client["architecture"] == "UNKNOWN"
|
||||
or client["architecture"] != _python_architecture()
|
||||
or fdb_version != "2.0.4"
|
||||
):
|
||||
raise RuntimeError("Firebird 2.5.9/fdb 2.0.4 compatibility is unproven")
|
||||
if not _preimport_read_only_api_proven():
|
||||
raise RuntimeError("Firebird read-only isolation API compatibility is unproven")
|
||||
return {"client": client, "provenance": provenance}
|
||||
|
||||
|
||||
def _preimport_read_only_api_proven() -> bool:
|
||||
"""Prove the pinned driver's read-only API from installed source, without import."""
|
||||
try:
|
||||
spec = __import__("importlib.util", fromlist=["find_spec"]).find_spec("fdb")
|
||||
locations = list(getattr(spec, "submodule_search_locations", ()) or ())
|
||||
if not locations:
|
||||
return False
|
||||
package = Path(locations[0])
|
||||
init_source = (package / "__init__.py").read_text(encoding="utf-8")
|
||||
core_source = (package / "fbcore.py").read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeError, ImportError, AttributeError, TypeError):
|
||||
return False
|
||||
return (
|
||||
"ISOLATION_LEVEL_READ_COMMITED_RO" in init_source
|
||||
and "ISOLATION_LEVEL_READ_COMMITED_RO = bs([" in core_source
|
||||
and all(token in core_source for token in (
|
||||
"isc_tpb_version3", "isc_tpb_read", "isc_tpb_wait",
|
||||
"isc_tpb_read_committed", "isc_tpb_rec_version",
|
||||
))
|
||||
)
|
||||
|
||||
|
||||
def _security_gate(path: os.PathLike[str] | str, driver: Any) -> dict[str, Any]:
|
||||
validate_authorized_database_path(path)
|
||||
if not os.environ.get("FB_USER") or not os.environ.get("FB_PASSWORD"):
|
||||
raise RuntimeError("Firebird credentials are unavailable in FB_USER/FB_PASSWORD")
|
||||
evidence = validate_firebird_runtime(fdb_module=getattr(driver, "_fdb", None))
|
||||
if evidence["status"] != "VALID":
|
||||
raise RuntimeError("Firebird driver/client compatibility is unproven")
|
||||
return {"client": evidence["client"], "compatibility_evidence": evidence}
|
||||
|
||||
|
||||
_SAMPLE_DENYLIST = (
|
||||
"CUSTOMER", "CLIENT", "EMAIL", "PHONE", "MOBILE", "CONTACT", "ADDRESS",
|
||||
"PRICE", "COST", "AMOUNT", "TOTAL", "TAX", "CRED", "PASSWORD",
|
||||
"SECRET", "TOKEN", "BLOB", "SOURCE", "DEFAULT", "SALARY", "MARGIN",
|
||||
"DISCOUNT", "PROFIT", "IMAGE", "PHOTO", "FILE", "PATH", "URL",
|
||||
)
|
||||
_SAMPLE_ALLOWLIST = (
|
||||
"ID", "KEY", "CODE", "CODIGO", "SKU", "ITEM", "PRODUCT", "DESCRIPTION",
|
||||
"DESCRICAO", "NAME", "NOME", "BARCODE", "EAN", "GTIN", "REFERENCE",
|
||||
"REFERENCIA", "REF", "UNIT", "UNIDADE", "CATEGORY", "GROUP", "GRUPO",
|
||||
"BRAND", "MARCA", "PRODUTO", "ESTOQUE", "STATUS", "ACTIVE", "ATIVO",
|
||||
)
|
||||
|
||||
|
||||
def _permitted_sample_plan(metadata: dict[str, Any]) -> list[tuple[str, list[str]]]:
|
||||
fields_by_relation: dict[str, list[dict[str, Any]]] = {}
|
||||
for field in metadata.get("fields", []):
|
||||
if not isinstance(field, dict):
|
||||
continue
|
||||
relation = str(field.get("RELATION_NAME", "")).strip()
|
||||
if relation:
|
||||
fields_by_relation.setdefault(relation.upper(), []).append(field)
|
||||
candidates = identify_product_candidates(metadata, {})
|
||||
plan = []
|
||||
for candidate in candidates:
|
||||
relation = str(candidate.get("relation", "")).strip()
|
||||
if not relation or candidate.get("classification") == "UNKNOWN":
|
||||
continue
|
||||
columns = []
|
||||
for field in fields_by_relation.get(relation.upper(), []):
|
||||
name = str(field.get("FIELD_NAME", "")).strip()
|
||||
type_name = str(field.get("TYPE_NAME", field.get("FIELD_TYPE", ""))).upper()
|
||||
upper_name = name.upper()
|
||||
if not name or any(word in upper_name for word in _SAMPLE_DENYLIST):
|
||||
continue
|
||||
if "BLOB" in type_name or not any(word in upper_name for word in _SAMPLE_ALLOWLIST):
|
||||
continue
|
||||
columns.append(name)
|
||||
if columns:
|
||||
plan.append((relation, columns[:8]))
|
||||
return plan
|
||||
|
||||
|
||||
def _collect_samples(reader: Any, metadata: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
|
||||
samples = {}
|
||||
plan = _permitted_sample_plan(metadata)
|
||||
if not isinstance(plan, list) or any(
|
||||
not isinstance(item, tuple)
|
||||
or len(item) != 2
|
||||
or not isinstance(item[0], str)
|
||||
or not isinstance(item[1], list)
|
||||
or not item[1]
|
||||
or any(
|
||||
not isinstance(column, str)
|
||||
or any(word in column.upper() for word in _SAMPLE_DENYLIST)
|
||||
or not any(word in column.upper() for word in _SAMPLE_ALLOWLIST)
|
||||
for column in item[1]
|
||||
)
|
||||
for item in plan
|
||||
):
|
||||
raise RuntimeError("sample plan is not permitted")
|
||||
for relation, columns in plan:
|
||||
rows = reader.sample_relation_rows(relation, columns, 5)
|
||||
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
|
||||
raise RuntimeError("sample evidence is malformed")
|
||||
samples[relation] = _safe_sample_rows(rows)
|
||||
return samples
|
||||
|
||||
|
||||
def _blocked(
|
||||
blocker: str,
|
||||
*,
|
||||
connection_attempts: int = 0,
|
||||
other_databases_accessed: int | str = 0,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"status": "MISSION_00_BLOCKED",
|
||||
"blocker": blocker,
|
||||
"application_writes": 0,
|
||||
"other_databases_accessed": other_databases_accessed,
|
||||
"connection_attempts": connection_attempts,
|
||||
}
|
||||
|
||||
|
||||
def _valid_metadata(metadata: Any) -> bool:
|
||||
if not isinstance(metadata, dict):
|
||||
return False
|
||||
if not all(isinstance(metadata.get(name), list) for name in ("relations", "fields", "errors", "unavailable")):
|
||||
return False
|
||||
return (
|
||||
all(isinstance(row, dict) for name in ("relations", "fields", "errors") for row in metadata[name])
|
||||
and all(isinstance(name, str) and bool(name) for name in metadata["unavailable"])
|
||||
)
|
||||
|
||||
|
||||
def _valid_mapping(mapping: Any) -> bool:
|
||||
required = {"product", "fields", "fiscal", "stock", "media", "incremental", "triggers"}
|
||||
if type(mapping) is not dict or set(mapping) != required:
|
||||
return False
|
||||
allowed = {"CONFIRMED", "INFERRED", "UNKNOWN"}
|
||||
|
||||
def strings(value: Any) -> bool:
|
||||
return type(value) is list and all(type(item) is str and bool(item) for item in value)
|
||||
|
||||
def classified(value: Any, keys: set[str]) -> bool:
|
||||
return type(value) is dict and set(value) == keys and value["classification"] in allowed
|
||||
|
||||
def field_record(value: Any) -> bool:
|
||||
if type(value) is not dict or not {"classification", "sample_present", "type"} <= set(value):
|
||||
return False
|
||||
if set(value) - {"classification", "sample_present", "type", "subtype", "value"}:
|
||||
return False
|
||||
return value["classification"] in allowed and type(value["sample_present"]) is bool and type(value["type"]) is str and bool(value["type"]) and all(
|
||||
type(value[key]) in (str, int, float, bool) for key in ("subtype", "value") if key in value
|
||||
)
|
||||
|
||||
def field_map(value: Any) -> bool:
|
||||
return type(value) is dict and all(type(key) is str and bool(key) and field_record(item) for key, item in value.items())
|
||||
|
||||
def relation_record(value: Any) -> bool:
|
||||
if type(value) is not dict or not {"classification", "fields", "relationships"} <= set(value):
|
||||
return False
|
||||
if set(value) - {"classification", "fields", "relationships", "evidence"}:
|
||||
return False
|
||||
relationships = value["relationships"]
|
||||
return value["classification"] in allowed and field_map(value["fields"]) and type(relationships) is list and all(
|
||||
type(item) is dict and set(item) == {"child_relation", "child_field", "referenced_relation", "referenced_field"}
|
||||
and all(type(item[key]) is str and bool(item[key]) for key in item)
|
||||
for item in relationships
|
||||
) and ("evidence" not in value or type(value["evidence"]) is str)
|
||||
|
||||
def relation_map(value: Any) -> bool:
|
||||
return type(value) is dict and all(type(key) is str and bool(key) and relation_record(item) for key, item in value.items())
|
||||
|
||||
product = mapping["product"]
|
||||
product_keys = {"classification", "relation", "evidence", "candidates"}
|
||||
candidate_keys = {"classification", "relation", "evidence"}
|
||||
evidence_keys = {"primary_key", "field_count", "semantic_fields", "sample_fields", "semantic_sample_fields", "approximate_count"}
|
||||
if not classified(product, product_keys) or (product["relation"] is not None and (type(product["relation"]) is not str or not product["relation"])) or type(product["evidence"]) is not dict or set(product["evidence"]) - evidence_keys:
|
||||
return False
|
||||
candidates = product.get("candidates")
|
||||
if not isinstance(candidates, list) or not all(
|
||||
classified(item, candidate_keys) and type(item["relation"]) is str and bool(item["relation"])
|
||||
and type(item["evidence"]) is dict and set(item["evidence"]) - evidence_keys == set()
|
||||
and strings(item["evidence"].get("primary_key"))
|
||||
and type(item["evidence"].get("field_count")) is int
|
||||
and strings(item["evidence"].get("semantic_fields")) for item in candidates
|
||||
):
|
||||
return False
|
||||
for evidence in (product["evidence"], *(item["evidence"] for item in candidates)):
|
||||
if not evidence:
|
||||
continue
|
||||
if not strings(evidence.get("primary_key")) or type(evidence.get("field_count")) is not int or evidence["field_count"] < 0 or not strings(evidence.get("semantic_fields")):
|
||||
return False
|
||||
if "sample_fields" in evidence and not strings(evidence["sample_fields"]):
|
||||
return False
|
||||
if "semantic_sample_fields" in evidence and not strings(evidence["semantic_sample_fields"]):
|
||||
return False
|
||||
if "approximate_count" in evidence and type(evidence["approximate_count"]) not in (str, int, float, bool, type(None)):
|
||||
return False
|
||||
if not field_map(mapping["fields"]):
|
||||
return False
|
||||
for section in ("fiscal", "media"):
|
||||
value = mapping[section]
|
||||
if not classified(value, {"classification", "fields", "relations"}) or not field_map(value["fields"]) or not relation_map(value["relations"]):
|
||||
return False
|
||||
if not classified(mapping["stock"], {"classification", "relations"}) or not relation_map(mapping["stock"]["relations"]):
|
||||
return False
|
||||
incremental = mapping["incremental"]
|
||||
evidence = incremental.get("evidence")
|
||||
if not classified(incremental, {"classification", "evidence"}) or type(evidence) is not dict or set(evidence) != {"timestamps", "triggers", "generators"}:
|
||||
return False
|
||||
if not strings(evidence["timestamps"]) or type(evidence["triggers"]) is not int or evidence["triggers"] < 0 or type(evidence["generators"]) is not int or evidence["generators"] < 0:
|
||||
return False
|
||||
triggers = mapping["triggers"]
|
||||
return type(triggers) is list and all(
|
||||
classified(trigger, {"classification", "relation", "trigger", "timing", "event", "trigger_type", "dependencies", "source"})
|
||||
and all(type(trigger[key]) is str and bool(trigger[key]) for key in ("relation", "trigger", "timing", "event", "source"))
|
||||
and type(trigger["trigger_type"]) in (str, int) and bool(str(trigger["trigger_type"]))
|
||||
and strings(trigger.get("dependencies"))
|
||||
for trigger in triggers
|
||||
)
|
||||
|
||||
|
||||
def _valid_fingerprint(value: Any) -> bool:
|
||||
return isinstance(value, str) and bool(re.fullmatch(r"[0-9a-f]{64}", value))
|
||||
|
||||
|
||||
def _public_compatibility(evidence: Any) -> dict[str, str]:
|
||||
"""Expose compatibility categories only; never serialize internal evidence."""
|
||||
if not isinstance(evidence, dict):
|
||||
return {"status": "UNKNOWN"}
|
||||
return {
|
||||
"status": evidence.get("status", "UNKNOWN") if evidence.get("status") in {"VALID", "UNKNOWN"} else "UNKNOWN",
|
||||
"architecture": "MATCH" if evidence.get("compatibility", {}).get("python_client_architecture_match") is True else "UNKNOWN",
|
||||
"fdb_client": "MATCH" if evidence.get("compatibility", {}).get("fdb_client_version_match") is True else "UNKNOWN",
|
||||
"read_only_api": "PROVEN" if evidence.get("compatibility", {}).get("read_only_api") is True else "UNKNOWN",
|
||||
}
|
||||
|
||||
|
||||
def _public_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Remove database identity rows, which may contain an absolute FDB path."""
|
||||
return {key: value for key, value in metadata.items() if key != "database"}
|
||||
|
||||
|
||||
def run_discovery(path: os.PathLike[str] | str, *, driver: Any) -> dict[str, Any]:
|
||||
"""Run the gated, structured, sanitized discovery workflow."""
|
||||
try:
|
||||
security = _security_gate(path, driver)
|
||||
client = security.get("client", security)
|
||||
except RuntimeError as error:
|
||||
return _blocked(_blocker_category(error))
|
||||
try:
|
||||
connection = connect_authorized_read_only(
|
||||
path, driver, client_library=str(CLIENT_LIBRARY)
|
||||
)
|
||||
except Exception as error:
|
||||
return _blocked(_blocker_category(error), connection_attempts=1)
|
||||
try:
|
||||
try:
|
||||
database_name = get_database_identity(connection)
|
||||
validate_authorized_database_identity(database_name, path)
|
||||
except Exception:
|
||||
return _blocked(
|
||||
"identity_mismatch",
|
||||
connection_attempts=1,
|
||||
other_databases_accessed="UNKNOWN",
|
||||
)
|
||||
try:
|
||||
facts = driver.facts() if callable(getattr(driver, "facts", None)) else {}
|
||||
metadata = collect_metadata(connection)
|
||||
if not _valid_metadata(metadata):
|
||||
return _blocked("discovery_invalid", connection_attempts=1)
|
||||
samples = _collect_samples(connection, metadata)
|
||||
mapping = build_schema_mapping(metadata, samples)
|
||||
if not _valid_mapping(mapping):
|
||||
return _blocked("discovery_invalid", connection_attempts=1)
|
||||
unavailable = set(metadata.get("unavailable", []))
|
||||
optional_catalog_gaps = {"events", "packages", "procedure_parameters"}
|
||||
blocking_errors = [
|
||||
error for error in metadata.get("errors", [])
|
||||
if not (
|
||||
error.get("kind") == "optional_unsupported"
|
||||
or error.get("query") == "monitoring"
|
||||
)
|
||||
]
|
||||
complete = (
|
||||
not blocking_errors
|
||||
and unavailable <= optional_catalog_gaps
|
||||
and (not metadata.get("incomplete") or bool(unavailable) and unavailable <= optional_catalog_gaps)
|
||||
and bool(metadata.get("relations"))
|
||||
and bool(samples)
|
||||
)
|
||||
fingerprint = None
|
||||
classification = SCHEMA_UNKNOWN
|
||||
if complete:
|
||||
evidence = {"metadata": metadata, "samples": samples, "mapping": mapping}
|
||||
fingerprint = build_fingerprint(evidence)
|
||||
if not _valid_fingerprint(fingerprint):
|
||||
return _blocked("discovery_invalid", connection_attempts=1)
|
||||
expected = getattr(driver, "expected_fingerprints", ())
|
||||
classification = classify_schema(
|
||||
metadata,
|
||||
expected,
|
||||
mapping_ambiguous=mapping.get("product", {}).get("classification") == "UNKNOWN",
|
||||
)
|
||||
if classification not in {SCHEMA_SUPPORTED, SCHEMA_CHANGED, SCHEMA_UNKNOWN}:
|
||||
return _blocked("discovery_invalid", connection_attempts=1)
|
||||
return {
|
||||
"status": "CONNECTED" if complete else SCHEMA_UNKNOWN,
|
||||
"driver_version": str(getattr(driver, "version", "unknown")),
|
||||
"architecture": platform.architecture()[0],
|
||||
"engine_version": facts.get("engine_version"),
|
||||
"dialect": facts.get("database_sql_dialect"),
|
||||
"ods": {
|
||||
"major": facts.get("ods_version"),
|
||||
"minor": facts.get("ods_minor_version"),
|
||||
},
|
||||
"charset": facts.get("charset"),
|
||||
"page_size": facts.get("page_size"),
|
||||
"metadata": _public_metadata(metadata),
|
||||
"samples": samples,
|
||||
"schema_mapping": mapping,
|
||||
"fingerprint": fingerprint,
|
||||
"schema_classification": classification,
|
||||
"compatibility": _public_compatibility(security.get("compatibility_evidence")),
|
||||
"application_writes": 0,
|
||||
"other_databases_accessed": 0,
|
||||
"connection_attempts": 1,
|
||||
}
|
||||
except Exception:
|
||||
return _blocked("discovery_failed", connection_attempts=1)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.parse_args(argv)
|
||||
try:
|
||||
if not os.environ.get("FB_USER") or not os.environ.get("FB_PASSWORD"):
|
||||
raise RuntimeError("credentials unavailable")
|
||||
compatibility = _validate_preimport_compatibility()
|
||||
import fdb
|
||||
|
||||
provenance = validate_imported_fdb_provenance(fdb)
|
||||
runtime_compatibility = validate_firebird_runtime(fdb_module=fdb)
|
||||
if runtime_compatibility["status"] != "VALID":
|
||||
raise RuntimeError("Firebird driver/client compatibility is unproven")
|
||||
result = run_discovery(DEFAULT_TARGET, driver=_FdbDriver(fdb))
|
||||
result["compatibility"] = _public_compatibility(runtime_compatibility)
|
||||
except Exception as error:
|
||||
result = {
|
||||
"status": "MISSION_00_BLOCKED",
|
||||
"blocker": _blocker_category(error),
|
||||
"application_writes": 0,
|
||||
"other_databases_accessed": 0,
|
||||
"connection_attempts": 0,
|
||||
}
|
||||
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
|
||||
return 0 if result["status"] == "CONNECTED" else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Run the MVP's first real, guarded Firebird write (test DB only) and verify it.
|
||||
|
||||
Loads a single write_eligible IPI proposal from a DRY_RUN_PLAN artifact,
|
||||
re-reads the live row to check for drift, performs the guarded UPDATE via
|
||||
firebird_write_executor, verifies persistence, and writes an audit log with
|
||||
no secrets. Only ever targets TESTESOFICIAIS.FDB (enforced by the executor).
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", ROOT / "scripts" / "firebird_write_executor.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ[key.strip()] = value.strip()
|
||||
|
||||
|
||||
def pick_proposal(plan: dict, product_code: str | None) -> dict:
|
||||
eligible = [p for p in plan["field_proposals"] if p["field"] == "IPI" and p["write_eligible"]]
|
||||
if not eligible:
|
||||
raise SystemExit("no write_eligible IPI proposal available")
|
||||
if product_code:
|
||||
chosen = next((p for p in eligible if p["product_code"] == product_code), None)
|
||||
if chosen is None:
|
||||
raise SystemExit(f"product {product_code} is not write_eligible")
|
||||
return chosen
|
||||
return eligible[0]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--plan", default=str(ROOT / "docs" / "DRY_RUN_PLAN_V3.json"))
|
||||
parser.add_argument("--product-code", default=None)
|
||||
parser.add_argument("--audit-log", default=str(ROOT / "docs" / "runs" / "first_write_audit.json"))
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv(ROOT / ".env")
|
||||
executor = load_executor()
|
||||
|
||||
plan = json.loads(Path(args.plan).read_text(encoding="utf-8"))
|
||||
proposal = pick_proposal(plan, args.product_code)
|
||||
|
||||
import fdb
|
||||
database_path = os.environ["FIREBIRD_DATABASE_PATH"]
|
||||
executor.assert_authorized_write_path(database_path)
|
||||
|
||||
connection = fdb.connect(
|
||||
database=database_path,
|
||||
user=os.environ["FB_USER"],
|
||||
password=os.environ["FB_PASSWORD"],
|
||||
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
try:
|
||||
cursor = connection.cursor()
|
||||
cursor.execute(
|
||||
"SELECT CODIGO, CLASSIFICACAO_FISCAL, IPI FROM C000025 WHERE CODIGO = ?",
|
||||
(proposal["product_code"],),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row is None:
|
||||
raise SystemExit(f"product {proposal['product_code']} not found")
|
||||
codigo, ncm, live_ipi = row
|
||||
if live_ipi != proposal["current_value"]:
|
||||
raise SystemExit(
|
||||
f"drift before write: plan expected {proposal['current_value']!r}, "
|
||||
f"live is {live_ipi!r} — refusing to write, choose another eligible product"
|
||||
)
|
||||
cursor.close()
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
record = executor.execute_guarded_update(
|
||||
connection,
|
||||
database_path=database_path,
|
||||
proposal=proposal,
|
||||
now=now,
|
||||
)
|
||||
audit = {
|
||||
**record.as_dict(),
|
||||
"ncm": ncm,
|
||||
"database": str(executor.AUTHORIZED_WRITE_DATABASE_PATH),
|
||||
"table": "C000025",
|
||||
}
|
||||
audit_path = Path(args.audit_log)
|
||||
audit_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
existing = json.loads(audit_path.read_text(encoding="utf-8")) if audit_path.exists() else []
|
||||
existing.append(audit)
|
||||
audit_path.write_text(json.dumps(existing, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
print(json.dumps(audit, indent=2, ensure_ascii=False))
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Fiscal Decision Engine V1 runner.
|
||||
|
||||
Reads all C000025 products (read-only), classifies via the Hub, applies
|
||||
central.app.fiscal_decision per domain, writes the matrix/coverage/priorities
|
||||
artifacts, and with --apply executes ONLY AUTO_APPLY_SAFE proposals against
|
||||
TESTESOFICIAIS.FDB (path guard enforced by firebird_write_executor), followed
|
||||
by an independent re-read.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
|
||||
from central.app.fiscal_decision import DECISION_ENGINE_VERSION, ERP_FIELD_MAP, NATURE, decide_product # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import _connect, parse_company_context # noqa: E402
|
||||
|
||||
DOMAIN_FIELDS = ("NCM", "IPI", "EX_TIPI", "CEST", "MVA", "CFOP", "CSOSN", "CST_ICMS", "ORIGEM", "PIS_CST", "COFINS_CST",
|
||||
"PIS_COFINS_RATE", "FCP", "RTC_CCLASSTRIB", "RTC_CBS_RATE", "RTC_CCREDPRES")
|
||||
NEXT_ACTION = {
|
||||
"NCM": "Review LEGACY/INVALID NCMs manually (no unambiguous successor); never auto-change.",
|
||||
"IPI": "Keep auto-apply; NON_NUMERIC/blank TIPI rows need a business decision on how ERP represents NT.",
|
||||
"EX_TIPI": "Propose only when a single concrete EX applies; needs product-description evidence.",
|
||||
"CEST": "Improve description evidence per candidate group (do not lower threshold); manual review queue for candidates.",
|
||||
"MVA": "Confirm CEST first; then identify the ERP destination field for MVA (none proven).",
|
||||
"CFOP": "Requires operation context (direction, destination UF, ST, purpose) — capture it at document level.",
|
||||
"CSOSN": "Requires sale scenario/ST retention context; consider per-product default review for ST-annex NCMs.",
|
||||
"CST_ICMS": "Not used under CRT=1; only relevant if the client changes regime.",
|
||||
"ORIGEM": "Obtain objective product origin (supplier/import flag) before any write; ERP CST prefix is not proof.",
|
||||
"PIS_CST": "Needs regime+incidence rule table (deferred 4.3.x) — measure need from client data first.",
|
||||
"COFINS_CST": "Same as PIS_CST.",
|
||||
"PIS_COFINS_RATE": "Needs source tables (deferred); never assume 0%.",
|
||||
"FCP": "Needs destination UF + operation + BA FCP item list per NCM.",
|
||||
"RTC_CCLASSTRIB": "Needs official NCM->annex/cClassTrib mapping (LC 214 annexes) — not in acquired datasets.",
|
||||
"RTC_CBS_RATE": "Blocked by cClassTrib; apply 0.009 only where the official rule applies.",
|
||||
"RTC_CCREDPRES": "Official conditions not derivable from product data.",
|
||||
}
|
||||
|
||||
|
||||
def load_dotenv(path):
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def http(url, token, payload=None):
|
||||
data = json.dumps(payload, ensure_ascii=True).encode() if payload is not None else None
|
||||
req = urllib.request.Request(url, data=data, headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, method="POST" if payload is not None else "GET")
|
||||
with urllib.request.urlopen(req, timeout=600) as r:
|
||||
return json.load(r)
|
||||
|
||||
|
||||
def read_all(env):
|
||||
conn = _connect(env)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""SELECT CODIGO,PRODUTO,CLASSIFICACAO_FISCAL,CFOP,CST,CSOSN,CEST,IPI,EX_TIPI,ORIGEM,COD_SAIDA_PIS,COD_SAIDA_COFINS,ALIQ_PIS FROM C000025 ORDER BY CODIGO""")
|
||||
names = [c[0].strip() for c in cur.description]
|
||||
rows = [dict(zip(names, r)) for r in cur.fetchall()]
|
||||
cur.execute("SELECT UF, CRT FROM C000004")
|
||||
return rows, parse_company_context(cur.fetchall())
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def build_refs(hub, token, products, company_uf):
|
||||
refs = {}
|
||||
for key, dom in (("csosn", "CSOSN"), ("cst_icms", "CST_ICMS"), ("cst_pis", "CST_PIS"), ("cst_cofins", "CST_COFINS"), ("origem", "ORIGEM_MERCADORIA")):
|
||||
refs[key] = set(http(f"{hub}/v1/reference/fiscal_codes/{dom}", token)["codes"])
|
||||
valid = set()
|
||||
for code in {str(p["CFOP"]).strip() for p in products if p.get("CFOP")}:
|
||||
if http(f"{hub}/v1/reference/cfop/{code}", token)["found"]:
|
||||
valid.add(code)
|
||||
refs["cfop_valid"] = valid
|
||||
refs["fcp_uf"] = {}
|
||||
r = http(f"{hub}/v1/reference/fcp/{company_uf}", token)
|
||||
if r["found"]:
|
||||
refs["fcp_uf"][company_uf] = r
|
||||
refs["cbs_rate_2026"] = http(f"{hub}/v1/reference/rtc_cbs_rate/2026", token).get("rate")
|
||||
status = http(f"{hub}/v1/datasets/status", token)
|
||||
refs["cclasstrib_count"] = sum(d["record_count"] for d in status if d["kind"] == "RTC_CCLASSTRIB" and d["status"] == "ACTIVE")
|
||||
return refs
|
||||
|
||||
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", ROOT / "scripts" / "firebird_write_executor.py")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def summarize(matrix):
|
||||
cov = {}
|
||||
for dom in DOMAIN_FIELDS:
|
||||
rows = [m["fields"][dom] for m in matrix]
|
||||
c = Counter(r["status"] for r in rows)
|
||||
act = Counter(r["action_class"] for r in rows)
|
||||
cov[dom] = {
|
||||
"nature": NATURE[dom],
|
||||
"DATA_AVAILABLE": sum(r["data_available"] for r in rows),
|
||||
"MATCH_FOUND": sum(r["match_found"] for r in rows),
|
||||
"RULE_RESOLVED": c["RULE_RESOLVED"], "RULE_CANDIDATE": c["RULE_CANDIDATE"], "RULE_UNRESOLVED": c["RULE_UNRESOLVED"],
|
||||
"OPERATION_CONTEXT_REQUIRED": c["OPERATION_CONTEXT_REQUIRED"],
|
||||
"CHANGE_REQUIRED": sum(1 for r in rows if r["write_eligible"]),
|
||||
"action_class": dict(act),
|
||||
"reason_codes": dict(Counter(r["reason_code"] for r in rows).most_common(6)),
|
||||
}
|
||||
return cov
|
||||
|
||||
|
||||
def priorities_md(cov):
|
||||
def key(d):
|
||||
c = cov[d]
|
||||
return (-c["action_class"].get("AUTO_APPLY_SAFE", 0), -c["CHANGE_REQUIRED"], -c["RULE_RESOLVED"], -c["RULE_CANDIDATE"], d)
|
||||
lines = ["# NEXT_FISCAL_AUTOMATION_PRIORITIES_V1", "",
|
||||
"Generated by `scripts/run_fiscal_decision_v1.py` from `docs/FISCAL_DECISION_COVERAGE_V1.json` (521 products).",
|
||||
"Ordering is mechanical: AUTO_APPLY_SAFE desc, CHANGE_REQUIRED desc, RESOLVED_PRODUCTS desc, candidates desc.", "",
|
||||
"| FIELD | RESOLVED_PRODUCTS | CHANGE_REQUIRED | AUTO_APPLY_SAFE | CANDIDATES | OP_CONTEXT_REQUIRED | BLOCKER | NEXT_ACTION |", "|---|---|---|---|---|---|---|---|"]
|
||||
for d in sorted(cov, key=key):
|
||||
c = cov[d]
|
||||
blocker = next(iter(c["reason_codes"]), "-")
|
||||
lines.append(f"| {d} | {c['RULE_RESOLVED']} | {c['CHANGE_REQUIRED']} | {c['action_class'].get('AUTO_APPLY_SAFE', 0)} | {c['RULE_CANDIDATE']} | {c['OPERATION_CONTEXT_REQUIRED']} | {blocker} | {NEXT_ACTION[d]} |")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def apply_writes(matrix, products_by_code, executor):
|
||||
"""Only AUTO_APPLY_SAFE + write_eligible; executor enforces path/allowlist/compare-before-write."""
|
||||
import fdb
|
||||
database_path = os.environ["FIREBIRD_DATABASE_PATH"]
|
||||
executor.assert_authorized_write_path(database_path)
|
||||
per_field = defaultdict(lambda: Counter())
|
||||
audit = []
|
||||
conn = fdb.connect(database=database_path, user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"))
|
||||
try:
|
||||
for m in matrix:
|
||||
for dom, rec in m["fields"].items():
|
||||
if not (rec["write_eligible"] and rec["action_class"] == "AUTO_APPLY_SAFE"):
|
||||
continue
|
||||
field = {"IPI": "IPI", "EX_TIPI": "EX_TIPI", "CEST": "CEST"}[dom]
|
||||
per_field[field]["ATTEMPTED"] += 1
|
||||
proposal = {"product_code": m["CODIGO"], "field": field, "current_value": rec["current_value"], "proposed_value": rec["proposed_value"],
|
||||
"status": "CHANGE_REQUIRED", "write_eligible": True, "source_dataset": rec["provenance"]["dataset"], "source_version": rec["provenance"]["version"]}
|
||||
try:
|
||||
r = executor.execute_guarded_update(conn, database_path=database_path, proposal=proposal, now=datetime.now(timezone.utc).isoformat())
|
||||
per_field[field]["SUCCEEDED"] += 1
|
||||
audit.append({**r.as_dict(), "outcome": "SUCCEEDED"})
|
||||
except executor.WriteRefused as e:
|
||||
per_field[field]["FAILED"] += 1
|
||||
audit.append({"product_code": m["CODIGO"], "field": field, "proposed_value": rec["proposed_value"], "outcome": "FAILED", "error": str(e)})
|
||||
finally:
|
||||
conn.close()
|
||||
ro = _connect(os.environ)
|
||||
try:
|
||||
cur = ro.cursor()
|
||||
for a in audit:
|
||||
if a["outcome"] != "SUCCEEDED":
|
||||
continue
|
||||
cur.execute(f"SELECT {a['field']} FROM C000025 WHERE CODIGO = ?", (a["product_code"],))
|
||||
live = cur.fetchone()[0]
|
||||
if str(live).strip() == str(a["proposed_value"]).strip():
|
||||
per_field[a["field"]]["POST_WRITE_VERIFIED"] += 1
|
||||
finally:
|
||||
ro.close()
|
||||
return {f: dict(c) for f, c in per_field.items()}, audit
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--token", required=True)
|
||||
ap.add_argument("--hub-url", default="http://localhost:8000")
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
args = ap.parse_args()
|
||||
load_dotenv(ROOT / ".env")
|
||||
|
||||
products, company = read_all(os.environ)
|
||||
refs = build_refs(args.hub_url, args.token, products, company['uf'])
|
||||
decisions = []
|
||||
for i in range(0, len(products), 100):
|
||||
chunk = products[i:i + 100]
|
||||
decisions += http(f"{args.hub_url}/v1/classify/batch", args.token, {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]})["results"]
|
||||
|
||||
matrix = []
|
||||
for p, d in zip(products, decisions):
|
||||
matrix.append({"CODIGO": p["CODIGO"], "DESCRIPTION": p["PRODUTO"], "NCM": p["CLASSIFICACAO_FISCAL"], "fields": decide_product(p, d, company, refs)})
|
||||
|
||||
cov = summarize(matrix)
|
||||
totals = Counter()
|
||||
for m in matrix:
|
||||
for rec in m["fields"].values():
|
||||
totals[rec["action_class"]] += 1
|
||||
(ROOT / "docs" / "FISCAL_DECISION_MATRIX_V1.json").write_text(json.dumps({
|
||||
"artifact": "FISCAL_DECISION_MATRIX_V1", "engine": DECISION_ENGINE_VERSION, "company_context": company,
|
||||
"total_products": len(matrix), "erp_field_map": ERP_FIELD_MAP, "products": matrix}, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
coverage = {"artifact": "FISCAL_DECISION_COVERAGE_V1", "engine": DECISION_ENGINE_VERSION, "total_products": len(matrix), "company_context": company,
|
||||
"erp_fields_mapped": sum(1 for f in ERP_FIELD_MAP if f["MAPPING"] == "CONFIRMED"),
|
||||
"erp_fields_unconfirmed": [f["ERP_FIELD"] for f in ERP_FIELD_MAP if f["MAPPING"] != "CONFIRMED"],
|
||||
"action_class_totals": dict(totals), "domains": cov, "reference_cbs_rate_2026": refs["cbs_rate_2026"]}
|
||||
write_result = None
|
||||
if args.apply:
|
||||
write_result, audit = apply_writes(matrix, {p["CODIGO"]: p for p in products}, load_executor())
|
||||
coverage["writes"] = write_result
|
||||
(ROOT / "docs" / "runs").mkdir(exist_ok=True)
|
||||
(ROOT / "docs" / "runs" / "fiscal_decision_v1_write_audit.json").write_text(json.dumps(audit, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
(ROOT / "docs" / "FISCAL_DECISION_COVERAGE_V1.json").write_text(json.dumps(coverage, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
(ROOT / "docs" / "NEXT_FISCAL_AUTOMATION_PRIORITIES_V1.md").write_text(priorities_md(cov), encoding="utf-8")
|
||||
print(json.dumps({"action_class_totals": dict(totals), "writes": write_result,
|
||||
"domains": {d: {k: v for k, v in c.items() if k in ("RULE_RESOLVED", "RULE_CANDIDATE", "RULE_UNRESOLVED", "OPERATION_CONTEXT_REQUIRED", "CHANGE_REQUIRED", "MATCH_FOUND")} for d, c in cov.items()}}, indent=1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,262 @@
|
||||
"""MVP first real batch: classify all of C000025, dry-run the IPI plan, then
|
||||
execute every write_eligible IPI proposal against TESTESOFICIAIS.FDB only.
|
||||
|
||||
Phases:
|
||||
1. Read all products from C000025 (read-only).
|
||||
2. Classify each via the running fiscal-hub /v1/classify/batch.
|
||||
3. Build field proposals (dry-run) with central.app.dry_run_planner.
|
||||
4. Execute every write_eligible IPI proposal via firebird_write_executor,
|
||||
sequentially, with compare-before-write and rowcount==1 per product.
|
||||
5. Re-read every written CODIGO independently and verify persistence.
|
||||
|
||||
Never touches CEST/EX_TIPI writes. Never touches any database other than
|
||||
TESTESOFICIAIS.FDB (enforced by firebird_write_executor's path guard).
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "central"))
|
||||
|
||||
from app.dry_run_planner import build_field_proposals, split_by_eligibility # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ[key.strip()] = value.strip()
|
||||
|
||||
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", ROOT / "scripts" / "firebird_write_executor.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def read_all_products(env):
|
||||
import fdb
|
||||
conn = fdb.connect(
|
||||
database=env["FIREBIRD_DATABASE_PATH"],
|
||||
user=env["FB_USER"],
|
||||
password=env["FB_PASSWORD"],
|
||||
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO,
|
||||
)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT CODIGO, PRODUTO, CLASSIFICACAO_FISCAL, IPI, EX_TIPI, CEST FROM C000025 ORDER BY CODIGO")
|
||||
names = [c[0].strip() for c in cur.description]
|
||||
return [dict(zip(names, row)) for row in cur.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def classify_batch(hub_url, token, products, chunk_size=100):
|
||||
results = []
|
||||
for start in range(0, len(products), chunk_size):
|
||||
chunk = products[start:start + chunk_size]
|
||||
payload = {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]}
|
||||
request = urllib.request.Request(
|
||||
f"{hub_url}/v1/classify/batch",
|
||||
data=json.dumps(payload, ensure_ascii=True).encode(),
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
results.extend(json.load(response)["results"])
|
||||
return results
|
||||
|
||||
|
||||
def classify_ipi_bucket(decision):
|
||||
ipi_status = decision.get("ipi_status")
|
||||
if ipi_status == "LEGACY_NCM":
|
||||
return None # tracked under NCM_LEGACY, not an IPI bucket
|
||||
if ipi_status == "NO_TIPI_MAPPING_FOUND":
|
||||
return "NO_MATCH"
|
||||
if ipi_status == "IPI_CANDIDATES":
|
||||
return "AMBIGUOUS"
|
||||
if ipi_status == "IPI_RESOLVED":
|
||||
rate_kind = (decision.get("classification", {}).get("ipi") or {}).get("rate_kind")
|
||||
return {"NUMERIC": "NUMERIC", "NON_NUMERIC": "NON_NUMERIC", "BLANK": "SOURCE_BLANK"}.get(rate_kind, "NO_MATCH")
|
||||
return "NO_MATCH"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--hub-url", default="http://localhost:8000")
|
||||
parser.add_argument("--token", required=True)
|
||||
parser.add_argument("--dry-run-output", default=str(ROOT / "docs" / "FULL_DATABASE_DRY_RUN_V1.json"))
|
||||
parser.add_argument("--audit-output", default=str(ROOT / "docs" / "runs" / "full_database_ipi_write_audit_v1.json"))
|
||||
parser.add_argument("--no-write", action="store_true", help="Only produce the dry-run artifact; skip Phase 4/5.")
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv(ROOT / ".env")
|
||||
executor = load_executor()
|
||||
database_path = os.environ["FIREBIRD_DATABASE_PATH"]
|
||||
executor.assert_authorized_write_path(database_path)
|
||||
|
||||
products = read_all_products(os.environ)
|
||||
total = len(products)
|
||||
|
||||
ncm_missing = sum(1 for p in products if not (p["CLASSIFICACAO_FISCAL"] or "").strip())
|
||||
|
||||
decisions = classify_batch(args.hub_url, args.token, products)
|
||||
|
||||
ncm_status_counts = Counter(d.get("status") for d in decisions)
|
||||
ncm_valid = ncm_status_counts.get("EXACT_NCM", 0)
|
||||
ncm_legacy = ncm_status_counts.get("LEGACY_NCM", 0)
|
||||
# invalid = has a CLASSIFICACAO_FISCAL but it isn't a recognized/legacy NCM
|
||||
ncm_invalid = sum(
|
||||
1 for p, d in zip(products, decisions)
|
||||
if (p["CLASSIFICACAO_FISCAL"] or "").strip() and d.get("status") not in ("EXACT_NCM", "LEGACY_NCM")
|
||||
)
|
||||
|
||||
ipi_buckets = Counter()
|
||||
field_proposals = []
|
||||
for product, decision in zip(products, decisions):
|
||||
code = product["CODIGO"]
|
||||
current = {"ipi": product.get("IPI"), "cest": product.get("CEST"), "ex_tipi": product.get("EX_TIPI")}
|
||||
field_proposals.extend(build_field_proposals(code, decision, current))
|
||||
bucket = classify_ipi_bucket(decision)
|
||||
if bucket:
|
||||
ipi_buckets[bucket] += 1
|
||||
|
||||
ipi_rows = [p for p in field_proposals if p["field"] == "IPI"]
|
||||
cest_rows = [p for p in field_proposals if p["field"] == "CEST"]
|
||||
split = split_by_eligibility(ipi_rows)
|
||||
|
||||
ipi_already_correct = sum(1 for p in ipi_rows if p["status"] == "SAME_VALUE")
|
||||
ipi_change_required = sum(1 for p in ipi_rows if p["status"] == "CHANGE_REQUIRED")
|
||||
ipi_write_eligible_rows = split["eligible"]
|
||||
|
||||
cest_db_match = sum(1 for p in cest_rows if p["status"] not in ("UNRESOLVED",) or p["reason_code"] != "NO_CEST_MAPPING_FOUND_FOR_NCM")
|
||||
cest_confirmed = sum(1 for p in cest_rows if p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED", "CURRENT_UNKNOWN"))
|
||||
|
||||
dry_run_artifact = {
|
||||
"artifact": "FULL_DATABASE_DRY_RUN_V1",
|
||||
"total_products": total,
|
||||
"ncm_valid": ncm_valid,
|
||||
"ncm_legacy": ncm_legacy,
|
||||
"ncm_invalid": ncm_invalid,
|
||||
"ncm_missing": ncm_missing,
|
||||
"ipi_buckets": dict(ipi_buckets),
|
||||
"ipi_already_correct": ipi_already_correct,
|
||||
"ipi_change_required": ipi_change_required,
|
||||
"ipi_write_eligible": len(ipi_write_eligible_rows),
|
||||
"cest_db_match_products": cest_db_match,
|
||||
"cest_confirmed": cest_confirmed,
|
||||
"field_proposals": field_proposals,
|
||||
}
|
||||
Path(args.dry_run_output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.dry_run_output).write_text(json.dumps(dry_run_artifact, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
summary = {
|
||||
"total_products": total,
|
||||
"ncm_valid": ncm_valid,
|
||||
"ncm_legacy": ncm_legacy,
|
||||
"ncm_invalid": ncm_invalid,
|
||||
"ncm_missing": ncm_missing,
|
||||
"ipi_buckets": dict(ipi_buckets),
|
||||
"ipi_already_correct": ipi_already_correct,
|
||||
"ipi_change_required": ipi_change_required,
|
||||
"ipi_write_eligible": len(ipi_write_eligible_rows),
|
||||
}
|
||||
|
||||
if args.no_write or not ipi_write_eligible_rows:
|
||||
summary["writes_attempted"] = 0
|
||||
print(json.dumps(summary, indent=2))
|
||||
return
|
||||
|
||||
import fdb
|
||||
connection = fdb.connect(
|
||||
database=database_path,
|
||||
user=os.environ["FB_USER"],
|
||||
password=os.environ["FB_PASSWORD"],
|
||||
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
audit_records = []
|
||||
succeeded = failed = 0
|
||||
try:
|
||||
for proposal in ipi_write_eligible_rows:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
try:
|
||||
record = executor.execute_guarded_update(
|
||||
connection,
|
||||
database_path=database_path,
|
||||
proposal=proposal,
|
||||
now=now,
|
||||
)
|
||||
audit_records.append({**record.as_dict(), "outcome": "SUCCEEDED"})
|
||||
succeeded += 1
|
||||
except executor.WriteRefused as error:
|
||||
audit_records.append({
|
||||
"product_code": proposal["product_code"],
|
||||
"field": proposal["field"],
|
||||
"before_value": proposal["current_value"],
|
||||
"proposed_value": proposal["proposed_value"],
|
||||
"after_value": None,
|
||||
"rowcount": 0,
|
||||
"source_dataset": proposal.get("source_dataset"),
|
||||
"source_version": proposal.get("source_version"),
|
||||
"timestamp": now,
|
||||
"outcome": "FAILED",
|
||||
"error": str(error),
|
||||
})
|
||||
failed += 1
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
Path(args.audit_output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.audit_output).write_text(json.dumps(audit_records, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
# Phase 5: independent re-read + verification of every written CODIGO.
|
||||
written_codes = [r["product_code"] for r in audit_records if r["outcome"] == "SUCCEEDED"]
|
||||
verify_conn = fdb.connect(
|
||||
database=database_path,
|
||||
user=os.environ["FB_USER"],
|
||||
password=os.environ["FB_PASSWORD"],
|
||||
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO,
|
||||
)
|
||||
verified = 0
|
||||
mismatches = []
|
||||
try:
|
||||
cur = verify_conn.cursor()
|
||||
for record in audit_records:
|
||||
if record["outcome"] != "SUCCEEDED":
|
||||
continue
|
||||
cur.execute("SELECT IPI FROM C000025 WHERE CODIGO = ?", (record["product_code"],))
|
||||
row = cur.fetchone()
|
||||
live_value = row[0] if row else None
|
||||
if live_value == record["proposed_value"]:
|
||||
verified += 1
|
||||
else:
|
||||
mismatches.append({"product_code": record["product_code"], "expected": record["proposed_value"], "found": live_value})
|
||||
finally:
|
||||
verify_conn.close()
|
||||
|
||||
summary.update({
|
||||
"writes_attempted": len(ipi_write_eligible_rows),
|
||||
"writes_succeeded": succeeded,
|
||||
"writes_failed": failed,
|
||||
"post_write_verified": verified,
|
||||
"post_write_mismatches": mismatches,
|
||||
"written_codes": written_codes,
|
||||
})
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,155 @@
|
||||
"""ICMS Context Engine V1 runner (READ-ONLY): ST applicability, ST withholding, product role, CSOSN V2 dry-run.
|
||||
Outputs: docs/ICMS_PRODUCT_CONTEXT_V1.json, ICMS_CONTEXT_COVERAGE_V1.json, CSOSN_DRY_RUN_V2.json,
|
||||
FISCAL_CONTEXT_REQUIREMENTS_V3.json, NEXT_FISCAL_FIELD_IMPLEMENTATION_V3.md. Never writes to Firebird.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
import run_fiscal_decision_v1 as base # noqa: E402
|
||||
from central.app.icms_context import ICMS_CONTEXT_ENGINE_VERSION, csosn_v2, product_role, st_conflict_class, st_status, withheld_status # noqa: E402
|
||||
from central.app.product_defaults import validate_against_history # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import _connect, parse_company_context # noqa: E402
|
||||
|
||||
s = lambda v: None if v is None else str(v).strip()
|
||||
dg = lambda v: "".join(c for c in str(v or "") if c.isdigit())
|
||||
|
||||
|
||||
def main():
|
||||
token = sys.argv[1]; hub = "http://localhost:8000"
|
||||
base.load_dotenv(ROOT / ".env")
|
||||
cur = _connect(os.environ).cursor()
|
||||
cur.execute("select CODIGO, PRODUTO, CLASSIFICACAO_FISCAL, CEST, CSOSN, CST, CFOP, TIPO, PROD_COZINHA, PRODUTO_SERVICO from C000025 order by CODIGO")
|
||||
products = [dict(zip(("CODIGO", "PRODUTO", "CLASSIFICACAO_FISCAL", "CEST", "CSOSN", "CST", "CFOP", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO"), r)) for r in cur.fetchall()]
|
||||
cur.execute("select ICMS_CSOSN from C000004"); company_default = s(cur.fetchone()[0])
|
||||
cur.execute("select UF, CRT from C000004"); company = parse_company_context(cur.fetchall()); uf, crt = company["uf"], company["crt"]
|
||||
cur.execute("select CODRECEITA from RECEITA_INGREDIENTES where CODRECEITA is not null"); recipes = {str(r[0]).zfill(6) for r in cur.fetchall()}
|
||||
cur.execute("select distinct CODPRODUTO from C000032 where TIPO = 'COMPOSICAO'"); composed = {s(r[0]) for r in cur.fetchall()}
|
||||
cur.execute("select CODPRODUTO, ICMS_RETIDO, ICMS_VALORRETIDO, SUB_VALOR, CFOP from C000088"); purchases = defaultdict(list)
|
||||
for cod, ret, vr, sv, cf in cur.fetchall():
|
||||
purchases[s(cod)].append({"icms_retido": s(ret), "valor_retido": vr, "sub_valor": sv, "cfop": s(cf)})
|
||||
cur.execute("select CODPRODUTO, ULTIMA_ENTRADA, NOTA_FISCAL from C000100"); stock_entry = {s(r[0]): (r[1] is not None or (s(r[2]) not in (None, "", "000000"))) for r in cur.fetchall()}
|
||||
cur.execute("select CODPRODUTO, CST, ICMS_SUB from C000062"); nf_out = defaultdict(list)
|
||||
for cod, cst, sub in cur.fetchall(): nf_out[s(cod)].append((s(cst), float(sub or 0)))
|
||||
cur.connection.close()
|
||||
|
||||
sh = defaultdict(list)
|
||||
out = subprocess.run(["docker", "exec", "projeto-postgres-1", "psql", "-U", "gti", "-d", "gti_fiscal", "-At", "-F|", "-c",
|
||||
"select e.item, n.ncm_sh from ba_st_mva_entries e join ba_st_mva_ncm_sh n on n.entry_id=e.id join datasets d on d.id=e.dataset_id where d.status='ACTIVE'"], capture_output=True, text=True, check=True).stdout
|
||||
for line in out.splitlines():
|
||||
it, n = line.split("|"); sh[it].append(dg(n))
|
||||
|
||||
refs = base.build_refs(hub, token, products, uf)
|
||||
decisions = []
|
||||
for i in range(0, len(products), 100):
|
||||
chunk = products[i:i + 100]
|
||||
decisions += base.http(f"{hub}/v1/classify/batch", token, {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]})["results"]
|
||||
|
||||
trusted = json.loads((ROOT / "docs" / "runs" / "trusted_historical_items.json").read_text(encoding="utf-8"))
|
||||
hist = defaultdict(list)
|
||||
for t in trusted:
|
||||
hist[s(t["cod"])].append(s(t.get("csosn")) or "")
|
||||
hist_cfop = defaultdict(list)
|
||||
for t in trusted:
|
||||
hist_cfop[s(t["cod"])].append(s(t.get("cfop")) or "")
|
||||
|
||||
rows = []
|
||||
for p, d in zip(products, decisions):
|
||||
code = s(p["CODIGO"]); cls = d.get("classification") or {}
|
||||
ncm_status = d.get("status"); mva = d.get("ba_st_mva_status"); ceststat = d.get("cest_status")
|
||||
ba = cls.get("ba_st_mva") or {}
|
||||
ba_item, ba_cest = ba.get("item"), ba.get("cest")
|
||||
ncm8 = dg(p["CLASSIFICACAO_FISCAL"])
|
||||
sh_len = max((len(x) for x in sh.get(ba_item, []) if x and ncm8.startswith(x)), default=None) if ba_item else None
|
||||
st, st_reason = st_status(ncm_status, mva, ceststat, p["CEST"], ba_cest, sh_len)
|
||||
wh, wh_reason = withheld_status(purchases.get(code, []))
|
||||
purchase_ev = bool(purchases.get(code)) or stock_entry.get(code, False)
|
||||
role, role_ev = product_role(p["TIPO"], p["PROD_COZINHA"] == 1, code in recipes, code in composed, bool(s(p["PRODUTO_SERVICO"])), purchase_ev)
|
||||
cs = csosn_v2(p["CSOSN"], company_default, ncm_status, st, wh, role, refs["csosn"])
|
||||
h = hist.get(code, [])
|
||||
hv = validate_against_history(cs["proposed_csosn"], [x for x in h if x])
|
||||
share = (sum(1 for x in h if x == cs["current_value"]) / len(h)) if h else None
|
||||
rows.append({
|
||||
"CODIGO": code, "DESCRIPTION": p["PRODUTO"], "NCM": p["CLASSIFICACAO_FISCAL"], "CEST": s(p["CEST"]) or None,
|
||||
"NCM_STATUS": ncm_status, "ST_STATUS": st, "ST_REASON": st_reason, "ST_WITHHELD_STATUS": wh, "ST_WITHHELD_REASON": wh_reason,
|
||||
"PRODUCT_ROLE": role, "PRODUCT_ROLE_EVIDENCE": role_ev, "CURRENT_CSOSN": cs["current_value"], "COMPANY_CRT": crt, "COMPANY_DEFAULT_CSOSN": company_default,
|
||||
"HISTORICAL_CSOSN": dict(Counter(x for x in h if x)), "HISTORICAL_CONSISTENCY": share,
|
||||
"ERP_EMITTED_ST_TREATMENT_NF_OUT": {"cst060_items": sum(1 for c, _ in nf_out.get(code, []) if c == "060"), "icms_sub_items": sum(1 for _, v in nf_out.get(code, []) if v > 0)},
|
||||
"CSOSN_V2": cs, "CSOSN_HISTORY_VALIDATION": hv,
|
||||
"_conf": {"cest_status": ceststat, "mva": mva, "sh_len": sh_len, "erp_cest": p["CEST"], "ba_cest": ba_cest, "cfop": s(p["CFOP"]),
|
||||
"hist_cfop_current": bool(hist_cfop.get(code)) and all(x == s(p["CFOP"]) for x in hist_cfop[code])},
|
||||
})
|
||||
|
||||
# ST conflict audit (the CFOP-conflict population: ST-subject with non-ST CFOP, or 5405 without ST)
|
||||
conflicts = []
|
||||
for r in rows:
|
||||
c = r["_conf"]
|
||||
st_subject = c["mva"] == "MVA_RESOLVED"
|
||||
cfop_conflict = r["NCM_STATUS"] == "EXACT_NCM" and ((st_subject and c["cfop"] == "5102") or (c["mva"] == "NO_MVA_MAPPING_FOUND" and c["cfop"] == "5405"))
|
||||
if cfop_conflict:
|
||||
primary, flags = st_conflict_class(c["cest_status"], c["mva"], c["sh_len"], c["erp_cest"], c["ba_cest"], c["cfop"], c["hist_cfop_current"], c["mva"] == "NO_MVA_MAPPING_FOUND" and c["cfop"] == "5405")
|
||||
conflicts.append({"CODIGO": r["CODIGO"], "DESCRIPTION": r["DESCRIPTION"], "NCM": r["NCM"], "erp_cfop": c["cfop"], "primary": primary, "flags": flags, "st_status": r["ST_STATUS"]})
|
||||
for r in rows:
|
||||
r.pop("_conf")
|
||||
|
||||
C = lambda key: Counter(r[key] for r in rows)
|
||||
st_c, wh_c, role_c = C("ST_STATUS"), C("ST_WITHHELD_STATUS"), C("PRODUCT_ROLE")
|
||||
cs_c = Counter(r["CSOSN_V2"]["status"] for r in rows)
|
||||
act_c = Counter(r["CSOSN_V2"]["action_class"] for r in rows)
|
||||
hv_c = Counter(r["CSOSN_HISTORY_VALIDATION"] for r in rows if r["CSOSN_V2"]["proposed_csosn"] is not None)
|
||||
ncm_blocked = sum(1 for r in rows if r["NCM_STATUS"] != "EXACT_NCM")
|
||||
ncm_only_blocked = ncm_blocked # every NCM-invalid product is UNRESOLVED solely because of NCM (no other context is evaluated)
|
||||
already = sum(1 for r in rows if r["CSOSN_V2"]["status"] == "CSOSN_RESOLVED" and r["CSOSN_V2"]["proposed_csosn"] == r["CURRENT_CSOSN"])
|
||||
change = sum(1 for r in rows if r["CSOSN_V2"]["status"] == "CSOSN_RESOLVED" and r["CSOSN_V2"]["proposed_csosn"] != r["CURRENT_CSOSN"])
|
||||
auto_safe = 0 # none may be write-eligible in this mission; readiness below
|
||||
ready = sum(1 for r in rows if r["CSOSN_V2"]["status"] == "CSOSN_RESOLVED" and r["CSOSN_HISTORY_VALIDATION"] == "MATCH")
|
||||
conf_primary = Counter(c["primary"] for c in conflicts)
|
||||
conf_flags = Counter(f for c in conflicts for f in c["flags"])
|
||||
|
||||
cov = {"artifact": "ICMS_CONTEXT_COVERAGE_V1", "firebird_writes": 0, "TOTAL_PRODUCTS": len(rows), "engine": ICMS_CONTEXT_ENGINE_VERSION,
|
||||
"ST": {k: st_c.get(k, 0) for k in ("ST_CONFIRMED", "ST_NOT_APPLICABLE", "ST_CANDIDATE", "ST_UNKNOWN")},
|
||||
"ST_reasons": dict(C("ST_REASON")),
|
||||
"ST_WITHHELD": {"ST_ALREADY_WITHHELD_CONFIRMED": wh_c.get("ST_ALREADY_WITHHELD_CONFIRMED", 0), "ST_ALREADY_WITHHELD_NOT_CONFIRMED": wh_c.get("ST_ALREADY_WITHHELD_NOT_CONFIRMED", 0), "ST_ALREADY_WITHHELD_UNKNOWN": wh_c.get("ST_WITHHOLDING_UNKNOWN", 0)},
|
||||
"PRODUCT_ROLE": {k: role_c.get(k, 0) for k in ("RESALE", "OWN_PRODUCTION", "INPUT", "SERVICE", "UNKNOWN")},
|
||||
"CSOSN": {"RESOLVED": cs_c.get("CSOSN_RESOLVED", 0), "ALREADY_CORRECT": already, "CHANGE_REQUIRED": change, "CANDIDATES": cs_c.get("CSOSN_CANDIDATE", 0),
|
||||
"CONTEXT_REQUIRED": cs_c.get("CSOSN_CONTEXT_REQUIRED", 0), "CONFLICT": cs_c.get("CSOSN_CONFLICT", 0), "UNRESOLVED": cs_c.get("CSOSN_UNRESOLVED", 0),
|
||||
"AUTO_APPLY_SAFE": auto_safe, "AUTO_APPLY_READY_IF_ENABLED": ready, "action_class": dict(act_c),
|
||||
"reasons": dict(Counter(r["CSOSN_V2"]["reason_code"] for r in rows).most_common(10))},
|
||||
"CSOSN_HISTORY_of_proposals": {"MATCH": hv_c.get("MATCH", 0), "MISMATCH": hv_c.get("MISMATCH", 0), "UNVALIDATED": hv_c.get("UNVALIDATED", 0) + sum(1 for r in rows if r["CSOSN_V2"]["proposed_csosn"] is None)},
|
||||
"CSOSN_HISTORY_of_proposals_note": "MATCH/MISMATCH/UNVALIDATED counted on products WITH a proposed value; UNVALIDATED additionally counts products with no proposal (nothing to validate).",
|
||||
"NCM_BLOCKED_PRODUCTS": ncm_blocked, "NCM_BLOCKED_ONLY_BY_NCM": ncm_only_blocked,
|
||||
"ST_CONFLICT_AUDIT": {"population": len(conflicts), "primary_class": dict(conf_primary), "any_flag": dict(conf_flags)}}
|
||||
(ROOT / "docs" / "ICMS_CONTEXT_COVERAGE_V1.json").write_text(json.dumps(cov, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
(ROOT / "docs" / "ICMS_PRODUCT_CONTEXT_V1.json").write_text(json.dumps({"artifact": "ICMS_PRODUCT_CONTEXT_V1", "engine": ICMS_CONTEXT_ENGINE_VERSION, "company": {"crt": crt, "uf": uf, "default_csosn": company_default},
|
||||
"products": rows, "st_conflict_audit": conflicts}, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
dry = {"artifact": "CSOSN_DRY_RUN_V2", "firebird_writes": 0, "CSOSN_AUTO_APPLY_SAFE": auto_safe, "CSOSN_ALREADY_CORRECT": already, "CSOSN_CHANGE_REQUIRED": change,
|
||||
"would_change": [], "states": dict(cs_c), "note": "No CSOSN proposal is write-eligible; readiness requires proven ST, withholding, role and ICMS treatment."}
|
||||
(ROOT / "docs" / "CSOSN_DRY_RUN_V2.json").write_text(json.dumps(dry, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
|
||||
n = lambda key, val: sum(1 for r in rows if r[key] == val)
|
||||
req = [
|
||||
{"INPUT": "valid_ncm", "SOURCE_EXPECTED": "product master", "AVAILABLE": "PARTIAL", "AFFECTED_PRODUCTS": ncm_blocked, "AFFECTED_FIELD": "CSOSN", "NEXT_ACTION": "Manual NCM review (3 legacy + 23 invalid); not auto-corrected."},
|
||||
{"INPUT": "st_confirmed_per_product (CEST corroboration)", "SOURCE_EXPECTED": "ERP CEST field filled and equal to the BA Anexo 1 item CEST", "AVAILABLE": "PARTIAL", "AFFECTED_PRODUCTS": st_c.get("ST_CANDIDATE", 0) + st_c.get("ST_UNKNOWN", 0) - ncm_blocked, "AFFECTED_FIELD": "CSOSN", "NEXT_ACTION": "Fill/confirm CEST for ST candidates (description review) — 490 of 521 ERP CEST values are blank."},
|
||||
{"INPUT": "st_withheld_by_supplier (purchase evidence)", "SOURCE_EXPECTED": "purchase items with ICMS_RETIDO/ST values (C000088)", "AVAILABLE": "NO (12 purchase items, all ICMS_RETIDO='N', ST values 0)", "AFFECTED_PRODUCTS": st_c.get("ST_CONFIRMED", 0) + st_c.get("ST_CANDIDATE", 0), "AFFECTED_FIELD": "CSOSN(500 vs other)", "NEXT_ACTION": "Import supplier NF-e XML (ICMS ST/CSOSN 500 on entrada) or obtain vendor confirmation of how retention is recorded."},
|
||||
{"INPUT": "product_role (resale/own production)", "SOURCE_EXPECTED": "C000025.TIPO/PROD_COZINHA + recipe + purchase docs", "AVAILABLE": "PARTIAL", "AFFECTED_PRODUCTS": role_c.get("UNKNOWN", 0), "AFFECTED_FIELD": "CSOSN,CFOP", "NEXT_ACTION": "TIPO_ITEM/PRODUTO_SERVICO are unused; only 7 INPUT and ~13 own-production products are objectively evidenced."},
|
||||
{"INPUT": "product_icms_treatment (taxed vs exempt/not taxed)", "SOURCE_EXPECTED": "official per-NCM ICMS treatment source (none acquired)", "AVAILABLE": False, "AFFECTED_PRODUCTS": st_c.get("ST_NOT_APPLICABLE", 0), "AFFECTED_FIELD": "CSOSN(102 vs 400/300/103)", "NEXT_ACTION": "Decide with accountant/vendor: ERP CST=040 with CSOSN=102 is the emitted practice; needs a documented rule."},
|
||||
]
|
||||
(ROOT / "docs" / "FISCAL_CONTEXT_REQUIREMENTS_V3.json").write_text(json.dumps({"artifact": "FISCAL_CONTEXT_REQUIREMENTS_V3", "requirements": req}, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
md = ["# NEXT_FISCAL_FIELD_IMPLEMENTATION_V3", "",
|
||||
"Mechanical ordering: mapping confirmed, rule available, context available, products affected, history validation, auto-apply possible.", "",
|
||||
"| FIELD | ERP_MAPPING | CONTEXT_AVAILABLE | RULE_AVAILABLE | POTENTIAL_PRODUCTS | HISTORICAL_VALIDATION | NEXT_ACTION |", "|---|---|---|---|---|---|---|",
|
||||
f"| CSOSN | CONFIRMED (HIGH) | NO (retention + ICMS treatment missing) | PARTIAL (ST→500 rule needs withholding proof) | {cs_c.get('CSOSN_CANDIDATE',0)+cs_c.get('CSOSN_CONTEXT_REQUIRED',0)+cs_c.get('CSOSN_CONFLICT',0)} | trusted 1146 items | Acquire supplier ST retention evidence; get accountant decision on 102 vs 400 |",
|
||||
"| CEST | CONFIRMED (HIGH) | PARTIAL | PARTIAL | " + str(st_c.get("ST_CANDIDATE", 0)) + " | weak | CEST fill/confirm drives ST_CONFIRMED |",
|
||||
"| CFOP | CONFIRMED (HIGH) | PARTIAL | YES (ST family) | 29 conflicts | trusted history | Depends on ST_CONFIRMED + retention |",
|
||||
"| IPI | CONFIRMED (HIGH) | YES | YES | 107 | n/a | Maintained |", "",
|
||||
"Blocking inputs with counts: `docs/FISCAL_CONTEXT_REQUIREMENTS_V3.json`."]
|
||||
(ROOT / "docs" / "NEXT_FISCAL_FIELD_IMPLEMENTATION_V3.md").write_text("\n".join(md) + "\n", encoding="utf-8")
|
||||
print(json.dumps({k: cov[k] for k in ("ST", "ST_reasons", "ST_WITHHELD", "PRODUCT_ROLE", "CSOSN", "CSOSN_HISTORY_of_proposals", "NCM_BLOCKED_PRODUCTS", "ST_CONFLICT_AUDIT")}, ensure_ascii=True, indent=1))
|
||||
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Fiscal Engine V1 coverage over every database in ./Banco — READ-ONLY (no Firebird writes, no new rules).
|
||||
|
||||
python scripts/run_multi_db_coverage.py <AGENT_TOKEN> [DIR]
|
||||
|
||||
Reuses the existing pure engines: fiscal_decision (NCM/IPI/EX/CEST/MVA), product_defaults (CFOP), icms_context (ST, role,
|
||||
withholding, CSOSN V2). Only GUARDS are added for company context: the BA Anexo 1 dataset is applied to UF=BA companies only, and
|
||||
CSOSN logic only under CRT=1; otherwise the engines are told the context is unavailable (no fiscal rule invented).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
import run_fiscal_decision_v1 as base # noqa: E402
|
||||
import fdb # noqa: E402
|
||||
from central.app.fiscal_decision import decide_product # noqa: E402
|
||||
from central.app.icms_context import csosn_v2, product_role, st_status, withheld_status # noqa: E402
|
||||
from central.app.product_defaults import cfop_default # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import parse_company_context # noqa: E402
|
||||
|
||||
HUB = "http://localhost:8000"
|
||||
s = lambda v: None if v is None else str(v).strip() # noqa: E731
|
||||
dg = lambda v: "".join(c for c in str(v or "") if c.isdigit()) # noqa: E731
|
||||
|
||||
|
||||
def connect(path):
|
||||
return fdb.connect(database=str(path), user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
|
||||
|
||||
def table_cols(cur):
|
||||
cur.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS")
|
||||
m = defaultdict(set)
|
||||
for t, f in cur.fetchall():
|
||||
m[t].add(f)
|
||||
return m
|
||||
|
||||
|
||||
def classify_all(token, products):
|
||||
"""Classify every distinct (ncm, description) with the production Classifier inside the hub container (read-only on PostgreSQL,
|
||||
memoized dataset lookups; identical code path to /v1/classify, verified against the HTTP API in tests/verification)."""
|
||||
import subprocess
|
||||
import tempfile
|
||||
pairs = sorted({(s(p["CLASSIFICACAO_FISCAL"]) or "", s(p["PRODUTO"]) or "") for p in products})
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fin, fout = Path(tmp) / "in.json", Path(tmp) / "out.json"
|
||||
fin.write_text(json.dumps(pairs, ensure_ascii=False), encoding="utf-8")
|
||||
subprocess.run(["docker", "cp", str(ROOT / "scripts" / "bulk_classify_in_container.py"), "projeto-fiscal-hub-1:/tmp/bulk.py"], check=True)
|
||||
subprocess.run(["docker", "cp", str(fin), "projeto-fiscal-hub-1:/tmp/in.json"], check=True)
|
||||
subprocess.run(["docker", "exec", "projeto-fiscal-hub-1", "sh", "-c", "cd /app && python /tmp/bulk.py /tmp/in.json /tmp/out.json"], check=True, capture_output=True)
|
||||
subprocess.run(["docker", "cp", "projeto-fiscal-hub-1:/tmp/out.json", str(fout)], check=True)
|
||||
results = json.loads(fout.read_text(encoding="utf-8"))
|
||||
print(f" classify: {len(pairs)} distinct (ncm, description) pairs for {len(products)} products", flush=True)
|
||||
return dict(zip(pairs, results))
|
||||
|
||||
|
||||
def analyze(path, token):
|
||||
conn = connect(path); cur = conn.cursor()
|
||||
tc = table_cols(cur)
|
||||
have = lambda t, *cols: t in tc and all(c in tc[t] for c in cols) # noqa: E731
|
||||
c25 = tc["C000025"]
|
||||
cols = [c for c in ("CODIGO", "PRODUTO", "CLASSIFICACAO_FISCAL", "CFOP", "CSOSN", "CST", "CEST", "IPI", "EX_TIPI", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA") if c in c25]
|
||||
cur.execute(f"SELECT {','.join(cols)} FROM C000025")
|
||||
products = [dict(zip(cols, r)) for r in cur.fetchall()]
|
||||
for p in products:
|
||||
for c in ("CFOP", "CSOSN", "CST", "CEST", "IPI", "EX_TIPI", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA", "PRODUTO", "CLASSIFICACAO_FISCAL"):
|
||||
p.setdefault(c, None)
|
||||
# company (fail closed; guards below)
|
||||
try:
|
||||
cur.execute("SELECT UF, CRT FROM C000004"); company = {"status": "OK", **parse_company_context(cur.fetchall())}
|
||||
except Exception as e: # noqa: BLE001
|
||||
company = {"status": "UNAVAILABLE", "error": str(e)[:120], "uf": None, "crt": None}
|
||||
company_default = None
|
||||
if have("C000004", "ICMS_CSOSN"):
|
||||
cur.execute("SELECT FIRST 1 ICMS_CSOSN FROM C000004 ORDER BY CODIGO"); company_default = s(cur.fetchone()[0])
|
||||
# context evidence (only where the tables exist in this database)
|
||||
recipes = set(); composed = set(); purchases = defaultdict(list); stock_entry = {}
|
||||
if have("RECEITA_INGREDIENTES", "CODRECEITA"):
|
||||
cur.execute("SELECT CODRECEITA FROM RECEITA_INGREDIENTES WHERE CODRECEITA IS NOT NULL"); recipes = {str(r[0]).zfill(6) for r in cur.fetchall()}
|
||||
if have("C000032", "CODPRODUTO", "TIPO"):
|
||||
cur.execute("SELECT DISTINCT CODPRODUTO FROM C000032 WHERE TIPO = 'COMPOSICAO'"); composed = {s(r[0]) for r in cur.fetchall()}
|
||||
if have("C000088", "CODPRODUTO", "ICMS_RETIDO", "ICMS_VALORRETIDO", "SUB_VALOR"):
|
||||
cur.execute("SELECT CODPRODUTO, ICMS_RETIDO, ICMS_VALORRETIDO, SUB_VALOR FROM C000088")
|
||||
for cod, ret, vr, sv in cur.fetchall():
|
||||
purchases[s(cod)].append({"icms_retido": s(ret), "valor_retido": vr, "sub_valor": sv})
|
||||
if have("C000100", "CODPRODUTO", "ULTIMA_ENTRADA", "NOTA_FISCAL"):
|
||||
cur.execute("SELECT CODPRODUTO, ULTIMA_ENTRADA, NOTA_FISCAL FROM C000100"); stock_entry = {s(r[0]): (r[1] is not None or s(r[2]) not in (None, "", "000000")) for r in cur.fetchall()}
|
||||
conn.close()
|
||||
|
||||
refs = base.build_refs(HUB, token, products, company["uf"] or "BA")
|
||||
decisions = classify_all(token, products)
|
||||
from gti_fiscal_agent.coverage import compute_coverage
|
||||
evidence = {"recipes": recipes, "composed": composed, "purchases": dict(purchases), "stock_entry": stock_entry}
|
||||
report = compute_coverage(products, [decisions[(s(p["CLASSIFICACAO_FISCAL"]) or "", s(p["PRODUTO"]) or "")] for p in products], company, company_default, refs, evidence)
|
||||
return {"file": path.name, "company": {k: company.get(k) for k in ("status", "uf", "crt", "error")}, **report}
|
||||
|
||||
|
||||
def main():
|
||||
base.load_dotenv(ROOT / ".env")
|
||||
token = sys.argv[1]
|
||||
directory = Path(sys.argv[2]) if len(sys.argv) > 2 else ROOT / "Banco"
|
||||
out = []
|
||||
for p in sorted(directory.glob("*.FDB")):
|
||||
print("analyzing", p.name, flush=True)
|
||||
out.append(analyze(p, token))
|
||||
(ROOT / "docs" / "runs" / "multi_db" / f"{p.stem}_coverage.json").write_text(json.dumps(out[-1], ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
print(json.dumps({k: out[-1][k] for k in ("file", "TOTAL_PRODUCTS", "company")}, ensure_ascii=False), flush=True)
|
||||
(ROOT / "docs" / "MULTI_DATABASE_FISCAL_COVERAGE_V1.json").write_text(json.dumps({"artifact": "MULTI_DATABASE_FISCAL_COVERAGE_V1", "firebird_writes": 0, "databases": out}, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Product Fiscal Defaults V1 runner: CFOP/CSOSN/CST over all C000025 products (read-only).
|
||||
|
||||
Writes docs/PRODUCT_FISCAL_DEFAULTS_MATRIX_V1.json, ..._DRY_RUN_V1.json, FISCAL_CONTEXT_REQUIREMENTS_V2.json,
|
||||
NEXT_FISCAL_FIELD_IMPLEMENTATION_V2.md. No Firebird writes: the engine can only emit write_eligible=False
|
||||
(see central/app/product_defaults.py); this runner refuses to write anything.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
import run_fiscal_decision_v1 as base # noqa: E402
|
||||
from central.app.product_defaults import PRODUCT_DEFAULTS_ENGINE_VERSION, cfop_default, csosn_default, cst_status, validate_against_history # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import _connect # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
token = sys.argv[1]
|
||||
hub = "http://localhost:8000"
|
||||
base.load_dotenv(ROOT / ".env")
|
||||
conn = _connect(os.environ); cur = conn.cursor()
|
||||
cur.execute("select CODIGO, PRODUTO, CLASSIFICACAO_FISCAL, CFOP, CSOSN, CST from C000025 order by CODIGO")
|
||||
products = [dict(zip(("CODIGO", "PRODUTO", "CLASSIFICACAO_FISCAL", "CFOP", "CSOSN", "CST"), r)) for r in cur.fetchall()]
|
||||
cur.execute("select ICMS_CSOSN, CRT, UF from C000004"); company_default, crt, uf = [str(x).strip() for x in cur.fetchone()]
|
||||
conn.close()
|
||||
refs = base.build_refs(hub, token, products, uf)
|
||||
|
||||
decisions = []
|
||||
for i in range(0, len(products), 100):
|
||||
chunk = products[i:i + 100]
|
||||
decisions += base.http(f"{hub}/v1/classify/batch", token, {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]})["results"]
|
||||
|
||||
trusted = json.loads((ROOT / "docs" / "runs" / "trusted_historical_items.json").read_text(encoding="utf-8"))
|
||||
hist = defaultdict(lambda: {"cfop": [], "csosn": []})
|
||||
for t in trusted:
|
||||
h = hist[str(t["cod"]).strip()]
|
||||
if (t.get("cfop") or "").strip(): h["cfop"].append(t["cfop"].strip())
|
||||
if (t.get("csosn") is not None) and str(t["csosn"]).strip(): h["csosn"].append(str(t["csosn"]).strip())
|
||||
|
||||
rows = []
|
||||
for p, d in zip(products, decisions):
|
||||
code = str(p["CODIGO"]).strip()
|
||||
ncm_status = d.get("status"); mva = d.get("ba_st_mva_status")
|
||||
cf = cfop_default(p, ncm_status, mva, refs["cfop_valid"])
|
||||
cs = csosn_default(p, ncm_status, mva, refs["csosn"], company_default)
|
||||
ct = cst_status(p, refs["cst_icms"], refs["origem"])
|
||||
h = hist.get(code, {"cfop": [], "csosn": []})
|
||||
cf["historical_validation"] = validate_against_history(cf["proposed_value"], h["cfop"])
|
||||
cs["historical_validation"] = validate_against_history(cs["proposed_value"], h["csosn"])
|
||||
cf_cur_share = (sum(1 for v in h["cfop"] if v == cf["current_value"]) / len(h["cfop"])) if h["cfop"] else None
|
||||
cs_cur_share = (sum(1 for v in h["csosn"] if v == cs["current_value"]) / len(h["csosn"])) if h["csosn"] else None
|
||||
rows.append({
|
||||
"CODIGO": code, "DESCRIPTION": p["PRODUTO"], "NCM": p["CLASSIFICACAO_FISCAL"],
|
||||
"CFOP_CURRENT": cf["current_value"], "CFOP_PROPOSED": cf["proposed_value"], "CFOP_STATUS": cf["status"], "CFOP_REASON": cf["reason_code"], "CFOP_WRITE_ELIGIBLE": cf["write_eligible"],
|
||||
"CSOSN_CURRENT": cs["current_value"], "CSOSN_PROPOSED": cs["proposed_value"], "CSOSN_STATUS": cs["status"], "CSOSN_REASON": cs["reason_code"], "CSOSN_WRITE_ELIGIBLE": cs["write_eligible"],
|
||||
"CST_CURRENT": ct["current_value"], "CST_STATUS": ct["status"],
|
||||
"HISTORICAL_OCCURRENCES": {"trusted_cfop_items": len(h["cfop"]), "trusted_csosn_items": len(h["csosn"])},
|
||||
"HISTORICAL_CONSISTENCY": {"cfop_current_equals_emitted_share": cf_cur_share, "csosn_current_equals_emitted_share": cs_cur_share,
|
||||
"cfop_validation": cf["historical_validation"], "csosn_validation": cs["historical_validation"]},
|
||||
"PROVENANCE": {"engine": PRODUCT_DEFAULTS_ENGINE_VERSION, "cfop_dataset": d.get("dataset_versions", {}).get("cfop"), "moc": d.get("dataset_versions", {}).get("nfe_moc_tables"),
|
||||
"ba_st_mva": d.get("dataset_versions", {}).get("ba_st_mva"), "company": {"crt": crt, "uf": uf, "icms_csosn": company_default}},
|
||||
"_cfop": cf, "_csosn": cs,
|
||||
})
|
||||
|
||||
def count(field, key):
|
||||
c = Counter(r[key] for r in rows)
|
||||
return c
|
||||
cf_states = Counter(r["CFOP_STATUS"] for r in rows); cs_states = Counter(r["CSOSN_STATUS"] for r in rows)
|
||||
cf_act = Counter(r["_cfop"]["action_class"] for r in rows); cs_act = Counter(r["_csosn"]["action_class"] for r in rows)
|
||||
cfv = Counter(r["HISTORICAL_CONSISTENCY"]["cfop_validation"] for r in rows if r["CFOP_PROPOSED"] is not None)
|
||||
csv_ = Counter(r["HISTORICAL_CONSISTENCY"]["csosn_validation"] for r in rows if r["CSOSN_PROPOSED"] is not None)
|
||||
cf_conflicts = [r for r in rows if r["CFOP_STATUS"] == "CFOP_DEFAULT_CANDIDATE"]
|
||||
conflict_hist = Counter()
|
||||
for r in cf_conflicts:
|
||||
share = r["HISTORICAL_CONSISTENCY"]["cfop_current_equals_emitted_share"]
|
||||
conflict_hist["no_trusted_history" if share is None else ("history_emitted_current_value" if share == 1.0 else "history_differs_from_current")] += 1
|
||||
summary = {
|
||||
"TOTAL_PRODUCTS": len(rows), "company": {"crt": crt, "uf": uf, "icms_csosn": company_default},
|
||||
"CFOP": {"RESOLVED": cf_states["CFOP_DEFAULT_RESOLVED"], "ALREADY_CORRECT": cf_act["ALREADY_CORRECT"], "CHANGE_REQUIRED": 0,
|
||||
"REVIEW": cf_act["REVIEW_REQUIRED"], "CONTEXT_REQUIRED": cf_states["CFOP_OPERATION_CONTEXT_REQUIRED"], "UNRESOLVED": cf_states["CFOP_UNRESOLVED"],
|
||||
"CANDIDATE": cf_states["CFOP_DEFAULT_CANDIDATE"], "AUTO_APPLY_SAFE": 0, "states": dict(cf_states), "historical_validation_of_proposals": dict(cfv),
|
||||
"conflict_candidates_vs_history": dict(conflict_hist)},
|
||||
"CSOSN": {"RESOLVED": cs_states["CSOSN_DEFAULT_RESOLVED"], "ALREADY_CORRECT": 0, "CHANGE_REQUIRED": 0, "REVIEW": cs_act["REVIEW_REQUIRED"],
|
||||
"CONTEXT_REQUIRED": cs_states["CSOSN_OPERATION_CONTEXT_REQUIRED"], "UNRESOLVED": cs_states["CSOSN_UNRESOLVED"], "CANDIDATE": cs_states["CSOSN_DEFAULT_CANDIDATE"],
|
||||
"AUTO_APPLY_SAFE": 0, "states": dict(cs_states), "historical_validation_of_proposals": dict(csv_),
|
||||
"st_conflicts": dict(Counter(r["_csosn"].get("consistency_flag") for r in rows if r["_csosn"].get("consistency_flag")))},
|
||||
"CST": dict(Counter(r["CST_STATUS"] for r in rows)),
|
||||
}
|
||||
for r in rows:
|
||||
r.pop("_cfop"); r.pop("_csosn")
|
||||
dm = {"artifact": "PRODUCT_FISCAL_DEFAULTS_MATRIX_V1", "engine": PRODUCT_DEFAULTS_ENGINE_VERSION, "products": rows}
|
||||
(ROOT / "docs" / "PRODUCT_FISCAL_DEFAULTS_MATRIX_V1.json").write_text(json.dumps(dm, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
dry = {"artifact": "PRODUCT_FISCAL_DEFAULTS_DRY_RUN_V1", "firebird_writes": 0, "write_candidates": [], **summary,
|
||||
"note": "No proposal is write-eligible: CFOP conflicts need ST-retention/resale context; CSOSN needs product ICMS-treatment and retention context."}
|
||||
(ROOT / "docs" / "PRODUCT_FISCAL_DEFAULTS_DRY_RUN_V1.json").write_text(json.dumps(dry, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
|
||||
# context requirements V2 (exact inputs still blocking)
|
||||
st_unknown = sum(1 for r in rows if r["CFOP_REASON"] == "ST_APPLICABILITY_UNCONFIRMED_NEEDS_CEST_CONFIRMATION")
|
||||
st_subject = sum(1 for r in rows if r["CSOSN_REASON"] == "ST_RETENTION_BY_SUPPLIER_UNKNOWN")
|
||||
ncm_bad = sum(1 for r in rows if r["CFOP_REASON"] == "NCM_NOT_VALID_CANNOT_ASSESS_ST")
|
||||
csosn_notst = sum(1 for r in rows if r["CSOSN_STATUS"] == "CSOSN_DEFAULT_CANDIDATE")
|
||||
blank_cfop = sum(1 for r in rows if r["CFOP_REASON"] == "NO_DECLARED_RESALE_OR_PRODUCTION_FAMILY")
|
||||
other_family = sum(1 for r in rows if r["CFOP_REASON"] in ("NO_RULE_FOR_DECLARED_CFOP_FAMILY", "DIRECTION_OR_UF_SCOPE_NOT_INTRA_UF_SAIDA"))
|
||||
conflicts = len(cf_conflicts)
|
||||
req = [
|
||||
{"INPUT": "valid_ncm", "SOURCE_EXPECTED": "product master (C000025.CLASSIFICACAO_FISCAL)", "AVAILABLE": "PARTIAL (495/521 valid)", "AFFECTED_PRODUCTS": ncm_bad, "AFFECTED_FIELD": "CFOP,CSOSN", "NEXT_ACTION": "Fix/review invalid+legacy NCMs (never auto-change)."},
|
||||
{"INPUT": "st_applicability_confirmed (CEST confirmation)", "SOURCE_EXPECTED": "CEST dataset + BA Anexo 1 item", "AVAILABLE": "PARTIAL (31 resolved, 237 candidates)", "AFFECTED_PRODUCTS": st_unknown, "AFFECTED_FIELD": "CFOP,CSOSN", "NEXT_ACTION": "Confirm CEST per product (description evidence/manual review) to settle ST applicability."},
|
||||
{"INPUT": "st_retention_by_supplier", "SOURCE_EXPECTED": "purchase document per product (entrada) or supplier flag", "AVAILABLE": False, "AFFECTED_PRODUCTS": st_subject, "AFFECTED_FIELD": "CSOSN(500 vs 102),CFOP(5405 vs 5102)", "NEXT_ACTION": "Locate structured retention evidence in purchase items (C000032/C000062 CST 060/CSOSN 500 history) or ask vendor."},
|
||||
{"INPUT": "product_icms_treatment (taxed / exempt / not taxed)", "SOURCE_EXPECTED": "state ICMS rules per NCM (no acquired dataset)", "AVAILABLE": False, "AFFECTED_PRODUCTS": csosn_notst, "AFFECTED_FIELD": "CSOSN(102 vs 400/300/103)", "NEXT_ACTION": "Needs an official per-NCM ICMS treatment source; ERP CST=040 with CSOSN=102 coexists, so ERP alone is ambiguous."},
|
||||
{"INPUT": "declared_resale_vs_production", "SOURCE_EXPECTED": "product nature/type (C000025.TIPO_ITEM unused) or company activity", "AVAILABLE": False, "AFFECTED_PRODUCTS": blank_cfop + other_family, "AFFECTED_FIELD": "CFOP", "NEXT_ACTION": "Confirm meaning of FOOD_ATRIBUICAO_VALORES/TIPO_ITEM with vendor before deriving 5101 vs 5102."},
|
||||
{"INPUT": "st_conflict_resolution (which of ERP CFOP or BA Anexo 1 is wrong)", "SOURCE_EXPECTED": "human/fiscal review of conflicting products", "AVAILABLE": False, "AFFECTED_PRODUCTS": conflicts, "AFFECTED_FIELD": "CFOP", "NEXT_ACTION": "Review queue; do not auto-apply."},
|
||||
{"INPUT": "destination_uf / consumer_final / direction", "SOURCE_EXPECTED": "NFC-e document model 65 + chave cUF (proven 1499/1499) and C000004.UF", "AVAILABLE": True, "AFFECTED_PRODUCTS": 0, "AFFECTED_FIELD": "CFOP", "NEXT_ACTION": "Already satisfied for NFC-e scope; NF-e needs customer data."},
|
||||
]
|
||||
(ROOT / "docs" / "FISCAL_CONTEXT_REQUIREMENTS_V2.json").write_text(json.dumps({"artifact": "FISCAL_CONTEXT_REQUIREMENTS_V2", "requirements": req}, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
|
||||
# priorities V2 (mechanical)
|
||||
prio = [
|
||||
("CFOP", "CONFIRMED (HIGH)", "PARTIAL", "YES (ST family)", summary["CFOP"]["RESOLVED"] + conflicts, "trusted history available", "AUTO_APPLY_SAFE=0; review the ST conflicts."),
|
||||
("CSOSN", "CONFIRMED (HIGH)", "NO", "NO (treatment/retention sources missing)", summary["CSOSN"]["CANDIDATE"] + summary["CSOSN"]["CONTEXT_REQUIRED"], "trusted history available", "Acquire retention/treatment evidence."),
|
||||
("CST", "CONFIRMED (MEDIUM)", "YES", "n/a (reference only, CRT=1)", 521, "trusted history 100%", "No write needed."),
|
||||
("CEST", "CONFIRMED (HIGH)", "YES", "PARTIAL", 15, "weak (1/15 supported)", "Description-evidence review of 15 candidates."),
|
||||
("IPI", "CONFIRMED (HIGH)", "YES", "YES", 107, "n/a", "Maintained; already reconciled."),
|
||||
]
|
||||
md = ["# NEXT_FISCAL_FIELD_IMPLEMENTATION_V2", "",
|
||||
"Mechanical ordering: mapping confirmed, rule available, context available, products affected, history validation, auto-apply possible (post PRODUCT DEFAULTS V1).", "",
|
||||
"| FIELD | ERP_MAPPING | CONTEXT_AVAILABLE | RULE_AVAILABLE | POTENTIAL_PRODUCTS | HISTORICAL_VALIDATION | NEXT_ACTION |", "|---|---|---|---|---|---|---|"]
|
||||
md += [f"| {a} | {b} | {c} | {d_} | {e} | {f} | {g} |" for a, b, c, d_, e, f, g in prio]
|
||||
md += ["", "Blocking inputs and affected counts: `docs/FISCAL_CONTEXT_REQUIREMENTS_V2.json`."]
|
||||
(ROOT / "docs" / "NEXT_FISCAL_FIELD_IMPLEMENTATION_V2.md").write_text("\n".join(md) + "\n", encoding="utf-8")
|
||||
print(json.dumps(summary, ensure_ascii=True, indent=1))
|
||||
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Re-export of gti_fiscal_agent.capabilities (the Agent ships the module; scripts keep their historical import path)."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "agent" / "src"))
|
||||
from gti_fiscal_agent.capabilities import * # noqa: F401,F403,E402
|
||||
from gti_fiscal_agent.capabilities import ( # noqa: F401,E402
|
||||
ACCEPT, CAPABILITIES, DEFAULT_ACCEPT, FAMILY, NAME_KEYWORDS, ORIGIN_FIELD, WRITE_TARGETS, classify_schema, detect_capabilities,
|
||||
field_compatibility, infer_segment, type_ok)
|
||||
@@ -0,0 +1,100 @@
|
||||
"""SGV UI oracle: snapshot/diff of ONE C000025 row (READ-ONLY; never writes to Firebird).
|
||||
|
||||
python scripts/sgv_product_snapshot.py before <CODIGO>
|
||||
-> reads the whole row and saves docs/runs/snapshots/<CODIGO>_before.json
|
||||
(operator changes exactly ONE field in the SGV product screen and saves)
|
||||
python scripts/sgv_product_snapshot.py diff <CODIGO> [--ui-label "CFOP saída"]
|
||||
-> re-reads, prints FIELD/BEFORE/AFTER for every changed column, and (with --ui-label) records the
|
||||
result in docs/runs/sgv_ui_diff_ledger.json. A label is CONFIRMED only when exactly ONE
|
||||
non-technical column changed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
SNAP_DIR = ROOT / "docs" / "runs" / "snapshots"
|
||||
LEDGER = ROOT / "docs" / "runs" / "sgv_ui_diff_ledger.json"
|
||||
# Columns the ERP rewrites by itself on every save; ignored ONLY because they are clearly automatic.
|
||||
TECHNICAL = {"ULTIMA_ALTERACAO", "DATA_ALTERACAO", "DATA_ULTIMA_ALTERACAO", "USUARIO_ALTERACAO"}
|
||||
|
||||
|
||||
def norm(v):
|
||||
if isinstance(v, (bytes, bytearray)):
|
||||
import hashlib
|
||||
return "blob:" + hashlib.sha256(bytes(v)).hexdigest()[:16]
|
||||
if isinstance(v, (datetime, date)):
|
||||
return v.isoformat()
|
||||
if isinstance(v, Decimal):
|
||||
return str(v)
|
||||
return v.strip() if isinstance(v, str) else v
|
||||
|
||||
|
||||
def read_row(env, code):
|
||||
from gti_fiscal_agent.firebird_reader import _connect # read-only isolation
|
||||
conn = _connect(env)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT * FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
raise SystemExit(f"product {code} not found")
|
||||
names = [c[0].strip() for c in cur.description]
|
||||
blob = {}
|
||||
return {n: norm(v) for n, v in zip(names, row)}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def compute_diff(before, after):
|
||||
out = []
|
||||
for f in sorted(set(before) | set(after)):
|
||||
if before.get(f) != after.get(f):
|
||||
out.append({"FIELD": f, "BEFORE": before.get(f), "AFTER": after.get(f), "technical": f in TECHNICAL})
|
||||
return out
|
||||
|
||||
|
||||
def load_env():
|
||||
for line in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, _, v = line.partition("="); os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def record(label, code, changes):
|
||||
real = [c for c in changes if not c["technical"]]
|
||||
ledger = json.loads(LEDGER.read_text(encoding="utf-8")) if LEDGER.exists() else []
|
||||
entry = {"ui_label": label, "product": code, "recorded": datetime.now().isoformat(timespec="seconds"), "changed_columns": [c["FIELD"] for c in real],
|
||||
"before": {c["FIELD"]: c["BEFORE"] for c in real}, "after": {c["FIELD"]: c["AFTER"] for c in real},
|
||||
"confirmed": len(real) == 1, "note": "" if len(real) == 1 else ("no column changed" if not real else "more than one column changed; repeat changing a single field")}
|
||||
ledger.append(entry)
|
||||
LEDGER.write_text(json.dumps(ledger, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
return entry
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("action", choices=["before", "diff"]); ap.add_argument("codigo"); ap.add_argument("--ui-label")
|
||||
a = ap.parse_args(argv)
|
||||
load_env()
|
||||
SNAP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
path = SNAP_DIR / f"{a.codigo}_before.json"
|
||||
if a.action == "before":
|
||||
path.write_text(json.dumps(read_row(os.environ, a.codigo), ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
||||
print(f"snapshot saved: {path}"); return
|
||||
if not path.exists():
|
||||
raise SystemExit("run 'before' first")
|
||||
changes = compute_diff(json.loads(path.read_text(encoding="utf-8")), read_row(os.environ, a.codigo))
|
||||
print(f"{'FIELD':<28}{'BEFORE':<28}AFTER")
|
||||
for c in changes:
|
||||
print(f"{c['FIELD']:<28}{str(c['BEFORE']):<28}{c['AFTER']}{' (technical, ignored)' if c['technical'] else ''}")
|
||||
if a.ui_label:
|
||||
print(json.dumps(record(a.ui_label, a.codigo, changes), ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user