gti
d21a9d1140
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
130 lines
7.7 KiB
Python
130 lines
7.7 KiB
Python
"""Read-only historical fiscal analysis over TESTESOFICIAIS.FDB: correlates emitted
|
|
NFC-e items (NFCE_ITENS) and sale/purchase item tables with C000025 product defaults.
|
|
No DML/DDL. Output: docs/runs/erp_fiscal_history_analysis.json
|
|
"""
|
|
import json, os, sys
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
|
from gti_fiscal_agent.firebird_reader import _connect # noqa: E402
|
|
|
|
for l in (ROOT / ".env").read_text(encoding="utf-8").splitlines():
|
|
if "=" in l and not l.startswith("#"):
|
|
k, _, v = l.partition("="); os.environ.setdefault(k.strip(), v.strip())
|
|
|
|
|
|
def s(v):
|
|
return None if v is None else str(v).strip()
|
|
|
|
|
|
def digits(v):
|
|
return "".join(c for c in str(v or "") if c.isdigit())
|
|
|
|
|
|
def main():
|
|
conn = _connect(os.environ); cur = conn.cursor()
|
|
cur.execute("select CODIGO, CFOP, CST, CSOSN, GRUPO_ICMS, CEST, CLASSIFICACAO_FISCAL, COD_SAIDA_PIS, INCIDENCIA_PISCOFINS, ALIQUOTA from C000025")
|
|
prod = {s(r[0]): dict(zip(("CFOP", "CST", "CSOSN", "GRUPO_ICMS", "CEST", "NCM", "PIS", "INC", "ALIQ"), map(s, r[1:]))) for r in cur.fetchall()}
|
|
|
|
res = {"products": len(prod)}
|
|
cur.execute("select min(DATA), max(DATA), count(*) from NFCE_ITENS"); mn, mx, n = cur.fetchone()
|
|
res["nfce_itens"] = {"rows": n, "date_min": str(mn), "date_max": str(mx)}
|
|
cur.execute("select count(*), min(DATA), max(DATA) from NFCE"); r = cur.fetchone(); res["nfce_docs"] = {"rows": r[0], "min": str(r[1]), "max": str(r[2])}
|
|
|
|
cur.execute("select CODPRODUTO, CFOP, CST, GRUPO_ICMS, CSOSN, NCM, CEST, CST_PIS_COFINS, ALIQUOTA, VALOR from NFCE_ITENS")
|
|
rows = cur.fetchall()
|
|
joined = 0; agree = defaultdict(Counter); per_prod = defaultdict(lambda: defaultdict(Counter)); orphan = 0
|
|
for cod, cfop, cst, gicms, csosn, ncm, cest, pispc, aliq, val in rows:
|
|
p = prod.get(s(cod))
|
|
if p is None:
|
|
orphan += 1; continue
|
|
joined += 1
|
|
item = {"CFOP": s(cfop), "CST": s(cst), "GRUPO_ICMS": s(gicms), "CSOSN": s(csosn), "NCM": s(ncm), "CEST": s(cest), "PIS": s(pispc)}
|
|
for f, iv in item.items():
|
|
if iv is None:
|
|
agree[f]["ITEM_NULL"] += 1; continue
|
|
pv = p.get(f if f != "PIS" else "PIS")
|
|
if f == "CEST":
|
|
agree[f]["MATCH" if digits(iv) == digits(pv) else "MISMATCH"] += 1
|
|
elif f == "PIS":
|
|
agree[f]["MATCH" if iv.lstrip("0") == (pv or "").lstrip("0") else "MISMATCH"] += 1
|
|
else:
|
|
agree[f]["MATCH" if iv == pv else "MISMATCH"] += 1
|
|
per_prod[s(cod)][f][iv] += 1
|
|
res["nfce_join"] = {"items_joined_to_C000025": joined, "orphan_items": orphan,
|
|
"agreement_item_vs_product_default": {f: dict(c) for f, c in agree.items()}}
|
|
res["nfce_products_seen"] = len(per_prod)
|
|
res["variability"] = {f: sum(1 for pp in per_prod.values() if len(pp[f]) > 1) for f in ("CFOP", "CST", "GRUPO_ICMS", "CSOSN", "CEST", "NCM")}
|
|
res["nfce_item_value_distributions"] = {}
|
|
for f, idx in (("CFOP", 1), ("CST", 2), ("GRUPO_ICMS", 3), ("CSOSN", 4), ("CST_PIS_COFINS", 7)):
|
|
res["nfce_item_value_distributions"][f] = Counter(s(r[idx]) for r in rows).most_common(8)
|
|
|
|
# Sale/purchase item tables (C000032 sales, C000062 purchase items) vs product defaults
|
|
for table, extra in (("C000032", "CST, CFOP"), ("C000062", "CST, CFOP")):
|
|
cur.execute(f"select CODPRODUTO, {extra} from {table}")
|
|
c = Counter(); cfopc = Counter(); cstc = Counter()
|
|
for cod, cst, cfop in cur.fetchall():
|
|
p = prod.get(s(cod))
|
|
if p is None: c["ORPHAN"] += 1; continue
|
|
c["JOINED"] += 1
|
|
if s(cst): cstc["MATCH" if s(cst) == p["CST"] else "MISMATCH"] += 1
|
|
if s(cfop): cfopc["MATCH" if s(cfop) == p["CFOP"] else "MISMATCH"] += 1
|
|
res[table] = {"joined": dict(c), "cst_vs_product": dict(cstc), "cfop_vs_product": dict(cfopc)}
|
|
|
|
# Natureza de operação (C000030) and operation-context sources
|
|
cur.execute("select TIPO, count(*) from C000030 group by TIPO"); res["C000030_tipo"] = dict(cur.fetchall())
|
|
cur.execute("select substring(CFOP from 1 for 1), count(*) from C000030 group by 1 order by 1"); res["C000030_cfop_first_digit"] = dict(cur.fetchall())
|
|
cur.execute("select CFOP from C000030 where CFOP in ('5102','5405','5656','5929','6102','1102')"); res["C000030_has_cfops"] = [s(r[0]) for r in cur.fetchall()]
|
|
cur.execute("select IND_CFOP_ENTRADA, IND_CFOP_SAIDA, IND_CFOP_PERDA from C000001"); res["C000001_cfop_defaults"] = [s(x) for x in cur.fetchone()]
|
|
cur.execute("select OPTANTE_SIMPLES, OPTANTE_SUPER_SIMPLES, CRT, ICMS_CSOSN, SUBSTITUTO_TRIBUTARIO, CONTRIBUINTE_IPI, UF from C000004"); res["C000004"] = [s(x) for x in cur.fetchone()]
|
|
cur.execute("select EMITENTE_CRT from EMISSOR"); res["EMISSOR_CRT"] = [s(x) for x in cur.fetchone()]
|
|
cur.execute("select CFOP, count(*) from C000062 group by CFOP"); res["C000062_cfop"] = {s(a): b for a, b in cur.fetchall()}
|
|
cur.execute("select CST, count(*) from C000062 group by CST"); res["C000062_cst"] = {s(a): b for a, b in cur.fetchall()}
|
|
cur.execute("select count(*), sum(case when MARGEM_AGREGADA <> 0 then 1 else 0 end), sum(case when ICMS_SUB <> 0 then 1 else 0 end) from C000062"); res["C000062_mva_usage"] = list(cur.fetchone())
|
|
cur.execute("select count(*) from PRODUTOS_MARGEM_AGREGADA"); res["PRODUTOS_MARGEM_AGREGADA_rows"] = cur.fetchone()[0]
|
|
cur.execute("select count(*) from C000025 where MARGEM_AGREGADA is not null"); res["C000025_MARGEM_AGREGADA_nonnull"] = cur.fetchone()[0]
|
|
|
|
# CEST history for the 15 CEST-candidate products (from the decision matrix)
|
|
matrix = json.loads((ROOT / "docs" / "FISCAL_DECISION_MATRIX_V1.json").read_text(encoding="utf-8"))
|
|
cand = [m for m in matrix["products"] if m["fields"]["CEST"]["status"] == "RULE_CANDIDATE"]
|
|
cest_hist = []
|
|
for m in cand:
|
|
code = s(m["CODIGO"])
|
|
hist = Counter(digits(k) for k, c in per_prod.get(code, {}).get("CEST", {}).items() for _ in range(c) if digits(k))
|
|
cest_hist.append({"CODIGO": code, "DESCRIPTION": m["DESCRIPTION"], "NCM": m["NCM"], "erp_current_cest": m["fields"]["CEST"]["current_value"], "historical_cest": dict(hist)})
|
|
res["cest_candidates_history"] = cest_hist
|
|
conn.close()
|
|
|
|
# Shadow validation: decision matrix engine value vs historical NFC-e item value
|
|
val = {}
|
|
for dom, hist_field in (("NCM", "NCM"), ("CEST", "CEST"), ("CFOP", "CFOP"), ("CST_ICMS", "CST"), ("CSOSN", "CSOSN"), ("PIS_CST", "PIS")):
|
|
val[dom] = Counter()
|
|
by_code = {s(m["CODIGO"]): m["fields"] for m in matrix["products"]}
|
|
conn = _connect(os.environ); cur = conn.cursor()
|
|
cur.execute("select first 5000 CODPRODUTO, NCM, CEST, CFOP, CST, CSOSN, CST_PIS_COFINS from NFCE_ITENS where NCM is not null")
|
|
n_sim = 0
|
|
for cod, ncm, cest, cfop, cst, csosn, pispc in cur.fetchall():
|
|
f = by_code.get(s(cod))
|
|
if f is None: continue
|
|
n_sim += 1
|
|
hist = {"NCM": s(ncm), "CEST": s(cest), "CFOP": s(cfop), "CST_ICMS": s(cst), "CSOSN": s(csosn), "PIS_CST": s(pispc)}
|
|
for dom, hv in hist.items():
|
|
rec = f[dom]
|
|
ev = rec["proposed_value"] if rec["status"] == "RULE_RESOLVED" else None
|
|
if rec["status"] == "RULE_RESOLVED" and dom == "NCM":
|
|
ev = rec["current_value"]
|
|
if ev is None:
|
|
val[dom]["UNRESOLVED"] += 1
|
|
elif digits(ev) == digits(hv) if dom in ("NCM", "CEST") else str(ev) == str(hv):
|
|
val[dom]["MATCH"] += 1
|
|
else:
|
|
val[dom]["MISMATCH"] += 1
|
|
res["shadow_validation"] = {"items": n_sim, **{d: dict(c) for d, c in val.items()}}
|
|
conn.close()
|
|
print(json.dumps(res, ensure_ascii=False, indent=1, default=str))
|
|
|
|
|
|
main()
|