116 lines
6.9 KiB
Python
116 lines
6.9 KiB
Python
"""Fiscal Engine V1 coverage over every database in ./Banco — READ-ONLY (no Firebird writes, no new rules).
|
|
|
|
python scripts/run_multi_db_coverage.py <AGENT_TOKEN> [DIR]
|
|
|
|
Reuses the existing pure engines: fiscal_decision (NCM/IPI/EX/CEST/MVA), product_defaults (CFOP), icms_context (ST, role,
|
|
withholding, CSOSN V2). Only GUARDS are added for company context: the BA Anexo 1 dataset is applied to UF=BA companies only, and
|
|
CSOSN logic only under CRT=1; otherwise the engines are told the context is unavailable (no fiscal rule invented).
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
|
|
import run_fiscal_decision_v1 as base # noqa: E402
|
|
import fdb # noqa: E402
|
|
from central.app.fiscal_decision import decide_product # noqa: E402
|
|
from central.app.icms_context import csosn_v2, product_role, st_status, withheld_status # noqa: E402
|
|
from central.app.product_defaults import cfop_default # noqa: E402
|
|
from gti_fiscal_agent.firebird_reader import parse_company_context # noqa: E402
|
|
|
|
HUB = "http://localhost:8000"
|
|
s = lambda v: None if v is None else str(v).strip() # noqa: E731
|
|
dg = lambda v: "".join(c for c in str(v or "") if c.isdigit()) # noqa: E731
|
|
|
|
|
|
def connect(path):
|
|
return fdb.connect(database=str(path), user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
|
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
|
|
|
|
|
def table_cols(cur):
|
|
cur.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS")
|
|
m = defaultdict(set)
|
|
for t, f in cur.fetchall():
|
|
m[t].add(f)
|
|
return m
|
|
|
|
|
|
def classify_all(token, products):
|
|
"""Classify every distinct (ncm, description) with the production Classifier inside the hub container (read-only on PostgreSQL,
|
|
memoized dataset lookups; identical code path to /v1/classify, verified against the HTTP API in tests/verification)."""
|
|
import subprocess
|
|
import tempfile
|
|
pairs = sorted({(s(p["CLASSIFICACAO_FISCAL"]) or "", s(p["PRODUTO"]) or "") for p in products})
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
fin, fout = Path(tmp) / "in.json", Path(tmp) / "out.json"
|
|
fin.write_text(json.dumps(pairs, ensure_ascii=False), encoding="utf-8")
|
|
subprocess.run(["docker", "cp", str(ROOT / "scripts" / "bulk_classify_in_container.py"), "projeto-fiscal-hub-1:/tmp/bulk.py"], check=True)
|
|
subprocess.run(["docker", "cp", str(fin), "projeto-fiscal-hub-1:/tmp/in.json"], check=True)
|
|
subprocess.run(["docker", "exec", "projeto-fiscal-hub-1", "sh", "-c", "cd /app && python /tmp/bulk.py /tmp/in.json /tmp/out.json"], check=True, capture_output=True)
|
|
subprocess.run(["docker", "cp", "projeto-fiscal-hub-1:/tmp/out.json", str(fout)], check=True)
|
|
results = json.loads(fout.read_text(encoding="utf-8"))
|
|
print(f" classify: {len(pairs)} distinct (ncm, description) pairs for {len(products)} products", flush=True)
|
|
return dict(zip(pairs, results))
|
|
|
|
|
|
def analyze(path, token):
|
|
conn = connect(path); cur = conn.cursor()
|
|
tc = table_cols(cur)
|
|
have = lambda t, *cols: t in tc and all(c in tc[t] for c in cols) # noqa: E731
|
|
c25 = tc["C000025"]
|
|
cols = [c for c in ("CODIGO", "PRODUTO", "CLASSIFICACAO_FISCAL", "CFOP", "CSOSN", "CST", "CEST", "IPI", "EX_TIPI", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA") if c in c25]
|
|
cur.execute(f"SELECT {','.join(cols)} FROM C000025")
|
|
products = [dict(zip(cols, r)) for r in cur.fetchall()]
|
|
for p in products:
|
|
for c in ("CFOP", "CSOSN", "CST", "CEST", "IPI", "EX_TIPI", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA", "PRODUTO", "CLASSIFICACAO_FISCAL"):
|
|
p.setdefault(c, None)
|
|
# company (fail closed; guards below)
|
|
try:
|
|
cur.execute("SELECT UF, CRT FROM C000004"); company = {"status": "OK", **parse_company_context(cur.fetchall())}
|
|
except Exception as e: # noqa: BLE001
|
|
company = {"status": "UNAVAILABLE", "error": str(e)[:120], "uf": None, "crt": None}
|
|
company_default = None
|
|
if have("C000004", "ICMS_CSOSN"):
|
|
cur.execute("SELECT FIRST 1 ICMS_CSOSN FROM C000004 ORDER BY CODIGO"); company_default = s(cur.fetchone()[0])
|
|
# context evidence (only where the tables exist in this database)
|
|
recipes = set(); composed = set(); purchases = defaultdict(list); stock_entry = {}
|
|
if have("RECEITA_INGREDIENTES", "CODRECEITA"):
|
|
cur.execute("SELECT CODRECEITA FROM RECEITA_INGREDIENTES WHERE CODRECEITA IS NOT NULL"); recipes = {str(r[0]).zfill(6) for r in cur.fetchall()}
|
|
if have("C000032", "CODPRODUTO", "TIPO"):
|
|
cur.execute("SELECT DISTINCT CODPRODUTO FROM C000032 WHERE TIPO = 'COMPOSICAO'"); composed = {s(r[0]) for r in cur.fetchall()}
|
|
if have("C000088", "CODPRODUTO", "ICMS_RETIDO", "ICMS_VALORRETIDO", "SUB_VALOR"):
|
|
cur.execute("SELECT CODPRODUTO, ICMS_RETIDO, ICMS_VALORRETIDO, SUB_VALOR FROM C000088")
|
|
for cod, ret, vr, sv in cur.fetchall():
|
|
purchases[s(cod)].append({"icms_retido": s(ret), "valor_retido": vr, "sub_valor": sv})
|
|
if have("C000100", "CODPRODUTO", "ULTIMA_ENTRADA", "NOTA_FISCAL"):
|
|
cur.execute("SELECT CODPRODUTO, ULTIMA_ENTRADA, NOTA_FISCAL FROM C000100"); stock_entry = {s(r[0]): (r[1] is not None or s(r[2]) not in (None, "", "000000")) for r in cur.fetchall()}
|
|
conn.close()
|
|
|
|
refs = base.build_refs(HUB, token, products, company["uf"] or "BA")
|
|
decisions = classify_all(token, products)
|
|
from gti_fiscal_agent.coverage import compute_coverage
|
|
evidence = {"recipes": recipes, "composed": composed, "purchases": dict(purchases), "stock_entry": stock_entry}
|
|
report = compute_coverage(products, [decisions[(s(p["CLASSIFICACAO_FISCAL"]) or "", s(p["PRODUTO"]) or "")] for p in products], company, company_default, refs, evidence)
|
|
return {"file": path.name, "company": {k: company.get(k) for k in ("status", "uf", "crt", "error")}, **report}
|
|
|
|
|
|
def main():
|
|
base.load_dotenv(ROOT / ".env")
|
|
token = sys.argv[1]
|
|
directory = Path(sys.argv[2]) if len(sys.argv) > 2 else ROOT / "Banco"
|
|
out = []
|
|
for p in sorted(directory.glob("*.FDB")):
|
|
print("analyzing", p.name, flush=True)
|
|
out.append(analyze(p, token))
|
|
(ROOT / "docs" / "runs" / "multi_db" / f"{p.stem}_coverage.json").write_text(json.dumps(out[-1], ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
|
print(json.dumps({k: out[-1][k] for k in ("file", "TOTAL_PRODUCTS", "company")}, ensure_ascii=False), flush=True)
|
|
(ROOT / "docs" / "MULTI_DATABASE_FISCAL_COVERAGE_V1.json").write_text(json.dumps({"artifact": "MULTI_DATABASE_FISCAL_COVERAGE_V1", "firebird_writes": 0, "databases": out}, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
|
|
|
|
|
|
main()
|