chore: establish project baseline
This commit is contained in:
OpenCode
@@ -0,0 +1 @@
|
||||
"""Read-only ERP to Fiscal Hub agent."""
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Pure capability detection / schema classification for SGV (C000025/C000004) databases. No I/O, no writes."""
|
||||
|
||||
WRITE_TARGETS = ["CLASSIFICACAO_FISCAL", "CFOP_ENTRADA", "CST_ENTRADA", "CFOP", "CST", "ALIQUOTA", "CEST", "SITUACAO_TRIBUTARIA", "COD_ENTRADA_COFINS",
|
||||
"COD_SAIDA_COFINS", "ALIQ_COFINS", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "CSOSN", "IPI", "CST_IPI", "IAT", "INCIDENCIA_PISCOFINS",
|
||||
"PISCOFINS", "IPPT"]
|
||||
ORIGIN_FIELD = "COD_ORIGEM_MERCADORIA"
|
||||
|
||||
# Firebird RDB$FIELD_TYPE -> family
|
||||
FAMILY = {7: "int", 8: "int", 16: "int", 10: "float", 11: "float", 27: "float", 12: "date", 13: "time", 35: "timestamp", 14: "text", 37: "text", 261: "blob"}
|
||||
# accepted families per field (UI edit controls take text or numbers; numeric columns may be int/float)
|
||||
ACCEPT = {"ALIQUOTA": {"float", "int"}, "IPI": {"float", "int"}, "COD_ORIGEM_MERCADORIA": {"int"}}
|
||||
DEFAULT_ACCEPT = {"text", "int"} # text columns; some ERPs keep codes such as CST_ENTRADA in SMALLINT
|
||||
|
||||
CAPABILITIES = {
|
||||
"supports_ncm": ["CLASSIFICACAO_FISCAL"],
|
||||
"supports_cfop_entry": ["CFOP_ENTRADA"],
|
||||
"supports_cfop_exit": ["CFOP"],
|
||||
"supports_cst_entry": ["CST_ENTRADA"],
|
||||
"supports_cst_exit": ["CST"],
|
||||
"supports_icms_rate": ["ALIQUOTA"],
|
||||
"supports_cest": ["CEST"],
|
||||
"supports_csosn": ["CSOSN"],
|
||||
"supports_ipi": ["IPI", "CST_IPI"],
|
||||
"supports_pis": ["COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS"],
|
||||
"supports_cofins": ["COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS"],
|
||||
"supports_origin": [ORIGIN_FIELD],
|
||||
"supports_iat": ["IAT"],
|
||||
"supports_incidence": ["INCIDENCIA_PISCOFINS"],
|
||||
"supports_ippt": ["IPPT"],
|
||||
}
|
||||
|
||||
|
||||
def type_ok(field, type_code):
|
||||
return FAMILY.get(type_code, "other") in ACCEPT.get(field, DEFAULT_ACCEPT)
|
||||
|
||||
|
||||
def detect_capabilities(columns):
|
||||
"""columns: {FIELD: rdb_type_code} of C000025. capability=true only if every required column exists with a usable type."""
|
||||
out = {}
|
||||
for cap, fields in CAPABILITIES.items():
|
||||
out[cap] = all(f in columns and type_ok(f, columns[f]) for f in fields)
|
||||
return out
|
||||
|
||||
|
||||
def field_compatibility(columns, reference_types=None):
|
||||
"""Per write target (+ origin): PRESENT_OK / PRESENT_TYPE_VARIATION / MISSING."""
|
||||
res = {}
|
||||
for f in WRITE_TARGETS + [ORIGIN_FIELD]:
|
||||
if f not in columns:
|
||||
res[f] = "MISSING"
|
||||
elif not type_ok(f, columns[f]):
|
||||
res[f] = "PRESENT_TYPE_INCOMPATIBLE"
|
||||
elif reference_types and reference_types.get(f) is not None and reference_types[f] != columns[f]:
|
||||
res[f] = "PRESENT_TYPE_VARIATION"
|
||||
else:
|
||||
res[f] = "PRESENT_OK"
|
||||
return res
|
||||
|
||||
|
||||
def classify_schema(has_c000004, has_c000025, fingerprint, reference_fingerprint, compat):
|
||||
"""REFERENCE_SCHEMA_MATCH | COMPATIBLE_WITH_VARIATIONS | INCOMPATIBLE. A different hash is not a failure
|
||||
when every required capability exists (origin is optional: it is only PROBABLE in the map)."""
|
||||
if not (has_c000004 and has_c000025):
|
||||
return "INCOMPATIBLE"
|
||||
required = {f: v for f, v in compat.items() if f in WRITE_TARGETS}
|
||||
if any(v in ("MISSING", "PRESENT_TYPE_INCOMPATIBLE") for v in required.values()):
|
||||
return "INCOMPATIBLE"
|
||||
if fingerprint == reference_fingerprint:
|
||||
return "REFERENCE_SCHEMA_MATCH"
|
||||
return "COMPATIBLE_WITH_VARIATIONS"
|
||||
|
||||
|
||||
NAME_KEYWORDS = {"RESTAURANT": ("RESTAURANTE", "PIZZARIA", "LANCHONETE", "CHURRASC"), "RETAIL_MARKET": ("MERCADO", "SUPERMERC", "MERCEARIA", "ATACAD"),
|
||||
"APPAREL": ("MODAS", "CONFEC", "BOUTIQUE", "ROUPAS"), "CONSTRUCTION": ("CONSTRU", "MATERIAIS"), "AUTO_REPAIR": ("OFICINA", "MECANICA", "AUTO PECAS", "AUTOPECAS")}
|
||||
|
||||
|
||||
def infer_segment(signals, company_name=""):
|
||||
"""Evidence-based segment guess from populated tables/flags (never from the file name).
|
||||
signals: {"restaurant_tables_rows": int, "kitchen_products": int, "recipes": int, "service_orders": int, "vehicle_products": int, "grade_items": int, "nfce_items": int}"""
|
||||
ev = []
|
||||
score = {"RESTAURANT": 0, "AUTO_REPAIR": 0, "APPAREL": 0, "RETAIL_MARKET": 0, "CONSTRUCTION": 0}
|
||||
import unicodedata
|
||||
up = unicodedata.normalize("NFKD", company_name or "").encode("ascii", "ignore").decode().upper()
|
||||
for seg_name, kws in NAME_KEYWORDS.items():
|
||||
hit = [k for k in kws if k in up]
|
||||
if hit:
|
||||
score[seg_name] += 1; ev.append(f"company name in C000004 contains {hit[0]} (weak, name is data, not the file name)")
|
||||
if signals.get("restaurant_tables_rows", 0) > 50: score["RESTAURANT"] += 2; ev.append(f"mesas/comandas rows={signals['restaurant_tables_rows']}")
|
||||
if signals.get("kitchen_products", 0) > 0: score["RESTAURANT"] += 1; ev.append(f"PROD_COZINHA products={signals['kitchen_products']}")
|
||||
if signals.get("recipes", 0) > 0: score["RESTAURANT"] += 1; ev.append(f"recipes={signals['recipes']}")
|
||||
if signals.get("service_orders", 0) > 20: score["AUTO_REPAIR"] += 2; ev.append(f"ordens de serviço={signals['service_orders']}")
|
||||
if signals.get("vehicle_products", 0) > 0: score["AUTO_REPAIR"] += 1; ev.append(f"veículos={signals['vehicle_products']}")
|
||||
if signals.get("grade_items", 0) > 50: score["APPAREL"] += 2; ev.append(f"grade (tamanho/cor) rows={signals['grade_items']}")
|
||||
if signals.get("nfce_items", 0) > 1000 and max(score.values()) == 0: score["RETAIL_MARKET"] += 1; ev.append("high-volume NFC-e retail without restaurant/OS/grade signals")
|
||||
best = max(score, key=score.get)
|
||||
return (best if score[best] > 0 else "UNKNOWN"), score, ev
|
||||
@@ -0,0 +1,37 @@
|
||||
import httpx
|
||||
|
||||
class HubClient:
|
||||
def __init__(self, url, token, chunk_size=100): self.url, self.token, self.chunk_size = url.rstrip("/"), token, chunk_size
|
||||
def _headers(self): return {"Authorization": f"Bearer {self.token}"}
|
||||
def classify_batch(self, products):
|
||||
results = []
|
||||
for start in range(0, len(products), self.chunk_size):
|
||||
chunk = products[start:start + self.chunk_size]
|
||||
response = httpx.post(f"{self.url}/v1/classify/batch", json={"products": chunk}, headers=self._headers(), timeout=60)
|
||||
response.raise_for_status(); results.extend(response.json()["results"])
|
||||
return results
|
||||
def get_json(self, path):
|
||||
response = httpx.get(f"{self.url}{path}", headers=self._headers(), timeout=30)
|
||||
response.raise_for_status(); return response.json()
|
||||
def health(self):
|
||||
response = httpx.get(f"{self.url}/health", timeout=10)
|
||||
return response.status_code == 200
|
||||
def heartbeat(self, status, schema_fingerprint=None, agent_version=None, last_job_id=None):
|
||||
payload = {"status": status, "schema_fingerprint": schema_fingerprint, "agent_version": agent_version, "last_job_id": last_job_id}
|
||||
response = httpx.post(f"{self.url}/v1/agents/heartbeat", json=payload, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def next_job(self):
|
||||
response = httpx.get(f"{self.url}/v1/jobs/next", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def create_job(self, kind, payload=None):
|
||||
response = httpx.post(f"{self.url}/v1/jobs", json={"kind": kind, "payload": payload or {}}, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def get_job(self, job_id):
|
||||
response = httpx.get(f"{self.url}/v1/jobs/{job_id}", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def complete_job(self, job_id, result_summary):
|
||||
response = httpx.post(f"{self.url}/v1/jobs/{job_id}/complete", json={"result_summary": result_summary}, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def fail_job(self, job_id, error):
|
||||
response = httpx.post(f"{self.url}/v1/jobs/{job_id}/fail", json={"error": str(error)}, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Deterministic, auditable company-context resolution for an Agent installation (READ-ONLY).
|
||||
|
||||
Evidence (proved on 4 SGV databases, see docs/COMPANY_CONTEXT_RESOLUTION_V1.json):
|
||||
* EMISSOR.EMITENTE_CNPJ/UF/CRT is the emitter block read by the emission programs (GtiVendas_NFCe.exe and Envio NF-e.exe both bind
|
||||
QrEmissor.EMITENTE_*; neither reads C000004.CRT/OPTANTE_SIMPLES). 100% of emitted NFC-e access keys carry the CNPJ of the C000004 row
|
||||
whose CNPJ equals EMISSOR.EMITENTE_CNPJ, and cUF=29.
|
||||
* C000004.CRT / OPTANTE_SIMPLES belong to the ERP filial registry (SPED, filial screen) and can disagree with EMISSOR.
|
||||
Never guesses: first row / lowest code / latest row are NOT used. Ambiguity -> FAIL with the reason, unless the operator configures
|
||||
COMPANY_CODE or COMPANY_CNPJ (company) and COMPANY_CRT_SOURCE = EMISSOR | C000004 (explicit choice when the two CRT sources conflict).
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
IBGE_UF = {"11": "RO", "12": "AC", "13": "AM", "14": "RR", "15": "PA", "16": "AP", "17": "TO", "21": "MA", "22": "PI", "23": "CE", "24": "RN", "25": "PB", "26": "PE",
|
||||
"27": "AL", "28": "SE", "29": "BA", "31": "MG", "32": "ES", "33": "RJ", "35": "SP", "41": "PR", "42": "SC", "43": "RS", "50": "MS", "51": "MT", "52": "GO",
|
||||
"53": "DF"}
|
||||
VALID_CRT = {"1", "2", "3"}
|
||||
|
||||
|
||||
class CompanyContextError(Exception):
|
||||
def __init__(self, result):
|
||||
super().__init__(f"COMPANY_CONTEXT=FAIL ({result.get('classification')}): {'; '.join(result.get('reasons', []))}")
|
||||
self.result = result
|
||||
|
||||
|
||||
def digits(value):
|
||||
return "".join(c for c in str(value or "") if c.isdigit())
|
||||
|
||||
|
||||
def mask_cnpj(value):
|
||||
d = digits(value)
|
||||
return f"{d[:2]}**********{d[-2:]}" if len(d) >= 4 else None
|
||||
|
||||
|
||||
def cnpj_fingerprint(value):
|
||||
d = digits(value)
|
||||
return hashlib.sha256(d.encode()).hexdigest()[:8] if d else None
|
||||
|
||||
|
||||
def _s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def config_from_env(env):
|
||||
crt_source = (env.get("COMPANY_CRT_SOURCE") or "").strip().upper() or None
|
||||
if crt_source == "AUTO": # AUTO = resolve automatically, only when there is no conflict
|
||||
crt_source = None
|
||||
invalid = False
|
||||
else:
|
||||
invalid = bool(crt_source and crt_source not in ("EMISSOR", "C000004"))
|
||||
uf = (env.get("COMPANY_UF") or "").strip().upper() or None
|
||||
return {"company_code": (env.get("COMPANY_CODE") or "").strip() or None, "company_cnpj": digits(env.get("COMPANY_CNPJ")) or None,
|
||||
"crt_source": crt_source if crt_source in ("EMISSOR", "C000004") else None, "crt_source_invalid": invalid,
|
||||
"uf_override": uf}
|
||||
|
||||
|
||||
def resolve(evidence, config=None):
|
||||
"""evidence: {"c000004": [{code, cnpj, ie, uf, crt, optante, name}], "emissor": [{cnpj, uf, crt}],
|
||||
"nfce_keys": [{"cuf", "cnpj", "n"}], "nf_by_filial": {code: n}}. Pure function."""
|
||||
cfg = config or {}
|
||||
out = {"status": "FAIL", "classification": "UNRESOLVED", "company_code": None, "company_cnpj": None, "company_cnpj_fingerprint": None, "company_uf": None, "company_crt": None,
|
||||
"source_of_truth": {}, "confidence": "NONE", "evidence": [], "warnings": [], "reasons": [], "configured": False}
|
||||
|
||||
def fail(cls, reason):
|
||||
out["classification"] = cls; out["reasons"].append(reason); return out
|
||||
|
||||
if cfg.get("crt_source_invalid"):
|
||||
return fail("CONFIG_REQUIRED", "COMPANY_CRT_SOURCE must be EMISSOR or C000004")
|
||||
companies = [{**c, "cnpj": digits(c.get("cnpj"))} for c in evidence.get("c000004", [])]
|
||||
emissors = [{**e, "cnpj": digits(e.get("cnpj"))} for e in evidence.get("emissor", [])]
|
||||
docs = evidence.get("nfce_keys", [])
|
||||
if not companies:
|
||||
return fail("UNRESOLVED", "C000004 has no company rows")
|
||||
|
||||
# ---- company selection -------------------------------------------------------------------
|
||||
selected = None; basis = None
|
||||
by_code, by_cnpj = cfg.get("company_code"), cfg.get("company_cnpj")
|
||||
if by_code or by_cnpj:
|
||||
out["configured"] = True
|
||||
cand = [c for c in companies if (not by_code or c["code"] == by_code) and (not by_cnpj or c["cnpj"] == by_cnpj)]
|
||||
if len(cand) != 1:
|
||||
return fail("CONFIG_REQUIRED", "configured company (COMPANY_CODE/COMPANY_CNPJ) does not identify exactly one C000004 row")
|
||||
selected, basis = cand[0], "CONFIGURED"
|
||||
elif len(companies) == 1:
|
||||
selected, basis = companies[0], "SINGLE_C000004_ROW"
|
||||
elif len(emissors) == 1 and emissors[0]["cnpj"]:
|
||||
match = [c for c in companies if c["cnpj"] == emissors[0]["cnpj"]]
|
||||
if len(match) != 1:
|
||||
return fail("CONFIG_REQUIRED", f"{len(companies)} C000004 rows and the EMISSOR CNPJ matches {len(match)} of them: configure COMPANY_CODE or COMPANY_CNPJ")
|
||||
selected, basis = match[0], "EMISSOR_CNPJ_MATCH"
|
||||
else:
|
||||
return fail("CONFIG_REQUIRED", f"{len(companies)} C000004 rows and no unique EMISSOR row to disambiguate: configure COMPANY_CODE or COMPANY_CNPJ")
|
||||
out["evidence"].append(f"company selected by {basis}")
|
||||
out.update(company_code=selected["code"], company_cnpj=mask_cnpj(selected["cnpj"]), company_cnpj_fingerprint=cnpj_fingerprint(selected["cnpj"]))
|
||||
out["source_of_truth"]["company"] = basis
|
||||
|
||||
# ---- emitter (EMISSOR) must describe the same company --------------------------------------
|
||||
emissor = None
|
||||
if emissors:
|
||||
same = [e for e in emissors if e["cnpj"] and e["cnpj"] == selected["cnpj"]]
|
||||
if same:
|
||||
emissor = same[0]; out["evidence"].append("EMISSOR.EMITENTE_CNPJ equals the selected company CNPJ")
|
||||
else:
|
||||
return fail("CONFLICT", "EMISSOR emitter CNPJ differs from the selected company (unexplained): refusing to guess")
|
||||
else:
|
||||
out["warnings"].append("NO_EMISSOR_ROW")
|
||||
|
||||
# ---- emitted documents must belong to the same CNPJ ------------------------------------------
|
||||
doc_ufs = set()
|
||||
if docs:
|
||||
total = sum(d["n"] for d in docs); own = sum(d["n"] for d in docs if digits(d["cnpj"]) == selected["cnpj"])
|
||||
out["evidence"].append(f"emitted NFC-e keys: {own}/{total} carry the selected CNPJ")
|
||||
if own == 0:
|
||||
return fail("CONFLICT", f"none of the {total} emitted NFC-e keys carries the selected company CNPJ")
|
||||
if own != total: # e.g. a database copy/migration that kept another emitter's history: evidence, not a reason to guess
|
||||
out["warnings"].append(f"HISTORICAL_NFCE_KEYS_FROM_OTHER_CNPJ {total - own}/{total}")
|
||||
doc_ufs = {IBGE_UF.get(d["cuf"]) for d in docs if digits(d["cnpj"]) == selected["cnpj"]} - {None}
|
||||
|
||||
# ---- UF ------------------------------------------------------------------------------------------
|
||||
uf_sources = {}
|
||||
if emissor and _s(emissor.get("uf")): uf_sources["EMISSOR"] = _s(emissor["uf"]).upper()
|
||||
if _s(selected.get("uf")): uf_sources["C000004"] = _s(selected["uf"]).upper()
|
||||
if len(doc_ufs) == 1: uf_sources["DOCUMENT_KEYS"] = next(iter(doc_ufs))
|
||||
if len(doc_ufs) > 1: return fail("CONFLICT", "emitted NFC-e keys carry several UFs")
|
||||
values = set(uf_sources.values())
|
||||
override = cfg.get("uf_override")
|
||||
if len(values) > 1:
|
||||
return fail("CONFLICT", f"UF_SOURCE_CONFLICT {uf_sources}")
|
||||
if override and values and override not in values:
|
||||
return fail("CONFLICT", f"COMPANY_UF={override} differs from the UF proved by the database {sorted(values)}: an override never replaces the resolver silently")
|
||||
if not values and override: # explicit override only when the database proves nothing
|
||||
uf_sources["CONFIG_OVERRIDE"] = override; values = {override}; out["warnings"].append("UF_FROM_EXPLICIT_OVERRIDE")
|
||||
if not values:
|
||||
return fail("UNRESOLVED", "UF_SOURCE_UNKNOWN")
|
||||
out["company_uf"] = next(iter(values)); out["source_of_truth"]["uf"] = "+".join(uf_sources)
|
||||
out["uf_status"] = "UF_SOURCE_CONFIRMED"
|
||||
|
||||
# ---- CRT ------------------------------------------------------------------------------------------
|
||||
e_crt = _s(emissor.get("crt")) if emissor else None
|
||||
c_crt = _s(selected.get("crt"))
|
||||
e_ok, c_ok = e_crt in VALID_CRT, c_crt in VALID_CRT
|
||||
if e_ok and c_ok and e_crt == c_crt:
|
||||
crt, src, status = e_crt, "EMISSOR+C000004", "CRT_SOURCE_CONFIRMED"
|
||||
elif e_ok and c_ok:
|
||||
choice = cfg.get("crt_source")
|
||||
if not choice:
|
||||
out["crt_candidates"] = {"EMISSOR": e_crt, "C000004": c_crt}
|
||||
return fail("CONFLICT", f"CRT_SOURCE_CONFLICT EMISSOR={e_crt} C000004={c_crt}: configure COMPANY_CRT_SOURCE (emission programs read EMISSOR)")
|
||||
crt, src, status = (e_crt if choice == "EMISSOR" else c_crt), f"{choice} (configured; conflict acknowledged)", "CRT_SOURCE_CONFLICT_RESOLVED_BY_CONFIG"
|
||||
out["warnings"].append(f"CRT_CONFLICT EMISSOR={e_crt} C000004={c_crt}")
|
||||
elif e_ok or c_ok:
|
||||
crt, src, status = (e_crt, "EMISSOR", "CRT_SOURCE_SINGLE") if e_ok else (c_crt, "C000004", "CRT_SOURCE_SINGLE")
|
||||
out["warnings"].append("CRT_HAS_A_SINGLE_SOURCE")
|
||||
else:
|
||||
return fail("UNRESOLVED", "CRT_SOURCE_UNKNOWN (no valid CRT in EMISSOR or C000004)")
|
||||
out.update(company_crt=crt, crt_status=status); out["source_of_truth"]["crt"] = src
|
||||
opt = _s(selected.get("optante"))
|
||||
if opt and ((crt in ("1", "2") and opt.upper() != "S") or (crt == "3" and opt.upper() == "S")):
|
||||
out["warnings"].append(f"OPTANTE_SIMPLES={opt} inconsistent with CRT={crt}")
|
||||
|
||||
agree = len(uf_sources) + (2 if status == "CRT_SOURCE_CONFIRMED" else 1)
|
||||
out["confidence"] = "HIGH" if (status == "CRT_SOURCE_CONFIRMED" and len(uf_sources) >= 2) else "MEDIUM"
|
||||
out["status"] = "PASS"
|
||||
out["classification"] = "CONFIGURED" if out["configured"] or status == "CRT_SOURCE_CONFLICT_RESOLVED_BY_CONFIG" else "AUTO_RESOLVED"
|
||||
out["reasons"] = []
|
||||
out["evidence_source_count"] = agree
|
||||
return out
|
||||
|
||||
|
||||
def collect_evidence(cursor):
|
||||
"""Read-only SELECTs. Missing optional tables/columns simply yield less evidence."""
|
||||
cursor.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000004','EMISSOR','NFCE','C000061')")
|
||||
cols = {}
|
||||
for t, f in cursor.fetchall():
|
||||
cols.setdefault(t, set()).add(f)
|
||||
ev = {"c000004": [], "emissor": [], "nfce_keys": [], "nf_by_filial": {}}
|
||||
c4 = cols.get("C000004", set())
|
||||
want = [("CODIGO", "code"), ("CNPJ", "cnpj"), ("IE", "ie"), ("UF", "uf"), ("CRT", "crt"), ("OPTANTE_SIMPLES", "optante"), ("FILIAL", "name")]
|
||||
use = [(c, k) for c, k in want if c in c4]
|
||||
if use:
|
||||
cursor.execute(f"SELECT {','.join(c for c, _ in use)} FROM C000004")
|
||||
for row in cursor.fetchall():
|
||||
ev["c000004"].append({k: _s(v) for (_, k), v in zip(use, row)})
|
||||
em = cols.get("EMISSOR", set())
|
||||
want = [("EMITENTE_CNPJ", "cnpj"), ("EMITENTE_UF", "uf"), ("EMITENTE_CRT", "crt")]
|
||||
use = [(c, k) for c, k in want if c in em]
|
||||
if use:
|
||||
cursor.execute(f"SELECT {','.join(c for c, _ in use)} FROM EMISSOR")
|
||||
for row in cursor.fetchall():
|
||||
ev["emissor"].append({k: _s(v) for (_, k), v in zip(use, row)})
|
||||
if "CHAVE" in cols.get("NFCE", set()):
|
||||
cursor.execute("SELECT SUBSTRING(CHAVE FROM 1 FOR 2), SUBSTRING(CHAVE FROM 7 FOR 14), COUNT(*) FROM NFCE WHERE CHAVE IS NOT NULL AND CHAR_LENGTH(TRIM(CHAVE)) = 44 "
|
||||
"GROUP BY SUBSTRING(CHAVE FROM 1 FOR 2), SUBSTRING(CHAVE FROM 7 FOR 14)")
|
||||
ev["nfce_keys"] = [{"cuf": _s(a), "cnpj": _s(b), "n": n} for a, b, n in cursor.fetchall()]
|
||||
if "CODFILIAL" in cols.get("C000061", set()):
|
||||
cursor.execute("SELECT CODFILIAL, COUNT(*) FROM C000061 GROUP BY CODFILIAL")
|
||||
ev["nf_by_filial"] = {_s(a): n for a, n in cursor.fetchall()}
|
||||
return ev
|
||||
|
||||
|
||||
def resolve_from_cursor(cursor, env=None):
|
||||
return resolve(collect_evidence(cursor), config_from_env(env or os.environ))
|
||||
|
||||
|
||||
def preflight_company_context(env=None):
|
||||
"""Opens the configured Firebird database read-only and resolves the company. Never prints the full CNPJ."""
|
||||
from .firebird_reader import _connect
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
return resolve_from_cursor(conn.cursor(), env)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def preflight_report(result):
|
||||
keys = ("company_code", "company_cnpj", "company_uf", "company_crt", "source_of_truth", "confidence", "classification", "warnings", "reasons")
|
||||
return {"COMPANY_CONTEXT": "PASS" if result["status"] == "PASS" else "FAIL", **{k: result.get(k) for k in keys}}
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Read-only fiscal coverage report for a set of products (used by the Agent's REPROCESS_ALL result and by the multi-database tooling).
|
||||
|
||||
Pure computation over Hub decisions + product rows + local evidence. It reuses the shipped pure engines in central.app
|
||||
(fiscal_decision, product_defaults, icms_context) — no fiscal rule lives here, only guards for the company context:
|
||||
the BA Anexo 1 ST/MVA dataset is applied to UF=BA only and CSOSN logic only under CRT=1.
|
||||
"""
|
||||
from collections import Counter
|
||||
|
||||
PRODUCT_COLUMNS = ("CODIGO", "PRODUTO", "CLASSIFICACAO_FISCAL", "CFOP", "CSOSN", "CST", "CEST", "IPI", "EX_TIPI", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA")
|
||||
|
||||
|
||||
def _s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def compute_coverage(products, decisions, company, company_default, refs, evidence):
|
||||
"""products/decisions aligned lists. company: {"uf","crt"} (resolved). evidence: {"recipes","composed","purchases","stock_entry"}."""
|
||||
from central.app.fiscal_decision import decide_product
|
||||
from central.app.icms_context import csosn_v2, product_role, st_status, withheld_status
|
||||
from central.app.product_defaults import cfop_default
|
||||
uf_ba = company.get("uf") == "BA"
|
||||
crt1 = company.get("crt") == "1"
|
||||
recipes, composed = evidence.get("recipes", set()), evidence.get("composed", set())
|
||||
purchases, stock_entry = evidence.get("purchases", {}), evidence.get("stock_entry", {})
|
||||
m, cfop_values, situ = Counter(), Counter(), Counter()
|
||||
for p, d in zip(products, decisions):
|
||||
p = {c: p.get(c) for c in PRODUCT_COLUMNS}
|
||||
code = _s(p["CODIGO"]); p["CODIGO"] = code
|
||||
cls = d.get("classification") or {}
|
||||
rec = decide_product(p, d, {"uf": company.get("uf") or "", "crt": company.get("crt") or ""}, refs)
|
||||
ncm_status = d.get("status"); raw_ncm = _s(p["CLASSIFICACAO_FISCAL"])
|
||||
m["NCM_MISSING" if not raw_ncm else ("NCM_VALID" if ncm_status == "EXACT_NCM" else ("NCM_LEGACY" if ncm_status == "LEGACY_NCM" else "NCM_INVALID"))] += 1
|
||||
ipi = rec["IPI"]; rk = (cls.get("ipi") or {}).get("rate_kind"); ist = d.get("ipi_status")
|
||||
if ist == "IPI_RESOLVED":
|
||||
m[{"NUMERIC": "IPI_NUMERIC", "NON_NUMERIC": "IPI_NON_NUMERIC", "BLANK": "IPI_SOURCE_BLANK"}.get(rk, "IPI_NO_MATCH")] += 1
|
||||
elif ist == "IPI_CANDIDATES":
|
||||
m["IPI_AMBIGUOUS"] += 1
|
||||
elif ist != "LEGACY_NCM" and raw_ncm:
|
||||
m["IPI_NO_MATCH"] += 1
|
||||
if ipi["action_class"] == "ALREADY_CORRECT":
|
||||
m["IPI_ALREADY_CORRECT"] += 1
|
||||
if ipi["write_eligible"]:
|
||||
m["IPI_CHANGE_REQUIRED"] += 1; m["IPI_WRITE_ELIGIBLE"] += 1
|
||||
cs_ = d.get("cest_status")
|
||||
if cs_ == "CEST_CONFIRMED":
|
||||
m["CEST_CONFIRMED"] += 1
|
||||
elif cs_ == "CEST_CANDIDATES":
|
||||
m["CEST_CANDIDATES"] += 1
|
||||
cf = cfop_default(p, ncm_status, d.get("ba_st_mva_status") if uf_ba else None, refs["cfop_valid"])
|
||||
cfop_values[_s(p["CFOP"])] += 1
|
||||
m["CFOP_RESOLVED" if cf["status"] == "CFOP_DEFAULT_RESOLVED" else "CFOP_CONTEXT_REQUIRED"] += 1
|
||||
ba = cls.get("ba_st_mva") or {}
|
||||
if uf_ba:
|
||||
st, _ = st_status(ncm_status, d.get("ba_st_mva_status"), cs_, p["CEST"], ba.get("cest"), None)
|
||||
else:
|
||||
st = "ST_UNKNOWN"
|
||||
m[st] += 1
|
||||
wh, _ = withheld_status(purchases.get(code, []))
|
||||
m[{"ST_ALREADY_WITHHELD_CONFIRMED": "ST_WITHHOLDING_CONFIRMED", "ST_ALREADY_WITHHELD_NOT_CONFIRMED": "ST_WITHHOLDING_NOT_CONFIRMED"}.get(wh, "ST_WITHHOLDING_UNKNOWN")] += 1
|
||||
role, _ = product_role(p["TIPO"], p["PROD_COZINHA"] == 1, code in recipes, code in composed, bool(_s(p["PRODUTO_SERVICO"])), bool(purchases.get(code)) or stock_entry.get(code, False))
|
||||
m["PRODUCT_ROLE_KNOWN" if role != "UNKNOWN" else "PRODUCT_ROLE_UNKNOWN"] += 1
|
||||
m["ROLE_" + role] += 1
|
||||
state = csosn_v2(p["CSOSN"], company_default, ncm_status, st if uf_ba else "ST_UNKNOWN", wh, role, refs["csosn"])["status"] if crt1 else "CSOSN_CONTEXT_REQUIRED"
|
||||
m["CSOSN_RESOLVED" if state == "CSOSN_RESOLVED" else ("CSOSN_CANDIDATES" if state == "CSOSN_CANDIDATE" else "CSOSN_CONTEXT_REQUIRED")] += 1
|
||||
situ[_s(p["SITUACAO_TRIBUTARIA"])] += 1
|
||||
n = len(products)
|
||||
fill = lambda f: sum(1 for p in products if _s(p.get(f)) not in (None, "")) # noqa: E731
|
||||
return {"TOTAL_PRODUCTS": n, "metrics": dict(m),
|
||||
"IPI_RESOLVED": m["IPI_NUMERIC"] + m["IPI_NON_NUMERIC"] + m["IPI_SOURCE_BLANK"],
|
||||
"quality": {"ncm_valid_share": round(m["NCM_VALID"] / n, 4) if n else None, "cest_filled": fill("CEST"), "csosn_filled": fill("CSOSN"), "cfop_filled": fill("CFOP"), "ipi_filled": fill("IPI"),
|
||||
"cfop_distinct": len(cfop_values), "cfop_top": cfop_values.most_common(6), "situacao_tributaria": dict(situ.most_common(6)),
|
||||
"purchase_items_C000088": sum(len(v) for v in purchases.values()), "products_with_purchase_evidence": len(purchases), "recipes": len(recipes), "composed_products": len(composed)},
|
||||
"guards_applied": {"st_dataset_ba_only": not uf_ba, "csosn_requires_crt1": not crt1}}
|
||||
|
||||
|
||||
def build_refs(get, products, company_uf):
|
||||
"""Reference-data snapshot from the Hub (`get(path) -> json`). Used by the engines for domain validity only."""
|
||||
refs = {}
|
||||
for key, domain in (("csosn", "CSOSN"), ("cst_icms", "CST_ICMS"), ("cst_pis", "CST_PIS"), ("cst_cofins", "CST_COFINS"), ("origem", "ORIGEM_MERCADORIA")):
|
||||
refs[key] = set(get(f"/v1/reference/fiscal_codes/{domain}")["codes"])
|
||||
valid = set()
|
||||
for code in {_s(p.get(f)) for p in products for f in ("CFOP", "CFOP_ENTRADA") if _s(p.get(f))}:
|
||||
if get(f"/v1/reference/cfop/{code}")["found"]:
|
||||
valid.add(code)
|
||||
refs["cfop_valid"] = valid
|
||||
refs["fcp_uf"] = {}
|
||||
fcp = get(f"/v1/reference/fcp/{company_uf}")
|
||||
if fcp.get("found"):
|
||||
refs["fcp_uf"][company_uf] = fcp
|
||||
refs["cbs_rate_2026"] = get("/v1/reference/rtc_cbs_rate/2026").get("rate")
|
||||
refs["cclasstrib_count"] = sum(d["record_count"] for d in get("/v1/datasets/status") if d["kind"] == "RTC_CCLASSTRIB" and d["status"] == "ACTIVE")
|
||||
return refs
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Shared read-only Firebird access for the Agent: full table or a CODIGO subset."""
|
||||
import os
|
||||
|
||||
FIELDS = "CODIGO,PRODUTO,GTIN AS EAN_GTIN,NCM,CLASSIFICACAO_FISCAL,CFOP,CST,ALIQUOTA,CEST,CSOSN,IPI,EX_TIPI"
|
||||
# Present in every audited SGV database but selected only when the column exists (capability-based, never assumed).
|
||||
OPTIONAL_FIELDS = ("TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA")
|
||||
|
||||
|
||||
def _select_fields(cursor):
|
||||
cursor.execute("SELECT TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME = 'C000025'")
|
||||
have = {r[0] for r in cursor.fetchall()}
|
||||
return FIELDS + "".join(f",{c}" for c in OPTIONAL_FIELDS if c in have)
|
||||
|
||||
|
||||
def _connect(env, read_only=True):
|
||||
import fdb
|
||||
kwargs = dict(
|
||||
database=env["FIREBIRD_DATABASE_PATH"],
|
||||
user=env["FB_USER"],
|
||||
password=env["FB_PASSWORD"],
|
||||
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
if read_only:
|
||||
kwargs["isolation_level"] = fdb.ISOLATION_LEVEL_READ_COMMITED_RO
|
||||
return fdb.connect(**kwargs)
|
||||
|
||||
|
||||
def check_connection(env=None):
|
||||
"""Best-effort read-only connectivity probe. Returns True/False, never raises."""
|
||||
env = env or os.environ
|
||||
try:
|
||||
conn = _connect(env)
|
||||
try:
|
||||
conn.cursor().execute("SELECT 1 FROM RDB$DATABASE")
|
||||
finally:
|
||||
conn.close()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def read_products(env=None, codigos=None):
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
fields = _select_fields(cursor)
|
||||
if codigos:
|
||||
placeholders = ",".join("?" for _ in codigos)
|
||||
cursor.execute(f"SELECT {fields} FROM C000025 WHERE CODIGO IN ({placeholders}) ORDER BY CODIGO", tuple(codigos))
|
||||
else:
|
||||
cursor.execute(f"SELECT {fields} FROM C000025 ORDER BY CODIGO")
|
||||
names = [column[0].strip() for column in cursor.description]
|
||||
return [dict(zip(names, row)) for row in cursor.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def parse_company_context(rows):
|
||||
"""Pure: rows of (UF, CRT) from C000004 -> {"uf","crt"}. Fails closed (ValueError) if absent,
|
||||
blank or ambiguous — the engines never fall back to a hardcoded UF/CRT."""
|
||||
pairs = {(str(uf or "").strip().upper(), str(crt or "").strip()) for uf, crt in rows}
|
||||
if len(pairs) != 1:
|
||||
raise ValueError(f"company context unavailable or ambiguous ({len(pairs)} distinct UF/CRT rows in C000004)")
|
||||
uf, crt = next(iter(pairs))
|
||||
if len(uf) != 2 or crt not in ("1", "2", "3"):
|
||||
raise ValueError(f"company context invalid: UF={uf!r} CRT={crt!r}")
|
||||
return {"uf": uf, "crt": crt}
|
||||
|
||||
|
||||
def read_company_context(env=None):
|
||||
"""Resolve the ACTIVE company (UF/CRT) with the deterministic resolver (see company_context.py); fails closed with
|
||||
CompanyContextError when the context is ambiguous, conflicting or unresolved. No hardcoded company values, no first-row fallback."""
|
||||
from .company_context import CompanyContextError, preflight_company_context
|
||||
result = preflight_company_context(env or os.environ)
|
||||
if result["status"] != "PASS":
|
||||
raise CompanyContextError(result)
|
||||
return {"uf": result["company_uf"], "crt": result["company_crt"], "company_code": result["company_code"], "source_of_truth": result["source_of_truth"], "confidence": result["confidence"]}
|
||||
|
||||
|
||||
def read_evidence(env=None, selected_company_code=None):
|
||||
"""Read-only evidence tables used by the coverage report (purchases, stock entries, recipes, compositions) and the company's
|
||||
default CSOSN. Every table/column is optional: a missing one just means less evidence (capability-based)."""
|
||||
from collections import defaultdict
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000004','C000032','C000088','C000100','RECEITA_INGREDIENTES')")
|
||||
cols = defaultdict(set)
|
||||
for t, f in cur.fetchall():
|
||||
cols[t].add(f)
|
||||
has = lambda t, *c: t in cols and all(x in cols[t] for x in c) # noqa: E731
|
||||
s = lambda v: None if v is None else str(v).strip() # noqa: E731
|
||||
ev = {"recipes": set(), "composed": set(), "purchases": defaultdict(list), "stock_entry": {}, "stock_in": {}, "company_default_csosn": None}
|
||||
if has("RECEITA_INGREDIENTES", "CODRECEITA"):
|
||||
cur.execute("SELECT CODRECEITA FROM RECEITA_INGREDIENTES WHERE CODRECEITA IS NOT NULL")
|
||||
ev["recipes"] = {str(r[0]).zfill(6) for r in cur.fetchall()}
|
||||
if has("C000032", "CODPRODUTO", "TIPO"):
|
||||
cur.execute("SELECT DISTINCT CODPRODUTO FROM C000032 WHERE TIPO = 'COMPOSICAO'")
|
||||
ev["composed"] = {s(r[0]) for r in cur.fetchall()}
|
||||
if has("C000088", "CODPRODUTO", "ICMS_RETIDO", "ICMS_VALORRETIDO", "SUB_VALOR"):
|
||||
cur.execute("SELECT CODPRODUTO, ICMS_RETIDO, ICMS_VALORRETIDO, SUB_VALOR FROM C000088")
|
||||
for cod, ret, vr, sv in cur.fetchall():
|
||||
ev["purchases"][s(cod)].append({"icms_retido": s(ret), "valor_retido": vr, "sub_valor": sv})
|
||||
if has("C000100", "CODPRODUTO", "ULTIMA_ENTRADA", "NOTA_FISCAL"):
|
||||
cur.execute("SELECT CODPRODUTO, ULTIMA_ENTRADA, NOTA_FISCAL FROM C000100")
|
||||
ev["stock_entry"] = {s(r[0]): (r[1] is not None or s(r[2]) not in (None, "", "000000")) for r in cur.fetchall()}
|
||||
if has("C000100", "CODPRODUTO", "ENTRADAS"):
|
||||
cur.execute("SELECT CODPRODUTO, ENTRADAS FROM C000100")
|
||||
ev["stock_in"] = {s(r[0]): bool(r[1] and float(r[1]) > 0) for r in cur.fetchall()}
|
||||
if has("C000004", "ICMS_CSOSN", "CODIGO"):
|
||||
cur.execute("SELECT CODIGO, ICMS_CSOSN FROM C000004")
|
||||
rows = {s(a): s(b) for a, b in cur.fetchall()}
|
||||
ev["company_default_csosn"] = rows.get(selected_company_code) if selected_company_code else (next(iter(rows.values())) if len(rows) == 1 else None)
|
||||
ev["purchases"] = dict(ev["purchases"])
|
||||
return ev
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Decide whether a classified product's IPI rate may be auto-applied.
|
||||
|
||||
Scope is deliberately narrow: C000025.IPI only, and only when the Hub's TIPI
|
||||
match is a single unambiguous numeric rate. Never auto-applies NT/non-numeric
|
||||
markers, blank source rates, ambiguous EX candidates, or legacy/invalid NCM.
|
||||
"""
|
||||
|
||||
|
||||
def ipi_write_decision(decision, current_ipi):
|
||||
"""current_ipi must be a value freshly read from Firebird immediately before
|
||||
this call — None here means the ERP field is genuinely NULL right now, not
|
||||
that it wasn't read."""
|
||||
if decision.get("status") == "LEGACY_NCM":
|
||||
return {"eligible": False, "reason_code": "LEGACY_NCM", "proposed_value": None}
|
||||
if decision.get("ipi_status") != "IPI_RESOLVED":
|
||||
return {"eligible": False, "reason_code": f"IPI_STATUS_{decision.get('ipi_status')}", "proposed_value": None}
|
||||
|
||||
ipi = (decision.get("classification") or {}).get("ipi")
|
||||
if not ipi:
|
||||
return {"eligible": False, "reason_code": "NO_IPI_DECISION", "proposed_value": None}
|
||||
|
||||
rate_kind = ipi.get("rate_kind")
|
||||
if rate_kind != "NUMERIC":
|
||||
return {"eligible": False, "reason_code": f"RATE_KIND_{rate_kind}", "proposed_value": None}
|
||||
|
||||
proposed_value = ipi.get("rate_numeric")
|
||||
if proposed_value is None:
|
||||
return {"eligible": False, "reason_code": "NO_NUMERIC_VALUE", "proposed_value": None}
|
||||
|
||||
if current_ipi is not None and str(current_ipi).strip() == str(proposed_value).strip():
|
||||
return {"eligible": False, "reason_code": "SAME_VALUE", "proposed_value": proposed_value}
|
||||
|
||||
return {"eligible": True, "reason_code": "CHANGE_REQUIRED", "proposed_value": proposed_value}
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Shared per-product IPI auto-apply pipeline: Hub decision -> dry-run ->
|
||||
guarded write -> local SQLite audit. Used by both the manual CLI
|
||||
(run_agent_auto_apply.py) and the long-running job-polling service
|
||||
(run_agent_service.py) so the processing logic exists in exactly one place.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .ipi_auto_apply import ipi_write_decision
|
||||
from .runner import product_fingerprint
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def run_pipeline(products, decisions, state, firebird_writes, write_connection, executor, database_path, authorized_path=None):
|
||||
"""Process every (product, decision) pair. Returns a summary dict.
|
||||
|
||||
products come from a fresh Firebird read (current IPI is always "known").
|
||||
decisions come from the Hub's /v1/classify/batch, one per product, same order.
|
||||
"""
|
||||
summary = {
|
||||
"processed": 0,
|
||||
"auto_apply_eligible": 0,
|
||||
"writes_attempted": 0,
|
||||
"writes_succeeded": 0,
|
||||
"writes_failed": 0,
|
||||
"skipped_not_eligible": 0,
|
||||
"no_ops": 0,
|
||||
}
|
||||
for product, decision in zip(products, decisions):
|
||||
code = product["CODIGO"]
|
||||
fingerprint = product_fingerprint(product)
|
||||
current_ipi = product.get("IPI")
|
||||
summary["processed"] += 1
|
||||
|
||||
wd = ipi_write_decision(decision, current_ipi)
|
||||
decision_status = decision.get("status")
|
||||
ipi_status = decision.get("ipi_status")
|
||||
|
||||
if not firebird_writes:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "DRY_RUN_ONLY", current_ipi, wd["proposed_value"], None, wd["reason_code"], now())
|
||||
if wd["eligible"]:
|
||||
summary["auto_apply_eligible"] += 1
|
||||
elif wd["reason_code"] == "SAME_VALUE":
|
||||
summary["no_ops"] += 1
|
||||
continue
|
||||
|
||||
if not wd["eligible"]:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "SKIPPED_NOT_ELIGIBLE", current_ipi, wd["proposed_value"], None, wd["reason_code"], now())
|
||||
summary["skipped_not_eligible"] += 1
|
||||
if wd["reason_code"] == "SAME_VALUE":
|
||||
summary["no_ops"] += 1
|
||||
continue
|
||||
|
||||
summary["auto_apply_eligible"] += 1
|
||||
summary["writes_attempted"] += 1
|
||||
proposal = {
|
||||
"product_code": code,
|
||||
"field": "IPI",
|
||||
"current_value": current_ipi,
|
||||
"proposed_value": wd["proposed_value"],
|
||||
"status": "CHANGE_REQUIRED",
|
||||
"write_eligible": True,
|
||||
"source_dataset": "TIPI_IPI",
|
||||
"source_version": decision.get("dataset_versions", {}).get("tipi_ipi"),
|
||||
}
|
||||
try:
|
||||
record = executor.execute_guarded_update(write_connection, database_path=database_path, proposal=proposal, now=now(), authorized_path=authorized_path)
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "SUCCEEDED", record.before_value, record.after_value, f"rowcount={record.rowcount}", wd["reason_code"], now())
|
||||
summary["writes_succeeded"] += 1
|
||||
except executor.WriteRefused as error:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "FAILED", current_ipi, wd["proposed_value"], str(error), wd["reason_code"], now())
|
||||
summary["writes_failed"] += 1
|
||||
|
||||
return summary
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Single installation preflight for the GTI Fiscal Agent. READ-ONLY: never writes to Firebird, never prints secrets.
|
||||
|
||||
PREFLIGHT=PASS | FAIL (+ blockers, one line each)
|
||||
|
||||
Checks: config, FISCAL_MODE/FIREBIRD_WRITES, Hub reachable/auth/version, Firebird reachable + version + ODS, C000004/C000025,
|
||||
capabilities (minimum: NCM, IPI, CEST), company context (UF/CRT, deterministic resolver), schema/type guards, SQLite state, log dir.
|
||||
"""
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_VERSION_DEFAULT = "1.0.0-rc1"
|
||||
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
|
||||
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
|
||||
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
|
||||
|
||||
|
||||
def _check(name, ok, detail="", warn=False):
|
||||
return {"name": name, "status": "PASS" if ok else ("WARN" if warn else "FAIL"), "detail": detail}
|
||||
|
||||
|
||||
def major_minor(version):
|
||||
parts = str(version or "").replace("-", ".").replace("rc", ".").split(".")
|
||||
return ".".join(parts[:2])
|
||||
|
||||
|
||||
def default_http_get(url, token=None, timeout=10):
|
||||
import httpx
|
||||
headers = {"Authorization": f"Bearer {token}"} if token else {}
|
||||
r = httpx.get(url, headers=headers, timeout=timeout)
|
||||
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
|
||||
|
||||
|
||||
def firebird_probe(env):
|
||||
"""Read-only introspection of the configured database."""
|
||||
import fdb
|
||||
from .company_context import collect_evidence, config_from_env, resolve
|
||||
conn = fdb.connect(database=env["FIREBIRD_DATABASE_PATH"], user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH)
|
||||
FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE WHERE rf.RDB$RELATION_NAME IN ('C000004', 'C000025')""")
|
||||
tables = {}
|
||||
for t, c, ty, ln in cur.fetchall():
|
||||
tables.setdefault(t, {})[c] = (ty, ln)
|
||||
products = None
|
||||
if "C000025" in tables:
|
||||
cur.execute("SELECT COUNT(*) FROM C000025")
|
||||
products = cur.fetchone()[0]
|
||||
company = resolve(collect_evidence(cur), config_from_env(env))
|
||||
return {"server_version": getattr(conn, "server_version", None), "ods": f"{getattr(conn, 'ods_version', '?')}.{getattr(conn, 'ods_minor_version', '?')}",
|
||||
"tables": tables, "products": products, "company": company}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def run_preflight(env=None, *, http_get=default_http_get, probe=firebird_probe, agent_version=None):
|
||||
from .capabilities import detect_capabilities, type_ok
|
||||
from .company_context import preflight_report
|
||||
env = env if env is not None else os.environ
|
||||
checks, info = [], {}
|
||||
agent_version = agent_version or env.get("AGENT_VERSION") or AGENT_VERSION_DEFAULT
|
||||
|
||||
missing = [k for k in REQUIRED_ENV if not (env.get(k) or "").strip()]
|
||||
checks.append(_check("CONFIG", not missing, f"missing: {', '.join(missing)}" if missing else "required variables present"))
|
||||
|
||||
mode = (env.get("FISCAL_MODE") or "").strip().upper()
|
||||
writes = (env.get("FIREBIRD_WRITES") or "0").strip() == "1"
|
||||
checks.append(_check("FISCAL_MODE", mode in FISCAL_MODES, f"FISCAL_MODE={mode or 'unset'} (must be one of {FISCAL_MODES})"))
|
||||
dry_conflict = mode == "DRY_RUN" and writes
|
||||
checks.append(_check("FIREBIRD_WRITES", not dry_conflict, "FIREBIRD_WRITES=1 conflicts with FISCAL_MODE=DRY_RUN (writes stay disabled; fix the config)" if dry_conflict else f"requested={int(writes)}"))
|
||||
effective = writes and mode == "AUTO_APPLY"
|
||||
info["effective_firebird_writes"] = effective
|
||||
if effective:
|
||||
ok = bool((env.get("AUTHORIZED_WRITE_DATABASE_PATH") or "").strip() and (env.get("EXPECTED_SCHEMA_FINGERPRINT") or "").strip())
|
||||
checks.append(_check("WRITE_AUTHORIZATION", ok, "AUTO_APPLY needs AUTHORIZED_WRITE_DATABASE_PATH and EXPECTED_SCHEMA_FINGERPRINT (two independent layers)"))
|
||||
|
||||
hub = (env.get("GTI_HUB_URL") or "").rstrip("/")
|
||||
hub_ok = False
|
||||
if hub:
|
||||
try:
|
||||
code, _ = http_get(f"{hub}/health")
|
||||
hub_ok = code == 200
|
||||
checks.append(_check("HUB_REACHABLE", hub_ok, f"GET /health -> {code}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("HUB_REACHABLE", False, type(e).__name__))
|
||||
if hub_ok:
|
||||
try:
|
||||
code, body = http_get(f"{hub}/version")
|
||||
hub_v = body.get("version")
|
||||
info["hub_version"] = hub_v
|
||||
checks.append(_check("HUB_VERSION_COMPATIBLE", code == 200 and major_minor(hub_v) == major_minor(agent_version), f"hub={hub_v} agent={agent_version} (major.minor must match)"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("HUB_VERSION_COMPATIBLE", False, type(e).__name__))
|
||||
try:
|
||||
code, _ = http_get(f"{hub}/v1/datasets/status", token=env.get("AGENT_TOKEN"))
|
||||
checks.append(_check("AUTH", code == 200, f"agent token -> HTTP {code}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("AUTH", False, type(e).__name__))
|
||||
|
||||
fb = None
|
||||
if all((env.get(k) or "").strip() for k in ("FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")):
|
||||
try:
|
||||
fb = probe(env)
|
||||
checks.append(_check("FIREBIRD_REACHABLE", True, "read-only connection opened"))
|
||||
info.update(firebird_version=fb.get("server_version"), ods=fb.get("ods"), products=fb.get("products"))
|
||||
checks.append(_check("FIREBIRD_VERSION_ODS", bool(fb.get("server_version")), f"{fb.get('server_version')} / ODS {fb.get('ods')}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("FIREBIRD_REACHABLE", False, f"{type(e).__name__}: {str(e)[:80]}"))
|
||||
if fb:
|
||||
t = fb["tables"]
|
||||
checks.append(_check("C000004", "C000004" in t, "company table present" if "C000004" in t else "missing"))
|
||||
checks.append(_check("C000025", "C000025" in t, "product table present" if "C000025" in t else "missing"))
|
||||
cols = {c: ty for c, (ty, _) in t.get("C000025", {}).items()}
|
||||
caps = detect_capabilities(cols)
|
||||
info["capabilities"] = caps
|
||||
lacking = [c for c in REQUIRED_CAPABILITIES if not caps.get(c)]
|
||||
checks.append(_check("CAPABILITIES", not lacking, f"missing required: {', '.join(lacking)}" if lacking else f"required present; optional true: {sum(1 for c, v in caps.items() if v and c not in REQUIRED_CAPABILITIES)}"))
|
||||
rep = preflight_report(fb["company"])
|
||||
info["company_context"] = rep
|
||||
checks.append(_check("COMPANY_CONTEXT", rep["COMPANY_CONTEXT"] == "PASS", "; ".join(rep.get("reasons") or []) or f"{rep['classification']} source={rep['source_of_truth']}"))
|
||||
checks.append(_check("UF", bool(rep.get("company_uf")), str(rep.get("company_uf"))))
|
||||
checks.append(_check("CRT", bool(rep.get("company_crt")), str(rep.get("company_crt"))))
|
||||
writable = [f for f in ("IPI", "CEST", "EX_TIPI", "CLASSIFICACAO_FISCAL") if f in t.get("C000025", {})]
|
||||
bad = [f for f in writable if not type_ok(f, t["C000025"][f][0])]
|
||||
lens = {f: t["C000025"][f][1] for f in ("CEST", "EX_TIPI") if f in t.get("C000025", {})}
|
||||
checks.append(_check("SCHEMA_TYPE_GUARDS", not bad and len(writable) >= 3, f"incompatible: {bad}" if bad else f"types ok for {writable}; text lengths {lens} (the executor re-checks length/type before any write)"))
|
||||
|
||||
state_db = env.get("AGENT_STATE_DB") or str(Path.cwd() / "agent_state.sqlite")
|
||||
try:
|
||||
Path(state_db).parent.mkdir(parents=True, exist_ok=True)
|
||||
c = sqlite3.connect(state_db)
|
||||
c.execute("SELECT 1")
|
||||
c.close()
|
||||
checks.append(_check("SQLITE", True, "state database opens"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("SQLITE", False, type(e).__name__))
|
||||
log_path = Path(env.get("AGENT_LOG_PATH") or Path.cwd() / "logs" / "agent.log")
|
||||
try:
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
checks.append(_check("LOG_DIRECTORY", os.access(log_path.parent, os.W_OK), "log directory writable"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("LOG_DIRECTORY", False, type(e).__name__))
|
||||
|
||||
blockers = [f"{c['name']}: {c['detail']}" for c in checks if c["status"] == "FAIL"]
|
||||
return {"PREFLIGHT": "FAIL" if blockers else "PASS", "blockers": blockers, "checks": checks, "agent_version": agent_version, "fiscal_mode": mode or None, **info}
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Gate Firebird reads on the registered schema fingerprint."""
|
||||
import hashlib, json
|
||||
REFERENCE_SCHEMA_V1 = "1198f89897532b643439f06b729e93b189018f56e7d478e95095f8f56977005b"
|
||||
|
||||
def product_fingerprint(product):
|
||||
fields = ("PRODUTO", "EAN_GTIN", "NCM", "CLASSIFICACAO_FISCAL", "CFOP", "CST", "ALIQUOTA", "CEST", "CSOSN")
|
||||
return hashlib.sha256(json.dumps({key: product.get(key) for key in fields}, sort_keys=True, default=str).encode()).hexdigest()
|
||||
def to_hub_product(product):
|
||||
# C000025 has NCM empty and stores the confirmed eight-digit NCM in CLASSIFICACAO_FISCAL.
|
||||
return {"product_id": product["CODIGO"], "description": product.get("PRODUTO", ""), "gtin": product.get("EAN_GTIN"), "current_ncm": product.get("CLASSIFICACAO_FISCAL")}
|
||||
def run_sample(products, client, state, schema_fingerprint):
|
||||
if schema_fingerprint != REFERENCE_SCHEMA_V1: return {"status": "SCHEMA_UNKNOWN", "results": [], "firebird_writes": 0}
|
||||
outbound = []
|
||||
for product in products:
|
||||
fingerprint = product_fingerprint(product)
|
||||
if state.enqueue(product["CODIGO"], fingerprint, "pending", "0.1.0"): outbound.append((product, fingerprint))
|
||||
if not outbound: return {"status": "OK", "results": [], "firebird_writes": 0, "api_requests": 0}
|
||||
try: results = client.classify_batch([to_hub_product(product) for product, _ in outbound])
|
||||
except Exception:
|
||||
for product, _ in outbound: state.retry(product["CODIGO"])
|
||||
return {"status": "API_UNAVAILABLE", "results": [], "firebird_writes": 0}
|
||||
for (product, fingerprint), result in zip(outbound, results): state.result_received(product["CODIGO"], fingerprint, result["dataset_versions"].get("ncm") or "none", result["engine_version"])
|
||||
return {"status": "OK", "results": results, "firebird_writes": 0, "api_requests": 1}
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Local SQLite outbox; it has no Firebird dependency or write path."""
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
class AgentState:
|
||||
def __init__(self, path):
|
||||
self.connection = sqlite3.connect(Path(path)); self.connection.row_factory = sqlite3.Row
|
||||
self.connection.executescript("CREATE TABLE IF NOT EXISTS product_state(product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, dataset_version TEXT NOT NULL, engine_version TEXT NOT NULL, status TEXT NOT NULL, last_processed_at TEXT, last_result_hash TEXT, reason_code TEXT); CREATE TABLE IF NOT EXISTS outbox(product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, status TEXT NOT NULL, retries INTEGER NOT NULL DEFAULT 0); CREATE TABLE IF NOT EXISTS ipi_write_log(product_id TEXT NOT NULL, source_fingerprint TEXT NOT NULL, decision_status TEXT, ipi_status TEXT, write_status TEXT NOT NULL, before_value TEXT, proposed_value TEXT, result TEXT, reason_code TEXT, timestamp TEXT NOT NULL);")
|
||||
def enqueue(self, product_id, source_fingerprint, dataset_version, engine_version):
|
||||
row = self.connection.execute("SELECT * FROM product_state WHERE product_id=?", (product_id,)).fetchone()
|
||||
if row and tuple(row[key] for key in ("source_fingerprint", "engine_version", "status")) == (source_fingerprint, engine_version, "RESULT_RECEIVED"): return False
|
||||
self.connection.execute("INSERT OR REPLACE INTO product_state(product_id,source_fingerprint,dataset_version,engine_version,status) VALUES(?,?,?,?,?)", (product_id, source_fingerprint, dataset_version, engine_version, "PENDING_API")); self.connection.execute("INSERT OR REPLACE INTO outbox(product_id,source_fingerprint,status) VALUES(?,?,?)", (product_id, source_fingerprint, "PENDING_API")); self.connection.commit(); return True
|
||||
def pending(self): return [dict(row) for row in self.connection.execute("SELECT * FROM outbox WHERE status='PENDING_API'")]
|
||||
def retry(self, product_id): self.connection.execute("UPDATE outbox SET retries=retries+1,status='PENDING_API' WHERE product_id=?", (product_id,)); self.connection.commit()
|
||||
def result_received(self, product_id, source_fingerprint, dataset_version, engine_version):
|
||||
self.connection.execute("UPDATE product_state SET source_fingerprint=?,dataset_version=?,engine_version=?,status='RESULT_RECEIVED' WHERE product_id=?", (source_fingerprint, dataset_version, engine_version, product_id)); self.connection.execute("DELETE FROM outbox WHERE product_id=?", (product_id,)); self.connection.commit()
|
||||
def log_ipi_write(self, product_id, source_fingerprint, decision_status, ipi_status, write_status, before_value, proposed_value, result, reason_code, timestamp):
|
||||
self.connection.execute(
|
||||
"INSERT INTO ipi_write_log(product_id,source_fingerprint,decision_status,ipi_status,write_status,before_value,proposed_value,result,reason_code,timestamp) VALUES(?,?,?,?,?,?,?,?,?,?)",
|
||||
(product_id, source_fingerprint, decision_status, ipi_status, write_status, str(before_value) if before_value is not None else None, str(proposed_value) if proposed_value is not None else None, result, reason_code, timestamp),
|
||||
)
|
||||
self.connection.commit()
|
||||
def ipi_write_log_for(self, product_id):
|
||||
return [dict(row) for row in self.connection.execute("SELECT * FROM ipi_write_log WHERE product_id=? ORDER BY timestamp", (product_id,))]
|
||||
Reference in New Issue
Block a user