152 lines
6.9 KiB
Python
152 lines
6.9 KiB
Python
"""Read-only fiscal coverage matrix over all C000025 products in TESTESOFICIAIS.FDB.
|
|
|
|
No Firebird writes. Classifies every product via the Hub's /v1/classify/batch
|
|
(itself read-only against Firebird — the Hub never touches Firebird at all)
|
|
and separates DATA_AVAILABLE (a dataset exists to consult) from RULE_RESOLVED
|
|
(a single defensible determination was reached) per the mission's own
|
|
instruction not to conflate the two.
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.request
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
|
|
|
from gti_fiscal_agent.firebird_reader import read_products # noqa: E402
|
|
|
|
|
|
def load_dotenv(path: Path) -> None:
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, _, value = line.partition("=")
|
|
os.environ.setdefault(key.strip(), value.strip())
|
|
|
|
|
|
def classify_batch(hub_url, token, products, chunk_size=100):
|
|
results = []
|
|
for start in range(0, len(products), chunk_size):
|
|
chunk = products[start:start + chunk_size]
|
|
payload = {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]}
|
|
request = urllib.request.Request(
|
|
f"{hub_url}/v1/classify/batch",
|
|
data=json.dumps(payload, ensure_ascii=True).encode(),
|
|
headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(request, timeout=60) as response:
|
|
results.extend(json.load(response)["results"])
|
|
return results
|
|
|
|
|
|
def get_dataset_status(hub_url, token):
|
|
request = urllib.request.Request(f"{hub_url}/v1/datasets/status", headers={"Authorization": f"Bearer {token}"})
|
|
with urllib.request.urlopen(request, timeout=15) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def main():
|
|
token = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("AGENT_DEV_TOKEN", "")
|
|
hub_url = os.environ.get("GTI_HUB_URL", "http://localhost:8000")
|
|
load_dotenv(ROOT / ".env")
|
|
|
|
products = read_products(os.environ)
|
|
total = len(products)
|
|
decisions = classify_batch(hub_url, token, products)
|
|
datasets = get_dataset_status(hub_url, token)
|
|
active_kinds = {d["kind"] for d in datasets if d["status"] == "ACTIVE"}
|
|
|
|
ncm_status = Counter(d.get("status") for d in decisions)
|
|
ncm_missing = sum(1 for p in products if not (p["CLASSIFICACAO_FISCAL"] or "").strip())
|
|
ncm_valid = ncm_status.get("EXACT_NCM", 0)
|
|
ncm_legacy = ncm_status.get("LEGACY_NCM", 0)
|
|
ncm_invalid = sum(
|
|
1 for p, d in zip(products, decisions)
|
|
if (p["CLASSIFICACAO_FISCAL"] or "").strip() and d.get("status") not in ("EXACT_NCM", "LEGACY_NCM")
|
|
)
|
|
|
|
ipi_buckets = Counter()
|
|
cest_match = cest_confirmed = cest_candidates = 0
|
|
mva_match = mva_resolved = mva_candidates = 0
|
|
|
|
for decision in decisions:
|
|
ipi_status = decision.get("ipi_status")
|
|
if ipi_status == "LEGACY_NCM":
|
|
pass
|
|
elif ipi_status == "NO_TIPI_MAPPING_FOUND":
|
|
ipi_buckets["NO_MATCH"] += 1
|
|
elif ipi_status == "IPI_CANDIDATES":
|
|
ipi_buckets["AMBIGUOUS"] += 1
|
|
elif ipi_status == "IPI_RESOLVED":
|
|
rate_kind = (decision.get("classification", {}).get("ipi") or {}).get("rate_kind")
|
|
ipi_buckets[{"NUMERIC": "NUMERIC", "NON_NUMERIC": "NON_NUMERIC", "BLANK": "SOURCE_BLANK"}.get(rate_kind, "NO_MATCH")] += 1
|
|
else:
|
|
ipi_buckets["NO_MATCH"] += 1
|
|
|
|
cest_status = decision.get("cest_status")
|
|
if cest_status not in (None, "NO_CEST_MAPPING_FOUND", "LEGACY_NCM"):
|
|
cest_match += 1
|
|
if cest_status == "CEST_CONFIRMED":
|
|
cest_confirmed += 1
|
|
elif cest_status == "CEST_CANDIDATES":
|
|
cest_candidates += 1
|
|
|
|
mva_status = decision.get("ba_st_mva_status")
|
|
if mva_status not in (None, "NO_MVA_MAPPING_FOUND", "LEGACY_NCM"):
|
|
mva_match += 1
|
|
if mva_status == "MVA_RESOLVED":
|
|
mva_resolved += 1
|
|
elif mva_status == "MVA_CANDIDATES":
|
|
mva_candidates += 1
|
|
|
|
coverage = {
|
|
"artifact": "FISCAL_COVERAGE_FULL_V1",
|
|
"firebird_writes": 0,
|
|
"total_products": total,
|
|
"ncm_valid": ncm_valid,
|
|
"ncm_legacy": ncm_legacy,
|
|
"ncm_invalid": ncm_invalid,
|
|
"ncm_missing": ncm_missing,
|
|
"ipi": {
|
|
"numeric": ipi_buckets["NUMERIC"],
|
|
"non_numeric": ipi_buckets["NON_NUMERIC"],
|
|
"source_blank": ipi_buckets["SOURCE_BLANK"],
|
|
"ambiguous": ipi_buckets["AMBIGUOUS"],
|
|
"no_match": ipi_buckets["NO_MATCH"],
|
|
},
|
|
"cest": {"match": cest_match, "confirmed": cest_confirmed, "candidates": cest_candidates},
|
|
"ba_st_mva": {"match": mva_match, "resolved": mva_resolved, "candidates": mva_candidates},
|
|
"pis_cofins": {
|
|
"data_available": True, # CST_PIS/CST_COFINS domain tables exist (fiscal_code_entries), no product-level linkage yet
|
|
"match": 0,
|
|
"unresolved": total,
|
|
"note": "CST_PIS/CST_COFINS code domains are ACTIVE and consultable, but no NCM-linked determination table (SPED Tabela 4.3.10-4.3.17) is available to resolve a specific product's PIS/COFINS situation yet. DATA_AVAILABLE=true, RULE_RESOLVED=0 for all products.",
|
|
},
|
|
"cfop": {"dataset_available": False, "note": "CFOP full table FAILED to acquire this session (portal-gated); MOC7 only references CFOP, does not enumerate it. See docs/SOURCE_REGISTRY_V2.md."},
|
|
"cst_icms": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 11},
|
|
"csosn": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 10},
|
|
"origin": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 9},
|
|
"fcp": {"dataset_available": False, "note": "No single canonical national table exists; FAILED (see docs/SOURCE_REGISTRY_V2.md)."},
|
|
"icms_monophasic": {"match": 0, "dataset_available": False},
|
|
"anp": {"match": 0, "dataset_available": False},
|
|
"ncm_utrib": {"dataset_available": False},
|
|
"rtc": {
|
|
"cclasstrib_match": 0, "ccredpres_match": 0, "cbs_rate_match": 0, "biofuel_index_available": False,
|
|
"note": "All RTC (Reforma Tributária) tables FAILED to acquire — 2025/26-era portal documents, not present in MOC7 or locally.",
|
|
},
|
|
"active_dataset_kinds": sorted(active_kinds),
|
|
}
|
|
|
|
output_path = ROOT / "docs" / "FISCAL_COVERAGE_FULL_V1.json"
|
|
output_path.write_text(json.dumps(coverage, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
print(json.dumps({k: v for k, v in coverage.items() if k not in ("pis_cofins", "rtc")}, indent=2, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|