Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
This commit is contained in:
gti
@@ -0,0 +1,151 @@
|
||||
"""Read-only fiscal coverage matrix over all C000025 products in TESTESOFICIAIS.FDB.
|
||||
|
||||
No Firebird writes. Classifies every product via the Hub's /v1/classify/batch
|
||||
(itself read-only against Firebird — the Hub never touches Firebird at all)
|
||||
and separates DATA_AVAILABLE (a dataset exists to consult) from RULE_RESOLVED
|
||||
(a single defensible determination was reached) per the mission's own
|
||||
instruction not to conflate the two.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
|
||||
from gti_fiscal_agent.firebird_reader import read_products # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def classify_batch(hub_url, token, products, chunk_size=100):
|
||||
results = []
|
||||
for start in range(0, len(products), chunk_size):
|
||||
chunk = products[start:start + chunk_size]
|
||||
payload = {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]}
|
||||
request = urllib.request.Request(
|
||||
f"{hub_url}/v1/classify/batch",
|
||||
data=json.dumps(payload, ensure_ascii=True).encode(),
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
results.extend(json.load(response)["results"])
|
||||
return results
|
||||
|
||||
|
||||
def get_dataset_status(hub_url, token):
|
||||
request = urllib.request.Request(f"{hub_url}/v1/datasets/status", headers={"Authorization": f"Bearer {token}"})
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def main():
|
||||
token = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("AGENT_DEV_TOKEN", "")
|
||||
hub_url = os.environ.get("GTI_HUB_URL", "http://localhost:8000")
|
||||
load_dotenv(ROOT / ".env")
|
||||
|
||||
products = read_products(os.environ)
|
||||
total = len(products)
|
||||
decisions = classify_batch(hub_url, token, products)
|
||||
datasets = get_dataset_status(hub_url, token)
|
||||
active_kinds = {d["kind"] for d in datasets if d["status"] == "ACTIVE"}
|
||||
|
||||
ncm_status = Counter(d.get("status") for d in decisions)
|
||||
ncm_missing = sum(1 for p in products if not (p["CLASSIFICACAO_FISCAL"] or "").strip())
|
||||
ncm_valid = ncm_status.get("EXACT_NCM", 0)
|
||||
ncm_legacy = ncm_status.get("LEGACY_NCM", 0)
|
||||
ncm_invalid = sum(
|
||||
1 for p, d in zip(products, decisions)
|
||||
if (p["CLASSIFICACAO_FISCAL"] or "").strip() and d.get("status") not in ("EXACT_NCM", "LEGACY_NCM")
|
||||
)
|
||||
|
||||
ipi_buckets = Counter()
|
||||
cest_match = cest_confirmed = cest_candidates = 0
|
||||
mva_match = mva_resolved = mva_candidates = 0
|
||||
|
||||
for decision in decisions:
|
||||
ipi_status = decision.get("ipi_status")
|
||||
if ipi_status == "LEGACY_NCM":
|
||||
pass
|
||||
elif ipi_status == "NO_TIPI_MAPPING_FOUND":
|
||||
ipi_buckets["NO_MATCH"] += 1
|
||||
elif ipi_status == "IPI_CANDIDATES":
|
||||
ipi_buckets["AMBIGUOUS"] += 1
|
||||
elif ipi_status == "IPI_RESOLVED":
|
||||
rate_kind = (decision.get("classification", {}).get("ipi") or {}).get("rate_kind")
|
||||
ipi_buckets[{"NUMERIC": "NUMERIC", "NON_NUMERIC": "NON_NUMERIC", "BLANK": "SOURCE_BLANK"}.get(rate_kind, "NO_MATCH")] += 1
|
||||
else:
|
||||
ipi_buckets["NO_MATCH"] += 1
|
||||
|
||||
cest_status = decision.get("cest_status")
|
||||
if cest_status not in (None, "NO_CEST_MAPPING_FOUND", "LEGACY_NCM"):
|
||||
cest_match += 1
|
||||
if cest_status == "CEST_CONFIRMED":
|
||||
cest_confirmed += 1
|
||||
elif cest_status == "CEST_CANDIDATES":
|
||||
cest_candidates += 1
|
||||
|
||||
mva_status = decision.get("ba_st_mva_status")
|
||||
if mva_status not in (None, "NO_MVA_MAPPING_FOUND", "LEGACY_NCM"):
|
||||
mva_match += 1
|
||||
if mva_status == "MVA_RESOLVED":
|
||||
mva_resolved += 1
|
||||
elif mva_status == "MVA_CANDIDATES":
|
||||
mva_candidates += 1
|
||||
|
||||
coverage = {
|
||||
"artifact": "FISCAL_COVERAGE_FULL_V1",
|
||||
"firebird_writes": 0,
|
||||
"total_products": total,
|
||||
"ncm_valid": ncm_valid,
|
||||
"ncm_legacy": ncm_legacy,
|
||||
"ncm_invalid": ncm_invalid,
|
||||
"ncm_missing": ncm_missing,
|
||||
"ipi": {
|
||||
"numeric": ipi_buckets["NUMERIC"],
|
||||
"non_numeric": ipi_buckets["NON_NUMERIC"],
|
||||
"source_blank": ipi_buckets["SOURCE_BLANK"],
|
||||
"ambiguous": ipi_buckets["AMBIGUOUS"],
|
||||
"no_match": ipi_buckets["NO_MATCH"],
|
||||
},
|
||||
"cest": {"match": cest_match, "confirmed": cest_confirmed, "candidates": cest_candidates},
|
||||
"ba_st_mva": {"match": mva_match, "resolved": mva_resolved, "candidates": mva_candidates},
|
||||
"pis_cofins": {
|
||||
"data_available": True, # CST_PIS/CST_COFINS domain tables exist (fiscal_code_entries), no product-level linkage yet
|
||||
"match": 0,
|
||||
"unresolved": total,
|
||||
"note": "CST_PIS/CST_COFINS code domains are ACTIVE and consultable, but no NCM-linked determination table (SPED Tabela 4.3.10-4.3.17) is available to resolve a specific product's PIS/COFINS situation yet. DATA_AVAILABLE=true, RULE_RESOLVED=0 for all products.",
|
||||
},
|
||||
"cfop": {"dataset_available": False, "note": "CFOP full table FAILED to acquire this session (portal-gated); MOC7 only references CFOP, does not enumerate it. See docs/SOURCE_REGISTRY_V2.md."},
|
||||
"cst_icms": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 11},
|
||||
"csosn": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 10},
|
||||
"origin": {"dataset_available": "NFE_MOC_TABLES" in active_kinds, "record_count": 9},
|
||||
"fcp": {"dataset_available": False, "note": "No single canonical national table exists; FAILED (see docs/SOURCE_REGISTRY_V2.md)."},
|
||||
"icms_monophasic": {"match": 0, "dataset_available": False},
|
||||
"anp": {"match": 0, "dataset_available": False},
|
||||
"ncm_utrib": {"dataset_available": False},
|
||||
"rtc": {
|
||||
"cclasstrib_match": 0, "ccredpres_match": 0, "cbs_rate_match": 0, "biofuel_index_available": False,
|
||||
"note": "All RTC (Reforma Tributária) tables FAILED to acquire — 2025/26-era portal documents, not present in MOC7 or locally.",
|
||||
},
|
||||
"active_dataset_kinds": sorted(active_kinds),
|
||||
}
|
||||
|
||||
output_path = ROOT / "docs" / "FISCAL_COVERAGE_FULL_V1.json"
|
||||
output_path.write_text(json.dumps(coverage, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(json.dumps({k: v for k, v in coverage.items() if k not in ("pis_cofins", "rtc")}, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user