"""MVP first real batch: classify all of C000025, dry-run the IPI plan, then execute every write_eligible IPI proposal against TESTESOFICIAIS.FDB only. Phases: 1. Read all products from C000025 (read-only). 2. Classify each via the running fiscal-hub /v1/classify/batch. 3. Build field proposals (dry-run) with central.app.dry_run_planner. 4. Execute every write_eligible IPI proposal via firebird_write_executor, sequentially, with compare-before-write and rowcount==1 per product. 5. Re-read every written CODIGO independently and verify persistence. Never touches CEST/EX_TIPI writes. Never touches any database other than TESTESOFICIAIS.FDB (enforced by firebird_write_executor's path guard). """ import argparse import importlib.util import json import os import sys import urllib.request from collections import Counter from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "central")) from app.dry_run_planner import build_field_proposals, split_by_eligibility # noqa: E402 def load_dotenv(path: Path) -> None: for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line or line.startswith("#") or "=" not in line: continue key, _, value = line.partition("=") os.environ[key.strip()] = value.strip() def load_executor(): spec = importlib.util.spec_from_file_location("firebird_write_executor", ROOT / "scripts" / "firebird_write_executor.py") module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def read_all_products(env): import fdb conn = fdb.connect( database=env["FIREBIRD_DATABASE_PATH"], user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO, ) try: cur = conn.cursor() cur.execute("SELECT CODIGO, PRODUTO, CLASSIFICACAO_FISCAL, IPI, EX_TIPI, CEST FROM C000025 ORDER BY CODIGO") names = [c[0].strip() for c in cur.description] return [dict(zip(names, row)) for row in cur.fetchall()] finally: conn.close() def classify_batch(hub_url, token, products, chunk_size=100): results = [] for start in range(0, len(products), chunk_size): chunk = products[start:start + chunk_size] payload = {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]} request = urllib.request.Request( f"{hub_url}/v1/classify/batch", data=json.dumps(payload, ensure_ascii=True).encode(), headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"}, method="POST", ) with urllib.request.urlopen(request, timeout=60) as response: results.extend(json.load(response)["results"]) return results def classify_ipi_bucket(decision): ipi_status = decision.get("ipi_status") if ipi_status == "LEGACY_NCM": return None # tracked under NCM_LEGACY, not an IPI bucket if ipi_status == "NO_TIPI_MAPPING_FOUND": return "NO_MATCH" if ipi_status == "IPI_CANDIDATES": return "AMBIGUOUS" if ipi_status == "IPI_RESOLVED": rate_kind = (decision.get("classification", {}).get("ipi") or {}).get("rate_kind") return {"NUMERIC": "NUMERIC", "NON_NUMERIC": "NON_NUMERIC", "BLANK": "SOURCE_BLANK"}.get(rate_kind, "NO_MATCH") return "NO_MATCH" def main(): parser = argparse.ArgumentParser() parser.add_argument("--hub-url", default="http://localhost:8000") parser.add_argument("--token", required=True) parser.add_argument("--dry-run-output", default=str(ROOT / "docs" / "FULL_DATABASE_DRY_RUN_V1.json")) parser.add_argument("--audit-output", default=str(ROOT / "docs" / "runs" / "full_database_ipi_write_audit_v1.json")) parser.add_argument("--no-write", action="store_true", help="Only produce the dry-run artifact; skip Phase 4/5.") args = parser.parse_args() load_dotenv(ROOT / ".env") executor = load_executor() database_path = os.environ["FIREBIRD_DATABASE_PATH"] executor.assert_authorized_write_path(database_path) products = read_all_products(os.environ) total = len(products) ncm_missing = sum(1 for p in products if not (p["CLASSIFICACAO_FISCAL"] or "").strip()) decisions = classify_batch(args.hub_url, args.token, products) ncm_status_counts = Counter(d.get("status") for d in decisions) ncm_valid = ncm_status_counts.get("EXACT_NCM", 0) ncm_legacy = ncm_status_counts.get("LEGACY_NCM", 0) # invalid = has a CLASSIFICACAO_FISCAL but it isn't a recognized/legacy NCM ncm_invalid = sum( 1 for p, d in zip(products, decisions) if (p["CLASSIFICACAO_FISCAL"] or "").strip() and d.get("status") not in ("EXACT_NCM", "LEGACY_NCM") ) ipi_buckets = Counter() field_proposals = [] for product, decision in zip(products, decisions): code = product["CODIGO"] current = {"ipi": product.get("IPI"), "cest": product.get("CEST"), "ex_tipi": product.get("EX_TIPI")} field_proposals.extend(build_field_proposals(code, decision, current)) bucket = classify_ipi_bucket(decision) if bucket: ipi_buckets[bucket] += 1 ipi_rows = [p for p in field_proposals if p["field"] == "IPI"] cest_rows = [p for p in field_proposals if p["field"] == "CEST"] split = split_by_eligibility(ipi_rows) ipi_already_correct = sum(1 for p in ipi_rows if p["status"] == "SAME_VALUE") ipi_change_required = sum(1 for p in ipi_rows if p["status"] == "CHANGE_REQUIRED") ipi_write_eligible_rows = split["eligible"] cest_db_match = sum(1 for p in cest_rows if p["status"] not in ("UNRESOLVED",) or p["reason_code"] != "NO_CEST_MAPPING_FOUND_FOR_NCM") cest_confirmed = sum(1 for p in cest_rows if p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED", "CURRENT_UNKNOWN")) dry_run_artifact = { "artifact": "FULL_DATABASE_DRY_RUN_V1", "total_products": total, "ncm_valid": ncm_valid, "ncm_legacy": ncm_legacy, "ncm_invalid": ncm_invalid, "ncm_missing": ncm_missing, "ipi_buckets": dict(ipi_buckets), "ipi_already_correct": ipi_already_correct, "ipi_change_required": ipi_change_required, "ipi_write_eligible": len(ipi_write_eligible_rows), "cest_db_match_products": cest_db_match, "cest_confirmed": cest_confirmed, "field_proposals": field_proposals, } Path(args.dry_run_output).parent.mkdir(parents=True, exist_ok=True) Path(args.dry_run_output).write_text(json.dumps(dry_run_artifact, indent=2, ensure_ascii=False), encoding="utf-8") summary = { "total_products": total, "ncm_valid": ncm_valid, "ncm_legacy": ncm_legacy, "ncm_invalid": ncm_invalid, "ncm_missing": ncm_missing, "ipi_buckets": dict(ipi_buckets), "ipi_already_correct": ipi_already_correct, "ipi_change_required": ipi_change_required, "ipi_write_eligible": len(ipi_write_eligible_rows), } if args.no_write or not ipi_write_eligible_rows: summary["writes_attempted"] = 0 print(json.dumps(summary, indent=2)) return import fdb connection = fdb.connect( database=database_path, user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), ) audit_records = [] succeeded = failed = 0 try: for proposal in ipi_write_eligible_rows: now = datetime.now(timezone.utc).isoformat() try: record = executor.execute_guarded_update( connection, database_path=database_path, proposal=proposal, now=now, ) audit_records.append({**record.as_dict(), "outcome": "SUCCEEDED"}) succeeded += 1 except executor.WriteRefused as error: audit_records.append({ "product_code": proposal["product_code"], "field": proposal["field"], "before_value": proposal["current_value"], "proposed_value": proposal["proposed_value"], "after_value": None, "rowcount": 0, "source_dataset": proposal.get("source_dataset"), "source_version": proposal.get("source_version"), "timestamp": now, "outcome": "FAILED", "error": str(error), }) failed += 1 finally: connection.close() Path(args.audit_output).parent.mkdir(parents=True, exist_ok=True) Path(args.audit_output).write_text(json.dumps(audit_records, indent=2, ensure_ascii=False), encoding="utf-8") # Phase 5: independent re-read + verification of every written CODIGO. written_codes = [r["product_code"] for r in audit_records if r["outcome"] == "SUCCEEDED"] verify_conn = fdb.connect( database=database_path, user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO, ) verified = 0 mismatches = [] try: cur = verify_conn.cursor() for record in audit_records: if record["outcome"] != "SUCCEEDED": continue cur.execute("SELECT IPI FROM C000025 WHERE CODIGO = ?", (record["product_code"],)) row = cur.fetchone() live_value = row[0] if row else None if live_value == record["proposed_value"]: verified += 1 else: mismatches.append({"product_code": record["product_code"], "expected": record["proposed_value"], "found": live_value}) finally: verify_conn.close() summary.update({ "writes_attempted": len(ipi_write_eligible_rows), "writes_succeeded": succeeded, "writes_failed": failed, "post_write_verified": verified, "post_write_mismatches": mismatches, "written_codes": written_codes, }) print(json.dumps(summary, indent=2)) if __name__ == "__main__": main()