263 lines
10 KiB
Python
263 lines
10 KiB
Python
"""MVP first real batch: classify all of C000025, dry-run the IPI plan, then
|
|
execute every write_eligible IPI proposal against TESTESOFICIAIS.FDB only.
|
|
|
|
Phases:
|
|
1. Read all products from C000025 (read-only).
|
|
2. Classify each via the running fiscal-hub /v1/classify/batch.
|
|
3. Build field proposals (dry-run) with central.app.dry_run_planner.
|
|
4. Execute every write_eligible IPI proposal via firebird_write_executor,
|
|
sequentially, with compare-before-write and rowcount==1 per product.
|
|
5. Re-read every written CODIGO independently and verify persistence.
|
|
|
|
Never touches CEST/EX_TIPI writes. Never touches any database other than
|
|
TESTESOFICIAIS.FDB (enforced by firebird_write_executor's path guard).
|
|
"""
|
|
import argparse
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.request
|
|
from collections import Counter
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "central"))
|
|
|
|
from app.dry_run_planner import build_field_proposals, split_by_eligibility # noqa: E402
|
|
|
|
|
|
def load_dotenv(path: Path) -> None:
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, _, value = line.partition("=")
|
|
os.environ[key.strip()] = value.strip()
|
|
|
|
|
|
def load_executor():
|
|
spec = importlib.util.spec_from_file_location("firebird_write_executor", ROOT / "scripts" / "firebird_write_executor.py")
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def read_all_products(env):
|
|
import fdb
|
|
conn = fdb.connect(
|
|
database=env["FIREBIRD_DATABASE_PATH"],
|
|
user=env["FB_USER"],
|
|
password=env["FB_PASSWORD"],
|
|
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
|
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO,
|
|
)
|
|
try:
|
|
cur = conn.cursor()
|
|
cur.execute("SELECT CODIGO, PRODUTO, CLASSIFICACAO_FISCAL, IPI, EX_TIPI, CEST FROM C000025 ORDER BY CODIGO")
|
|
names = [c[0].strip() for c in cur.description]
|
|
return [dict(zip(names, row)) for row in cur.fetchall()]
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def classify_batch(hub_url, token, products, chunk_size=100):
|
|
results = []
|
|
for start in range(0, len(products), chunk_size):
|
|
chunk = products[start:start + chunk_size]
|
|
payload = {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]}
|
|
request = urllib.request.Request(
|
|
f"{hub_url}/v1/classify/batch",
|
|
data=json.dumps(payload, ensure_ascii=True).encode(),
|
|
headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(request, timeout=60) as response:
|
|
results.extend(json.load(response)["results"])
|
|
return results
|
|
|
|
|
|
def classify_ipi_bucket(decision):
|
|
ipi_status = decision.get("ipi_status")
|
|
if ipi_status == "LEGACY_NCM":
|
|
return None # tracked under NCM_LEGACY, not an IPI bucket
|
|
if ipi_status == "NO_TIPI_MAPPING_FOUND":
|
|
return "NO_MATCH"
|
|
if ipi_status == "IPI_CANDIDATES":
|
|
return "AMBIGUOUS"
|
|
if ipi_status == "IPI_RESOLVED":
|
|
rate_kind = (decision.get("classification", {}).get("ipi") or {}).get("rate_kind")
|
|
return {"NUMERIC": "NUMERIC", "NON_NUMERIC": "NON_NUMERIC", "BLANK": "SOURCE_BLANK"}.get(rate_kind, "NO_MATCH")
|
|
return "NO_MATCH"
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--hub-url", default="http://localhost:8000")
|
|
parser.add_argument("--token", required=True)
|
|
parser.add_argument("--dry-run-output", default=str(ROOT / "docs" / "FULL_DATABASE_DRY_RUN_V1.json"))
|
|
parser.add_argument("--audit-output", default=str(ROOT / "docs" / "runs" / "full_database_ipi_write_audit_v1.json"))
|
|
parser.add_argument("--no-write", action="store_true", help="Only produce the dry-run artifact; skip Phase 4/5.")
|
|
args = parser.parse_args()
|
|
|
|
load_dotenv(ROOT / ".env")
|
|
executor = load_executor()
|
|
database_path = os.environ["FIREBIRD_DATABASE_PATH"]
|
|
executor.assert_authorized_write_path(database_path)
|
|
|
|
products = read_all_products(os.environ)
|
|
total = len(products)
|
|
|
|
ncm_missing = sum(1 for p in products if not (p["CLASSIFICACAO_FISCAL"] or "").strip())
|
|
|
|
decisions = classify_batch(args.hub_url, args.token, products)
|
|
|
|
ncm_status_counts = Counter(d.get("status") for d in decisions)
|
|
ncm_valid = ncm_status_counts.get("EXACT_NCM", 0)
|
|
ncm_legacy = ncm_status_counts.get("LEGACY_NCM", 0)
|
|
# invalid = has a CLASSIFICACAO_FISCAL but it isn't a recognized/legacy NCM
|
|
ncm_invalid = sum(
|
|
1 for p, d in zip(products, decisions)
|
|
if (p["CLASSIFICACAO_FISCAL"] or "").strip() and d.get("status") not in ("EXACT_NCM", "LEGACY_NCM")
|
|
)
|
|
|
|
ipi_buckets = Counter()
|
|
field_proposals = []
|
|
for product, decision in zip(products, decisions):
|
|
code = product["CODIGO"]
|
|
current = {"ipi": product.get("IPI"), "cest": product.get("CEST"), "ex_tipi": product.get("EX_TIPI")}
|
|
field_proposals.extend(build_field_proposals(code, decision, current))
|
|
bucket = classify_ipi_bucket(decision)
|
|
if bucket:
|
|
ipi_buckets[bucket] += 1
|
|
|
|
ipi_rows = [p for p in field_proposals if p["field"] == "IPI"]
|
|
cest_rows = [p for p in field_proposals if p["field"] == "CEST"]
|
|
split = split_by_eligibility(ipi_rows)
|
|
|
|
ipi_already_correct = sum(1 for p in ipi_rows if p["status"] == "SAME_VALUE")
|
|
ipi_change_required = sum(1 for p in ipi_rows if p["status"] == "CHANGE_REQUIRED")
|
|
ipi_write_eligible_rows = split["eligible"]
|
|
|
|
cest_db_match = sum(1 for p in cest_rows if p["status"] not in ("UNRESOLVED",) or p["reason_code"] != "NO_CEST_MAPPING_FOUND_FOR_NCM")
|
|
cest_confirmed = sum(1 for p in cest_rows if p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED", "CURRENT_UNKNOWN"))
|
|
|
|
dry_run_artifact = {
|
|
"artifact": "FULL_DATABASE_DRY_RUN_V1",
|
|
"total_products": total,
|
|
"ncm_valid": ncm_valid,
|
|
"ncm_legacy": ncm_legacy,
|
|
"ncm_invalid": ncm_invalid,
|
|
"ncm_missing": ncm_missing,
|
|
"ipi_buckets": dict(ipi_buckets),
|
|
"ipi_already_correct": ipi_already_correct,
|
|
"ipi_change_required": ipi_change_required,
|
|
"ipi_write_eligible": len(ipi_write_eligible_rows),
|
|
"cest_db_match_products": cest_db_match,
|
|
"cest_confirmed": cest_confirmed,
|
|
"field_proposals": field_proposals,
|
|
}
|
|
Path(args.dry_run_output).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(args.dry_run_output).write_text(json.dumps(dry_run_artifact, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
|
|
summary = {
|
|
"total_products": total,
|
|
"ncm_valid": ncm_valid,
|
|
"ncm_legacy": ncm_legacy,
|
|
"ncm_invalid": ncm_invalid,
|
|
"ncm_missing": ncm_missing,
|
|
"ipi_buckets": dict(ipi_buckets),
|
|
"ipi_already_correct": ipi_already_correct,
|
|
"ipi_change_required": ipi_change_required,
|
|
"ipi_write_eligible": len(ipi_write_eligible_rows),
|
|
}
|
|
|
|
if args.no_write or not ipi_write_eligible_rows:
|
|
summary["writes_attempted"] = 0
|
|
print(json.dumps(summary, indent=2))
|
|
return
|
|
|
|
import fdb
|
|
connection = fdb.connect(
|
|
database=database_path,
|
|
user=os.environ["FB_USER"],
|
|
password=os.environ["FB_PASSWORD"],
|
|
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
|
)
|
|
audit_records = []
|
|
succeeded = failed = 0
|
|
try:
|
|
for proposal in ipi_write_eligible_rows:
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
try:
|
|
record = executor.execute_guarded_update(
|
|
connection,
|
|
database_path=database_path,
|
|
proposal=proposal,
|
|
now=now,
|
|
)
|
|
audit_records.append({**record.as_dict(), "outcome": "SUCCEEDED"})
|
|
succeeded += 1
|
|
except executor.WriteRefused as error:
|
|
audit_records.append({
|
|
"product_code": proposal["product_code"],
|
|
"field": proposal["field"],
|
|
"before_value": proposal["current_value"],
|
|
"proposed_value": proposal["proposed_value"],
|
|
"after_value": None,
|
|
"rowcount": 0,
|
|
"source_dataset": proposal.get("source_dataset"),
|
|
"source_version": proposal.get("source_version"),
|
|
"timestamp": now,
|
|
"outcome": "FAILED",
|
|
"error": str(error),
|
|
})
|
|
failed += 1
|
|
finally:
|
|
connection.close()
|
|
|
|
Path(args.audit_output).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(args.audit_output).write_text(json.dumps(audit_records, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
|
|
# Phase 5: independent re-read + verification of every written CODIGO.
|
|
written_codes = [r["product_code"] for r in audit_records if r["outcome"] == "SUCCEEDED"]
|
|
verify_conn = fdb.connect(
|
|
database=database_path,
|
|
user=os.environ["FB_USER"],
|
|
password=os.environ["FB_PASSWORD"],
|
|
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
|
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO,
|
|
)
|
|
verified = 0
|
|
mismatches = []
|
|
try:
|
|
cur = verify_conn.cursor()
|
|
for record in audit_records:
|
|
if record["outcome"] != "SUCCEEDED":
|
|
continue
|
|
cur.execute("SELECT IPI FROM C000025 WHERE CODIGO = ?", (record["product_code"],))
|
|
row = cur.fetchone()
|
|
live_value = row[0] if row else None
|
|
if live_value == record["proposed_value"]:
|
|
verified += 1
|
|
else:
|
|
mismatches.append({"product_code": record["product_code"], "expected": record["proposed_value"], "found": live_value})
|
|
finally:
|
|
verify_conn.close()
|
|
|
|
summary.update({
|
|
"writes_attempted": len(ipi_write_eligible_rows),
|
|
"writes_succeeded": succeeded,
|
|
"writes_failed": failed,
|
|
"post_write_verified": verified,
|
|
"post_write_mismatches": mismatches,
|
|
"written_codes": written_codes,
|
|
})
|
|
print(json.dumps(summary, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|