Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
This commit is contained in:
gti
@@ -0,0 +1,262 @@
|
||||
"""MVP first real batch: classify all of C000025, dry-run the IPI plan, then
|
||||
execute every write_eligible IPI proposal against TESTESOFICIAIS.FDB only.
|
||||
|
||||
Phases:
|
||||
1. Read all products from C000025 (read-only).
|
||||
2. Classify each via the running fiscal-hub /v1/classify/batch.
|
||||
3. Build field proposals (dry-run) with central.app.dry_run_planner.
|
||||
4. Execute every write_eligible IPI proposal via firebird_write_executor,
|
||||
sequentially, with compare-before-write and rowcount==1 per product.
|
||||
5. Re-read every written CODIGO independently and verify persistence.
|
||||
|
||||
Never touches CEST/EX_TIPI writes. Never touches any database other than
|
||||
TESTESOFICIAIS.FDB (enforced by firebird_write_executor's path guard).
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "central"))
|
||||
|
||||
from app.dry_run_planner import build_field_proposals, split_by_eligibility # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ[key.strip()] = value.strip()
|
||||
|
||||
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", ROOT / "scripts" / "firebird_write_executor.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def read_all_products(env):
|
||||
import fdb
|
||||
conn = fdb.connect(
|
||||
database=env["FIREBIRD_DATABASE_PATH"],
|
||||
user=env["FB_USER"],
|
||||
password=env["FB_PASSWORD"],
|
||||
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO,
|
||||
)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT CODIGO, PRODUTO, CLASSIFICACAO_FISCAL, IPI, EX_TIPI, CEST FROM C000025 ORDER BY CODIGO")
|
||||
names = [c[0].strip() for c in cur.description]
|
||||
return [dict(zip(names, row)) for row in cur.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def classify_batch(hub_url, token, products, chunk_size=100):
|
||||
results = []
|
||||
for start in range(0, len(products), chunk_size):
|
||||
chunk = products[start:start + chunk_size]
|
||||
payload = {"products": [{"current_ncm": p["CLASSIFICACAO_FISCAL"], "description": p["PRODUTO"] or ""} for p in chunk]}
|
||||
request = urllib.request.Request(
|
||||
f"{hub_url}/v1/classify/batch",
|
||||
data=json.dumps(payload, ensure_ascii=True).encode(),
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
results.extend(json.load(response)["results"])
|
||||
return results
|
||||
|
||||
|
||||
def classify_ipi_bucket(decision):
|
||||
ipi_status = decision.get("ipi_status")
|
||||
if ipi_status == "LEGACY_NCM":
|
||||
return None # tracked under NCM_LEGACY, not an IPI bucket
|
||||
if ipi_status == "NO_TIPI_MAPPING_FOUND":
|
||||
return "NO_MATCH"
|
||||
if ipi_status == "IPI_CANDIDATES":
|
||||
return "AMBIGUOUS"
|
||||
if ipi_status == "IPI_RESOLVED":
|
||||
rate_kind = (decision.get("classification", {}).get("ipi") or {}).get("rate_kind")
|
||||
return {"NUMERIC": "NUMERIC", "NON_NUMERIC": "NON_NUMERIC", "BLANK": "SOURCE_BLANK"}.get(rate_kind, "NO_MATCH")
|
||||
return "NO_MATCH"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--hub-url", default="http://localhost:8000")
|
||||
parser.add_argument("--token", required=True)
|
||||
parser.add_argument("--dry-run-output", default=str(ROOT / "docs" / "FULL_DATABASE_DRY_RUN_V1.json"))
|
||||
parser.add_argument("--audit-output", default=str(ROOT / "docs" / "runs" / "full_database_ipi_write_audit_v1.json"))
|
||||
parser.add_argument("--no-write", action="store_true", help="Only produce the dry-run artifact; skip Phase 4/5.")
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv(ROOT / ".env")
|
||||
executor = load_executor()
|
||||
database_path = os.environ["FIREBIRD_DATABASE_PATH"]
|
||||
executor.assert_authorized_write_path(database_path)
|
||||
|
||||
products = read_all_products(os.environ)
|
||||
total = len(products)
|
||||
|
||||
ncm_missing = sum(1 for p in products if not (p["CLASSIFICACAO_FISCAL"] or "").strip())
|
||||
|
||||
decisions = classify_batch(args.hub_url, args.token, products)
|
||||
|
||||
ncm_status_counts = Counter(d.get("status") for d in decisions)
|
||||
ncm_valid = ncm_status_counts.get("EXACT_NCM", 0)
|
||||
ncm_legacy = ncm_status_counts.get("LEGACY_NCM", 0)
|
||||
# invalid = has a CLASSIFICACAO_FISCAL but it isn't a recognized/legacy NCM
|
||||
ncm_invalid = sum(
|
||||
1 for p, d in zip(products, decisions)
|
||||
if (p["CLASSIFICACAO_FISCAL"] or "").strip() and d.get("status") not in ("EXACT_NCM", "LEGACY_NCM")
|
||||
)
|
||||
|
||||
ipi_buckets = Counter()
|
||||
field_proposals = []
|
||||
for product, decision in zip(products, decisions):
|
||||
code = product["CODIGO"]
|
||||
current = {"ipi": product.get("IPI"), "cest": product.get("CEST"), "ex_tipi": product.get("EX_TIPI")}
|
||||
field_proposals.extend(build_field_proposals(code, decision, current))
|
||||
bucket = classify_ipi_bucket(decision)
|
||||
if bucket:
|
||||
ipi_buckets[bucket] += 1
|
||||
|
||||
ipi_rows = [p for p in field_proposals if p["field"] == "IPI"]
|
||||
cest_rows = [p for p in field_proposals if p["field"] == "CEST"]
|
||||
split = split_by_eligibility(ipi_rows)
|
||||
|
||||
ipi_already_correct = sum(1 for p in ipi_rows if p["status"] == "SAME_VALUE")
|
||||
ipi_change_required = sum(1 for p in ipi_rows if p["status"] == "CHANGE_REQUIRED")
|
||||
ipi_write_eligible_rows = split["eligible"]
|
||||
|
||||
cest_db_match = sum(1 for p in cest_rows if p["status"] not in ("UNRESOLVED",) or p["reason_code"] != "NO_CEST_MAPPING_FOUND_FOR_NCM")
|
||||
cest_confirmed = sum(1 for p in cest_rows if p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED", "CURRENT_UNKNOWN"))
|
||||
|
||||
dry_run_artifact = {
|
||||
"artifact": "FULL_DATABASE_DRY_RUN_V1",
|
||||
"total_products": total,
|
||||
"ncm_valid": ncm_valid,
|
||||
"ncm_legacy": ncm_legacy,
|
||||
"ncm_invalid": ncm_invalid,
|
||||
"ncm_missing": ncm_missing,
|
||||
"ipi_buckets": dict(ipi_buckets),
|
||||
"ipi_already_correct": ipi_already_correct,
|
||||
"ipi_change_required": ipi_change_required,
|
||||
"ipi_write_eligible": len(ipi_write_eligible_rows),
|
||||
"cest_db_match_products": cest_db_match,
|
||||
"cest_confirmed": cest_confirmed,
|
||||
"field_proposals": field_proposals,
|
||||
}
|
||||
Path(args.dry_run_output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.dry_run_output).write_text(json.dumps(dry_run_artifact, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
summary = {
|
||||
"total_products": total,
|
||||
"ncm_valid": ncm_valid,
|
||||
"ncm_legacy": ncm_legacy,
|
||||
"ncm_invalid": ncm_invalid,
|
||||
"ncm_missing": ncm_missing,
|
||||
"ipi_buckets": dict(ipi_buckets),
|
||||
"ipi_already_correct": ipi_already_correct,
|
||||
"ipi_change_required": ipi_change_required,
|
||||
"ipi_write_eligible": len(ipi_write_eligible_rows),
|
||||
}
|
||||
|
||||
if args.no_write or not ipi_write_eligible_rows:
|
||||
summary["writes_attempted"] = 0
|
||||
print(json.dumps(summary, indent=2))
|
||||
return
|
||||
|
||||
import fdb
|
||||
connection = fdb.connect(
|
||||
database=database_path,
|
||||
user=os.environ["FB_USER"],
|
||||
password=os.environ["FB_PASSWORD"],
|
||||
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
audit_records = []
|
||||
succeeded = failed = 0
|
||||
try:
|
||||
for proposal in ipi_write_eligible_rows:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
try:
|
||||
record = executor.execute_guarded_update(
|
||||
connection,
|
||||
database_path=database_path,
|
||||
proposal=proposal,
|
||||
now=now,
|
||||
)
|
||||
audit_records.append({**record.as_dict(), "outcome": "SUCCEEDED"})
|
||||
succeeded += 1
|
||||
except executor.WriteRefused as error:
|
||||
audit_records.append({
|
||||
"product_code": proposal["product_code"],
|
||||
"field": proposal["field"],
|
||||
"before_value": proposal["current_value"],
|
||||
"proposed_value": proposal["proposed_value"],
|
||||
"after_value": None,
|
||||
"rowcount": 0,
|
||||
"source_dataset": proposal.get("source_dataset"),
|
||||
"source_version": proposal.get("source_version"),
|
||||
"timestamp": now,
|
||||
"outcome": "FAILED",
|
||||
"error": str(error),
|
||||
})
|
||||
failed += 1
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
Path(args.audit_output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.audit_output).write_text(json.dumps(audit_records, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
# Phase 5: independent re-read + verification of every written CODIGO.
|
||||
written_codes = [r["product_code"] for r in audit_records if r["outcome"] == "SUCCEEDED"]
|
||||
verify_conn = fdb.connect(
|
||||
database=database_path,
|
||||
user=os.environ["FB_USER"],
|
||||
password=os.environ["FB_PASSWORD"],
|
||||
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO,
|
||||
)
|
||||
verified = 0
|
||||
mismatches = []
|
||||
try:
|
||||
cur = verify_conn.cursor()
|
||||
for record in audit_records:
|
||||
if record["outcome"] != "SUCCEEDED":
|
||||
continue
|
||||
cur.execute("SELECT IPI FROM C000025 WHERE CODIGO = ?", (record["product_code"],))
|
||||
row = cur.fetchone()
|
||||
live_value = row[0] if row else None
|
||||
if live_value == record["proposed_value"]:
|
||||
verified += 1
|
||||
else:
|
||||
mismatches.append({"product_code": record["product_code"], "expected": record["proposed_value"], "found": live_value})
|
||||
finally:
|
||||
verify_conn.close()
|
||||
|
||||
summary.update({
|
||||
"writes_attempted": len(ipi_write_eligible_rows),
|
||||
"writes_succeeded": succeeded,
|
||||
"writes_failed": failed,
|
||||
"post_write_verified": verified,
|
||||
"post_write_mismatches": mismatches,
|
||||
"written_codes": written_codes,
|
||||
})
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user