"""Promote RICMS-BA Anexo 1 (BA_ST_MVA) from the local official PDF snapshot. No network fetch: RICMS-BA Anexo 1 has no stable public direct-download URL (the SEFAZ-BA legislation portal serves it via a session-gated viewer), so the authoritative copy is kept in sources/raw/bootstrap_v1/. This script re-validates it against the currently ACTIVE Postgres dataset and only promotes a new version if the parsed content actually differs, never overwriting a valid ACTIVE version with a partial/regressed parse. """ import hashlib import json from datetime import datetime, timezone from pathlib import Path from pypdf import PdfReader from central.app.datasets.ba_st_mva import parse_page from central.app.datasets.service import DatasetService from central.app.db import SessionLocal from central.app.models import Dataset SOURCE = Path("sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf") SOURCE_URL = f"file://{SOURCE.name}" def main(): raw_bytes = SOURCE.read_bytes() raw_sha256 = hashlib.sha256(raw_bytes).hexdigest() records, rejected = [], [] for page, value in enumerate(PdfReader(SOURCE).pages, start=1): found, failed = parse_page(value.extract_text() or "", page) records.extend(found) rejected.extend(failed) if not records: print(json.dumps({"status": "FAILED", "reason": "parser produced zero records; keeping existing ACTIVE dataset"})) return with SessionLocal() as session: current_active = session.scalar( __import__("sqlalchemy").select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE") ) if current_active is not None and current_active.raw_sha256 == raw_sha256: print(json.dumps({ "status": "UNCHANGED", "dataset_id": current_active.id, "raw_sha256": raw_sha256, "record_count": current_active.record_count, })) return if current_active is not None and len(records) < current_active.record_count * 0.9: # Smoke gate: refuse to promote a parse that lost >10% of records versus # the currently ACTIVE version — never silently replace a good version # with a broken one. print(json.dumps({ "status": "FAILED", "reason": f"new parse yielded {len(records)} records, active has {current_active.record_count} (>10% loss); keeping ACTIVE", })) return dataset = DatasetService(session).import_ba_st_mva(records, raw_sha256, SOURCE_URL, datetime.now(timezone.utc)) print(json.dumps({ "status": dataset.status, "dataset_id": dataset.id, "raw_sha256": raw_sha256, "record_count": dataset.record_count, "rejected": len(rejected), "previously_active_count": current_active.record_count if current_active else None, })) if __name__ == "__main__": main()