71 lines
2.9 KiB
Python
71 lines
2.9 KiB
Python
"""Promote RICMS-BA Anexo 1 (BA_ST_MVA) from the local official PDF snapshot.
|
|
|
|
No network fetch: RICMS-BA Anexo 1 has no stable public direct-download URL
|
|
(the SEFAZ-BA legislation portal serves it via a session-gated viewer), so
|
|
the authoritative copy is kept in sources/raw/bootstrap_v1/. This script
|
|
re-validates it against the currently ACTIVE Postgres dataset and only
|
|
promotes a new version if the parsed content actually differs, never
|
|
overwriting a valid ACTIVE version with a partial/regressed parse.
|
|
"""
|
|
import hashlib
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from pypdf import PdfReader
|
|
|
|
from central.app.datasets.ba_st_mva import parse_page
|
|
from central.app.datasets.service import DatasetService
|
|
from central.app.db import SessionLocal
|
|
from central.app.models import Dataset
|
|
|
|
SOURCE = Path("sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf")
|
|
SOURCE_URL = f"file://{SOURCE.name}"
|
|
|
|
|
|
def main():
|
|
raw_bytes = SOURCE.read_bytes()
|
|
raw_sha256 = hashlib.sha256(raw_bytes).hexdigest()
|
|
|
|
records, rejected = [], []
|
|
for page, value in enumerate(PdfReader(SOURCE).pages, start=1):
|
|
found, failed = parse_page(value.extract_text() or "", page)
|
|
records.extend(found)
|
|
rejected.extend(failed)
|
|
|
|
if not records:
|
|
print(json.dumps({"status": "FAILED", "reason": "parser produced zero records; keeping existing ACTIVE dataset"}))
|
|
return
|
|
|
|
with SessionLocal() as session:
|
|
current_active = session.scalar(
|
|
__import__("sqlalchemy").select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE")
|
|
)
|
|
if current_active is not None and current_active.raw_sha256 == raw_sha256:
|
|
print(json.dumps({
|
|
"status": "UNCHANGED", "dataset_id": current_active.id, "raw_sha256": raw_sha256,
|
|
"record_count": current_active.record_count,
|
|
}))
|
|
return
|
|
if current_active is not None and len(records) < current_active.record_count * 0.9:
|
|
# Smoke gate: refuse to promote a parse that lost >10% of records versus
|
|
# the currently ACTIVE version — never silently replace a good version
|
|
# with a broken one.
|
|
print(json.dumps({
|
|
"status": "FAILED",
|
|
"reason": f"new parse yielded {len(records)} records, active has {current_active.record_count} (>10% loss); keeping ACTIVE",
|
|
}))
|
|
return
|
|
|
|
dataset = DatasetService(session).import_ba_st_mva(records, raw_sha256, SOURCE_URL, datetime.now(timezone.utc))
|
|
|
|
print(json.dumps({
|
|
"status": dataset.status, "dataset_id": dataset.id, "raw_sha256": raw_sha256,
|
|
"record_count": dataset.record_count, "rejected": len(rejected),
|
|
"previously_active_count": current_active.record_count if current_active else None,
|
|
}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|