chore: establish project baseline

2026-09-21 15:51:47 -03:00
commit 9dcc263217
459 changed files with 447890 additions and 0 deletions
@@ -0,0 +1,70 @@
+"""Promote RICMS-BA Anexo 1 (BA_ST_MVA) from the local official PDF snapshot.
+
+No network fetch: RICMS-BA Anexo 1 has no stable public direct-download URL
+(the SEFAZ-BA legislation portal serves it via a session-gated viewer), so
+the authoritative copy is kept in sources/raw/bootstrap_v1/. This script
+re-validates it against the currently ACTIVE Postgres dataset and only
+promotes a new version if the parsed content actually differs, never
+overwriting a valid ACTIVE version with a partial/regressed parse.
+"""
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from pypdf import PdfReader
+
+from central.app.datasets.ba_st_mva import parse_page
+from central.app.datasets.service import DatasetService
+from central.app.db import SessionLocal
+from central.app.models import Dataset
+
+SOURCE = Path("sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf")
+SOURCE_URL = f"file://{SOURCE.name}"
+
+
+def main():
+    raw_bytes = SOURCE.read_bytes()
+    raw_sha256 = hashlib.sha256(raw_bytes).hexdigest()
+
+    records, rejected = [], []
+    for page, value in enumerate(PdfReader(SOURCE).pages, start=1):
+        found, failed = parse_page(value.extract_text() or "", page)
+        records.extend(found)
+        rejected.extend(failed)
+
+    if not records:
+        print(json.dumps({"status": "FAILED", "reason": "parser produced zero records; keeping existing ACTIVE dataset"}))
+        return
+
+    with SessionLocal() as session:
+        current_active = session.scalar(
+            __import__("sqlalchemy").select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE")
+        )
+        if current_active is not None and current_active.raw_sha256 == raw_sha256:
+            print(json.dumps({
+                "status": "UNCHANGED", "dataset_id": current_active.id, "raw_sha256": raw_sha256,
+                "record_count": current_active.record_count,
+            }))
+            return
+        if current_active is not None and len(records) < current_active.record_count * 0.9:
+            # Smoke gate: refuse to promote a parse that lost >10% of records versus
+            # the currently ACTIVE version — never silently replace a good version
+            # with a broken one.
+            print(json.dumps({
+                "status": "FAILED",
+                "reason": f"new parse yielded {len(records)} records, active has {current_active.record_count} (>10% loss); keeping ACTIVE",
+            }))
+            return
+
+        dataset = DatasetService(session).import_ba_st_mva(records, raw_sha256, SOURCE_URL, datetime.now(timezone.utc))
+
+    print(json.dumps({
+        "status": dataset.status, "dataset_id": dataset.id, "raw_sha256": raw_sha256,
+        "record_count": dataset.record_count, "rejected": len(rejected),
+        "previously_active_count": current_active.record_count if current_active else None,
+    }))
+
+
+if __name__ == "__main__":
+    main()