chore: establish project baseline

central/scripts/import_ba_st_mva.py

@@ -0,0 +1,70 @@
+"""Promote RICMS-BA Anexo 1 (BA_ST_MVA) from the local official PDF snapshot.
+
+No network fetch: RICMS-BA Anexo 1 has no stable public direct-download URL
+(the SEFAZ-BA legislation portal serves it via a session-gated viewer), so
+the authoritative copy is kept in sources/raw/bootstrap_v1/. This script
+re-validates it against the currently ACTIVE Postgres dataset and only
+promotes a new version if the parsed content actually differs, never
+overwriting a valid ACTIVE version with a partial/regressed parse.
+"""
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from pypdf import PdfReader
+
+from central.app.datasets.ba_st_mva import parse_page
+from central.app.datasets.service import DatasetService
+from central.app.db import SessionLocal
+from central.app.models import Dataset
+
+SOURCE = Path("sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf")
+SOURCE_URL = f"file://{SOURCE.name}"
+
+
+def main():
+    raw_bytes = SOURCE.read_bytes()
+    raw_sha256 = hashlib.sha256(raw_bytes).hexdigest()
+
+    records, rejected = [], []
+    for page, value in enumerate(PdfReader(SOURCE).pages, start=1):
+        found, failed = parse_page(value.extract_text() or "", page)
+        records.extend(found)
+        rejected.extend(failed)
+
+    if not records:
+        print(json.dumps({"status": "FAILED", "reason": "parser produced zero records; keeping existing ACTIVE dataset"}))
+        return
+
+    with SessionLocal() as session:
+        current_active = session.scalar(
+            __import__("sqlalchemy").select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE")
+        )
+        if current_active is not None and current_active.raw_sha256 == raw_sha256:
+            print(json.dumps({
+                "status": "UNCHANGED", "dataset_id": current_active.id, "raw_sha256": raw_sha256,
+                "record_count": current_active.record_count,
+            }))
+            return
+        if current_active is not None and len(records) < current_active.record_count * 0.9:
+            # Smoke gate: refuse to promote a parse that lost >10% of records versus
+            # the currently ACTIVE version — never silently replace a good version
+            # with a broken one.
+            print(json.dumps({
+                "status": "FAILED",
+                "reason": f"new parse yielded {len(records)} records, active has {current_active.record_count} (>10% loss); keeping ACTIVE",
+            }))
+            return
+
+        dataset = DatasetService(session).import_ba_st_mva(records, raw_sha256, SOURCE_URL, datetime.now(timezone.utc))
+
+    print(json.dumps({
+        "status": dataset.status, "dataset_id": dataset.id, "raw_sha256": raw_sha256,
+        "record_count": dataset.record_count, "rejected": len(rejected),
+        "previously_active_count": current_active.record_count if current_active else None,
+    }))
+
+
+if __name__ == "__main__":
+    main()

central/scripts/import_bootstrap_v1_extra.py

@@ -0,0 +1,46 @@
+"""Promote CFOP/FCP/ANP/NCM_UTRIB/RTC_BIOFUEL_INDEX/RTC_CCREDPRES from the
+official spreadsheets the operator placed in sources/raw/bootstrap_v1/ after
+the Portal NF-e/SPED downloads were session-blocked. No network fetch.
+"""
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from central.app.datasets import bootstrap_v1_extra as parsers
+from central.app.datasets.service import DatasetService
+from central.app.db import SessionLocal
+
+ROOT = Path("sources/raw/bootstrap_v1")
+
+JOBS = [
+    ("CFOP", ROOT / "IT_2023_002_v2_10_Tabela_CFOP_Descrição.xlsx", parsers.parse_cfop, "import_cfop"),
+    ("FCP", ROOT / "Tabela_Alíquotas_FCP_por_UF_2025-02_07.xlsx", parsers.parse_fcp, "import_fcp"),
+    ("ANP", ROOT / "Tabela_Codigo_Produto_ANP-20241118.xlsx", parsers.parse_anp, "import_anp"),
+    ("NCM_UTRIB", ROOT / "IT2024.001v2.40 - Tabela NCM 2022 com Utrib_Comércio Exterior_vigência 01.10.26.xlsx", parsers.parse_ncm_utrib, "import_ncm_utrib"),
+    ("RTC_BIOFUEL_INDEX", ROOT / "IT 2025.004 v.1.20 - Tabela de Índice.xlsx", parsers.parse_rtc_biofuel_index, "import_rtc_biofuel_index"),
+    ("RTC_CCREDPRES", ROOT / "cCredPres_2026-06-22.xlsx", parsers.parse_rtc_cred_pres, "import_rtc_cred_pres"),
+    ("RTC_CCLASSTRIB", ROOT / "cClassTrib 2026-06-22.xlsx", parsers.parse_rtc_cclasstrib, "import_rtc_cclasstrib"),
+    ("RTC_CBS_RATES", ROOT / "IT 2026.002 v.1.00 - Aliquotas da CBS.xlsx", parsers.parse_rtc_cbs_rates, "import_rtc_cbs_rates"),
+]
+
+
+def main():
+    results = {}
+    with SessionLocal() as session:
+        service = DatasetService(session)
+        for domain, path, parse_fn, method_name in JOBS:
+            try:
+                if not path.exists():
+                    results[domain] = {"status": "MISSING_BOOTSTRAP_FILE", "path": str(path)}
+                    continue
+                payload = parse_fn(path.read_bytes())
+                dataset = getattr(service, method_name)(payload, f"file://{path.name}", datetime.now(timezone.utc))
+                results[domain] = {"status": dataset.status, "dataset_id": dataset.id, "sha256": dataset.sha256, "record_count": dataset.record_count}
+            except Exception as error:  # noqa: BLE001 — isolate one domain's failure from the rest
+                results[domain] = {"status": "FAILED", "reason": f"{type(error).__name__}: {error}"}
+
+    print(json.dumps(results, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

central/scripts/import_cest.py

@@ -0,0 +1,27 @@
+"""Download and promote the official CONFAZ CV142_18 HTML CEST source."""
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+import httpx
+
+from central.app.config import settings
+from central.app.datasets.cest import parse_cest_html
+from central.app.datasets.service import DatasetService
+from central.app.db import SessionLocal
+
+
+def main():
+    response = httpx.get(settings.cest_dataset_url, timeout=settings.http_timeout_seconds, follow_redirects=True)
+    response.raise_for_status()
+    parsed = parse_cest_html(response.content)
+    path = Path(settings.dataset_storage_path)
+    path.mkdir(parents=True, exist_ok=True)
+    (path / f"{parsed.sha256}.html").write_bytes(response.content)
+    with SessionLocal() as session:
+        dataset = DatasetService(session).import_cest_payload(parsed, settings.cest_dataset_url, datetime.now(timezone.utc))
+    print({"id": dataset.id, "status": dataset.status, "sha256": dataset.sha256, "record_count": dataset.record_count, "version": parsed.version})
+
+
+if __name__ == "__main__":
+    main()

central/scripts/import_legacy_ncm.py

@@ -0,0 +1,25 @@
+"""Seed reviewed official legacy NCM evidence; it never selects a successor."""
+
+from central.app.db import SessionLocal
+from central.app.models import LegacyNcmEntry
+
+
+LEGACY_NCM = {
+    "code": "02071400",
+    "source": "Siscomex NCM official nomenclature download",
+    "source_date": "2026-09-17",
+    "reason": "NCM not present in current official nomenclature; no official successor.",
+}
+
+
+def main():
+    with SessionLocal() as session:
+        entry = session.get(LegacyNcmEntry, LEGACY_NCM["code"])
+        if entry is None:
+            session.add(LegacyNcmEntry(**LEGACY_NCM, successor_code=None))
+            session.commit()
+    print({"code": LEGACY_NCM["code"], "status": "LEGACY_NCM", "successor_code": None})
+
+
+if __name__ == "__main__":
+    main()

central/scripts/import_moc_tables.py

@@ -0,0 +1,34 @@
+"""Promote ORIGEM_MERCADORIA/CST_ICMS/CSOSN/CST_PIS/CST_COFINS from the local
+official MOC (Anexo I NF-e/NFC-e) PDF. No network fetch — this manual has no
+stable public direct-download URL either (session-gated portal), so the
+authoritative copy lives in sources/raw/bootstrap_v1/.
+"""
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from central.app.datasets.nfe_moc_tables import parse_moc_tables
+from central.app.datasets.service import DatasetService
+from central.app.db import SessionLocal
+
+SOURCE = Path("sources/raw/bootstrap_v1/moc7-anexo-i-leiaute-e-rv.pdf")
+SOURCE_URL = f"file://{SOURCE.name}"
+
+
+def main():
+    raw_bytes = SOURCE.read_bytes()
+    payload = parse_moc_tables(raw_bytes)
+
+    with SessionLocal() as session:
+        dataset = DatasetService(session).import_moc_tables(payload, SOURCE_URL, datetime.now(timezone.utc))
+
+    from collections import Counter
+    domains = Counter(e.domain for e in payload.entries)
+    print(json.dumps({
+        "status": dataset.status, "dataset_id": dataset.id, "sha256": dataset.sha256,
+        "record_count": dataset.record_count, "domains": dict(domains),
+    }, indent=2))
+
+
+if __name__ == "__main__":
+    main()

central/scripts/import_ncm.py

@@ -0,0 +1,19 @@
+"""Download, persist and promote the official NCM dataset."""
+from datetime import datetime, timezone
+from pathlib import Path
+import httpx
+from central.app.config import settings
+from central.app.datasets.ncm import parse_ncm_payload
+from central.app.datasets.service import DatasetService
+from central.app.db import SessionLocal
+
+def main():
+    response = httpx.get(settings.ncm_dataset_url, timeout=settings.http_timeout_seconds, follow_redirects=True)
+    response.raise_for_status()
+    parsed = parse_ncm_payload(response.content)
+    path = Path(settings.dataset_storage_path); path.mkdir(parents=True, exist_ok=True)
+    (path / f"{parsed.sha256}.json").write_bytes(response.content)
+    with SessionLocal() as session:
+        dataset = DatasetService(session).import_payload(parsed, settings.ncm_dataset_url, datetime.now(timezone.utc))
+    print({"id": dataset.id, "status": dataset.status, "sha256": dataset.sha256, "record_count": dataset.record_count})
+if __name__ == "__main__": main()

central/scripts/inspect_ba_st_mva.py

@@ -0,0 +1,12 @@
+import json
+from pathlib import Path
+from pypdf import PdfReader
+from central.app.datasets.ba_st_mva import parse_page
+
+source = Path("sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf")
+records, rejected = [], []
+for page, value in enumerate(PdfReader(source).pages, start=1):
+    found, failed = parse_page(value.extract_text() or "", page)
+    records.extend(found); rejected.extend(failed)
+Path("docs/ba_st_mva_inspection.json").write_text(json.dumps({"records": records, "rejected": rejected}, ensure_ascii=False, indent=2), encoding="utf-8")
+print(json.dumps({"pages": page, "records": len(records), "rejected": len(rejected), "cest": sum(bool(x["cest"]) for x in records), "ncm": sum(bool(x["ncm_sh"]) for x in records), "mva": sum(bool(x["mva_original"] or x["mva_adjusted"]) for x in records)}))

central/scripts/register_agent.py

@@ -0,0 +1,61 @@
+"""Administrative CLI: register a new Agent directly against Postgres.
+
+No new HTTP endpoint is needed for this — it's a one-time, operator-run
+action per installation, run from wherever DATABASE_URL is reachable (the
+Hub host, or a machine with network access to Postgres). Only the token's
+SHA-256 hash is ever stored; the plaintext token is printed once and must
+be copied into that installation's local Agent config immediately.
+
+Usage:
+    python central/scripts/register_agent.py --company "Restaurante X" --name "loja-01"
+"""
+import argparse
+import secrets
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT / "central"))
+
+from sqlalchemy.orm import Session  # noqa: E402
+
+from app.auth import token_hash  # noqa: E402
+from app.db import engine  # noqa: E402
+from app.models import Agent, Tenant  # noqa: E402
+
+
+def register_agent(session, company_name, agent_name, token=None):
+    token = token or secrets.token_urlsafe(32)
+    tenant = session.query(Tenant).filter_by(name=company_name).one_or_none()
+    if tenant is None:
+        tenant = Tenant(name=company_name)
+        session.add(tenant)
+        session.flush()
+    agent = Agent(tenant_id=tenant.id, name=agent_name, token_hash=token_hash(token), enabled=True)
+    session.add(agent)
+    session.commit()
+    # Read everything the caller needs now, while the session is still open —
+    # session.commit() expires ORM attributes by default, and callers may use
+    # these values after the session (and its `with` block) has closed.
+    return {"id": agent.id, "tenant_id": tenant.id, "enabled": agent.enabled}, token
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--company", required=True, help="Company/tenant name (created if it doesn't exist)")
+    parser.add_argument("--name", required=True, help="Human-readable name for this Agent installation, e.g. 'loja-01'")
+    args = parser.parse_args()
+
+    with Session(engine) as session:
+        info, token = register_agent(session, args.company, args.name)
+
+    print("Agent registered. Copy these into that installation's local .env NOW — the token is shown only once:")
+    print(f"  AGENT_ID={info['id']}")
+    print(f"  AGENT_TOKEN={token}")
+    print(f"  COMPANY_ID={info['tenant_id']}")
+    print("  FIREBIRD_WRITES=0")
+    print(f"enabled={info['enabled']}")
+
+
+if __name__ == "__main__":
+    main()