GTI_GERENCIADOR_FISCAL/scripts/ingest_reference_dataset.py

"""Missão 5 / Fase 1.3 — CLI ingestion for the Reference Data Registry.

Read-only against Firebird (never touches it); writes only to the Hub's own Postgres, and only a
DRAFT row (never auto-activates - an ADMIN approves the diff separately, via the admin screen or
scripts/approve_reference_dataset.py). Rate-limit friendly: one HTTP GET per invocation, identifiable
User-Agent, respects Cache-Control (--force to bypass the local file cache).

    python scripts/ingest_reference_dataset.py ncm --dry-run
    python scripts/ingest_reference_dataset.py ncm

Currently supports: ncm (Siscomex, no login/captcha, JSON, official).
"""
import argparse
import hashlib
import json
import sys
import urllib.request
from datetime import date
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
RAW_DIR = ROOT / "sources" / "raw" / "reference_registry"
USER_AGENT = "GTI-Fiscal-Hub/1.0 (dataset ingestion; contato: operador GTI)"

SOURCES = {
    "ncm": {
        "kind": "NCM", "provider": "Siscomex",
        "url": "https://portalunico.siscomex.gov.br/classif/api/publico/nomenclatura/download/json",
        "licence": "Dado público oficial (Governo Federal / Siscomex); sem termos de uso restritivos conhecidos para consulta e reuso de classificação fiscal.",
    },
}


def fetch(name: str, *, force: bool) -> Path:
    src = SOURCES[name]
    RAW_DIR.mkdir(parents=True, exist_ok=True)
    out = RAW_DIR / f"{name}_{date.today().isoformat()}.json"
    if out.exists() and not force:
        print(f"CACHE_HIT: {out} (use --force para baixar de novo)")
        return out
    req = urllib.request.Request(src["url"], headers={"User-Agent": USER_AGENT})
    with urllib.request.urlopen(req, timeout=120) as resp:
        data = resp.read()
    out.write_bytes(data)
    print(f"DOWNLOADED: {out} ({len(data)} bytes)")
    return out


def validate_and_parse(name: str, path: Path):
    from central.app.datasets.ncm import parse_ncm_payload
    if name != "ncm":
        raise SystemExit(f"fonte '{name}' ainda não tem parser registrado")
    payload = payload_bytes = path.read_bytes()
    parsed = parse_ncm_payload(payload_bytes)
    errors = {}
    if len(parsed.entries) < 1000:
        errors["MINIMUM_ENTRIES"] = f"apenas {len(parsed.entries)} entradas (esperado >= 1000)"
    # amostra conhecida: capítulo "01" (Animais vivos) deve sempre existir na tabela vigente
    if not any(e.code == "01" for e in parsed.entries):
        errors["KNOWN_SAMPLE_MISSING"] = "capítulo NCM '01' não encontrado"
    return parsed, errors


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("source", choices=sorted(SOURCES))
    ap.add_argument("--dry-run", action="store_true")
    ap.add_argument("--force", action="store_true")
    a = ap.parse_args()
    src = SOURCES[a.source]
    path = fetch(a.source, force=a.force)
    sha256 = hashlib.sha256(path.read_bytes()).hexdigest()
    size = path.stat().st_size
    parsed, errors = validate_and_parse(a.source, path)
    report = {"source": a.source, "kind": src["kind"], "provider": src["provider"], "url": src["url"], "sha256": sha256,
             "file_size_bytes": size, "record_count": len(parsed.entries), "version_label": parsed.version, "act": parsed.act,
             "parse_errors": errors, "raw_file_path": str(path.relative_to(ROOT))}
    print(json.dumps(report, ensure_ascii=False, indent=1))
    if a.dry_run:
        print("DRY_RUN: nada foi gravado no Postgres.")
        return
    if errors:
        print("PARSE_ERRORS presentes: registrando como DRAFT com parse_errors preenchido (não pode ser aprovado até corrigir).")
    from central.app.reference_registry import register_draft
    from central.app.main import create_app  # noqa: F401  (ensures models are imported/registered)
    from sqlalchemy import create_engine
    from sqlalchemy.orm import Session
    import os
    engine = create_engine(os.environ["DATABASE_URL"])
    with Session(engine) as session:
        dataset = register_draft(session, kind=src["kind"], provider=src["provider"], source_url=src["url"], sha256=sha256,
                                  version_label=parsed.version, act=parsed.act, record_count=len(parsed.entries), parse_errors=errors,
                                  licence_terms=src["licence"], raw_file_path=report["raw_file_path"], file_size_bytes=size, submitted_by="cli")
        print(f"REGISTERED: dataset_id={dataset.id} status={dataset.status}")


if __name__ == "__main__":
    main()