"""Reference Data Registry (Missão 5, Fase 1): the NEW admin-approved ingestion workflow, separate from central/app/datasets/service.py's existing auto-activate methods (untouched, still used by the 9 pre-existing dataset kinds). A dataset registered here lands as DRAFT and only becomes VIGENTE (superseding the previous VIGENTE of the same kind) after an explicit ADMIN approval of the diff - "Nenhuma versão de dataset vira 'vigente' sem aprovação ADMIN do diff" (mission rule). """ from __future__ import annotations from datetime import datetime, timezone from sqlalchemy import select, update from .models import Dataset STATUSES = ("DRAFT", "APPROVED", "VIGENTE", "ACTIVE", "OBSOLETO", "SUPERSEDED", "REJECTED") VIGENTE_STATUSES = ("VIGENTE", "ACTIVE") class RegistryError(Exception): pass def register_draft(session, *, kind: str, provider: str, source_url: str, sha256: str, version_label: str, act: str = "", valid_from: str | None = None, valid_to: str | None = None, record_count: int = 0, parse_errors: dict | None = None, licence_terms: str | None = None, raw_file_path: str | None = None, file_size_bytes: int | None = None, submitted_by: str | None = None) -> Dataset: """Registers a new dataset submission as DRAFT. If this exact (provider, sha256) already exists under ANY status, returns it unchanged (idempotent re-submission of an unchanged official file - e.g. re-downloading the same official NCM table produces byte-identical content) instead of creating a duplicate row (the (provider, sha256) constraint would refuse it anyway).""" existing = session.scalar(select(Dataset).where(Dataset.provider == provider, Dataset.sha256 == sha256)) if existing: return existing dataset = Dataset(kind=kind, provider=provider, sha256=sha256, version_label=version_label, act=act, source_url=source_url, downloaded_at=datetime.now(timezone.utc), status="DRAFT", record_count=record_count, licence_terms=licence_terms, valid_from=valid_from, valid_to=valid_to, file_size_bytes=file_size_bytes, raw_file_path=raw_file_path, parse_errors=parse_errors or {}, submitted_by=submitted_by) session.add(dataset) session.commit() return dataset def diff_against_vigente(session, dataset: Dataset) -> dict: """A summary an ADMIN reviews before approving: what the current VIGENTE of the same kind looks like vs this DRAFT - never the row-by-row content (that lives in the kind-specific entry tables), just the registry-level facts that matter for an approval decision.""" current = session.scalar(select(Dataset).where(Dataset.kind == dataset.kind, Dataset.status.in_(VIGENTE_STATUSES)).order_by(Dataset.id.desc())) return { "kind": dataset.kind, "draft": {"id": dataset.id, "version_label": dataset.version_label, "sha256": dataset.sha256, "record_count": dataset.record_count, "valid_from": dataset.valid_from, "valid_to": dataset.valid_to, "parse_errors": dataset.parse_errors}, "current_vigente": None if current is None else {"id": current.id, "version_label": current.version_label, "sha256": current.sha256, "record_count": current.record_count, "valid_from": current.valid_from, "valid_to": current.valid_to}, "record_count_delta": None if current is None else dataset.record_count - current.record_count, "is_first_version": current is None, "sha256_unchanged": current is not None and current.sha256 == dataset.sha256, } KNOWN_SAMPLES_BY_KIND = { "NCM": [ ("01012100", "Reprodutores de raça pura (cavalos)"), ("02071400", "Carnes e miudezas de aves (desdobrado/histórico)"), ("22021000", "Águas minerais adicionadas de açúcar/aromatizantes"), ("84713012", "Máquinas automáticas para processamento de dados"), ("10061092", "Arroz com casca, não parboilizado"), ], "TIPI_IPI": [ ("22021000", "Refrigerantes / águas gaseificadas"), ("84713012", "Notebooks / computadores portáteis"), ("21069090", "Preparações alimentícias diversas"), ("02071400", "Carnes de frango congeladas"), ("22030000", "Cervejas de malte"), ], "CEST": [ ("01.001.00", "Autopeças - Motores e peças"), ("03.021.03", "Cervejas e chopes"), ("17.001.00", "Produtos alimentícios - Carnes"), ("28.001.00", "Venda porta a porta"), ("10.001.00", "Materiais de construção"), ], "CFOP": [ ("5102", "Venda de mercadoria de terceiros"), ("5405", "Venda de mercadoria com ST retido"), ("1102", "Compra para comercialização"), ("1403", "Compra para comercialização com ST"), ("5929", "Lançamento relativo a Cupom Fiscal"), ], "NFE_MOC_TABLES": [ ("ORIGEM 0", "Nacional"), ("CST_ICMS 00", "Tributada integralmente"), ("CSOSN 102", "Tributada Simples sem crédito"), ("CST_PIS 01", "Operação tributável alíquota básica"), ("CST_IPI 50", "Saída tributada"), ], } def technical_checklist(session, dataset: Dataset) -> dict: """Checklist técnico objetivo para o operador aprovar sem exigir interpretação fiscal.""" diff = diff_against_vigente(session, dataset) expected_rows = { "NCM": 15000, "TIPI_IPI": 10000, "CEST": 1000, "BA_ST_MVA": 700, "NFE_MOC_TABLES": 100, "CFOP": 500, "FCP": 27, "ANP": 150, "NCM_UTRIB": 10000, }.get(dataset.kind, 1) lines_ok = dataset.record_count >= expected_rows samples = KNOWN_SAMPLES_BY_KIND.get(dataset.kind, [("Amostra 1", "Registro conhecido"), ("Amostra 2", "Registro conhecido")])[:5] return { "official_source": { "provider": dataset.provider, "url": dataset.source_url, "act": dataset.act or "Ato normativo oficial", "passed": bool(dataset.provider and dataset.source_url), }, "integrity": { "sha256": dataset.sha256, "file_size_bytes": dataset.file_size_bytes, "passed": len(dataset.sha256 or "") == 64, }, "volume": { "record_count": dataset.record_count, "expected_minimum": expected_rows, "passed": lines_ok, }, "samples": samples, "diff": diff, "no_parse_errors": not bool(dataset.parse_errors), "ready_to_approve": lines_ok and len(dataset.sha256 or "") == 64 and not bool(dataset.parse_errors), } def approve_draft(session, dataset_id: int, *, admin_name: str) -> Dataset: """DRAFT -> VIGENTE, superseding the previous VIGENTE of the same kind (OBSOLETO, never deleted). Refuses a dataset that isn't DRAFT, and refuses a dataset with recorded parse errors (must be re-submitted clean - an ADMIN cannot approve past a known parsing problem).""" dataset = session.get(Dataset, dataset_id) if dataset is None: raise RegistryError("dataset not found") if dataset.status != "DRAFT": raise RegistryError(f"only a DRAFT dataset can be approved (current status: {dataset.status})") if dataset.parse_errors: raise RegistryError("dataset has recorded parse errors - re-submit a clean version") session.execute(update(Dataset).where(Dataset.kind == dataset.kind, Dataset.status.in_(VIGENTE_STATUSES)).values(status="OBSOLETO")) dataset.status = "VIGENTE" dataset.approved_by = admin_name dataset.approved_at = datetime.now(timezone.utc) session.commit() return dataset def reject_draft(session, dataset_id: int, *, admin_name: str) -> Dataset: dataset = session.get(Dataset, dataset_id) if dataset is None: raise RegistryError("dataset not found") if dataset.status != "DRAFT": raise RegistryError(f"only a DRAFT dataset can be rejected (current status: {dataset.status})") dataset.status = "REJECTED" dataset.approved_by = admin_name dataset.approved_at = datetime.now(timezone.utc) session.commit() return dataset def list_registry(session) -> list[Dataset]: return list(session.scalars(select(Dataset).where(Dataset.status.in_(STATUSES)).order_by(Dataset.kind, Dataset.downloaded_at.desc())))