Files
GTI_GERENCIADOR_FISCAL/central/tests/test_reference_registry.py
T
gtiandClaude Sonnet 5 21e8282224 feat: Reference Data Registry (Fase 1) + P1 NCM validated + feasibility studies (Fase 5)
Missão 5:

Fase 1 (Reference Data Registry): migration 0034 adds licence_terms, valid_from/
valid_to, file_size_bytes, raw_file_path, parse_errors, submitted_by/approved_by/
approved_at to the existing `datasets` table (additive only - the 9 pre-existing
dataset kinds keep their current auto-activate behavior in
central/app/datasets/service.py, completely untouched, so this changes nothing
about the currently-running engine). New central/app/reference_registry.py is a
separate, NEW ingestion workflow: register_draft() always lands as DRAFT (never
auto-activates), diff_against_vigente() summarizes a draft against the current
VIGENTE of the same kind for admin review, approve_draft()/reject_draft() are
the only way to promote a DRAFT (refuses non-DRAFT rows and refuses a draft
with recorded parse_errors). New admin screen "Tabelas oficiais"
(/admin/reference-datasets, ADMIN only, CSRF, every decision audited as
REFERENCE_DATASET_APPROVE/REJECT) lists the registry with a diff view and
approve/reject buttons. New CLI (scripts/ingest_reference_dataset.py) downloads
an official source (currently: NCM/Siscomex, no login/captcha), validates
(minimum row count, a known-sample spot check), and registers a DRAFT - --dry-run
never writes anything.

P1 (NCM) validated for real: downloaded the live official Siscomex NCM JSON
(2026-09-28, sha256 8a3bf89a..., 15,156 entries) with an identifiable User-Agent,
one request. Its hash is IDENTICAL to the NCM dataset already ACTIVE in the Hub -
confirms the currently loaded table is already current against the official
source today. Raw file kept out of git (.gitignore:
sources/raw/reference_registry/); manifest (URL, date, hash, size) recorded in
docs/REFERENCE_DATASET_MANIFEST_V1.md, including P2-P6 sources still pending URL
confirmation.

Fase 5 (feasibility studies, no code): docs/VIABILIDADE_CCG_DFE_V1.md - CCG/SEFAZ
GTIN lookup and Distribuição DF-e both require the client's own ICP-Brasil
certificate; recommendation is NOT to implement now (would require either the
Hub holding a client certificate - breaks the project's core security model - or
the Agent making new outbound internet calls, against this mission's fixed
rules). Reading the already-authorized purchase-NFe XML from disk via the Agent
(path already stored in XML_MES) is recommended as the next concrete step
instead: no certificate, no new internet call, opt-in per installation, result
always tagged DECLARADO (never auto-promoted - consensus across purchase
invoices is not fiscal truth). Open Food Facts stays identity-only, unchanged.

21 new tests, all passing. Full suite: 1147 passed, 1 skipped.

No write to any Firebird database anywhere. No Agent code touched.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-28 15:24:55 -03:00

92 lines
4.9 KiB
Python

"""Missão 5 / Fase 1: Reference Data Registry - DRAFT-first ingestion, admin approval, diff summary.
Never auto-activates (unlike the 9 legacy dataset kinds in central/app/datasets/service.py, which
this module never touches)."""
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session
from sqlalchemy.pool import StaticPool
from central.app.models import Base, Dataset
from central.app.reference_registry import RegistryError, approve_draft, diff_against_vigente, list_registry, register_draft, reject_draft
def engine():
e = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
Base.metadata.create_all(e)
return e
def test_register_draft_never_auto_activates():
with Session(engine()) as s:
d = register_draft(s, kind="NCM", provider="Siscomex", source_url="https://x", sha256="a" * 64, version_label="v1", record_count=15156)
assert d.status == "DRAFT"
assert s.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "VIGENTE")) is None
def test_resubmitting_the_same_sha256_returns_the_existing_row_not_a_duplicate():
with Session(engine()) as s:
d1 = register_draft(s, kind="NCM", provider="Siscomex", source_url="https://x", sha256="a" * 64, version_label="v1", record_count=15156)
d2 = register_draft(s, kind="NCM", provider="Siscomex", source_url="https://x", sha256="a" * 64, version_label="v1-again", record_count=99999)
assert d1.id == d2.id and d2.version_label == "v1" # untouched, not overwritten by the resubmission
assert len(list(s.scalars(select(Dataset)))) == 1
def test_approve_promotes_to_vigente_and_supersedes_the_previous_one():
with Session(engine()) as s:
first = register_draft(s, kind="NCM", provider="Siscomex", source_url="https://x", sha256="a" * 64, version_label="v1", record_count=100)
approve_draft(s, first.id, admin_name="admin1")
assert s.get(Dataset, first.id).status == "VIGENTE"
second = register_draft(s, kind="NCM", provider="Siscomex", source_url="https://x", sha256="b" * 64, version_label="v2", record_count=101)
diff = diff_against_vigente(s, second)
assert diff["current_vigente"]["version_label"] == "v1" and diff["record_count_delta"] == 1 and not diff["is_first_version"]
approve_draft(s, second.id, admin_name="admin2")
assert s.get(Dataset, first.id).status == "OBSOLETO"
assert s.get(Dataset, second.id).status == "VIGENTE"
assert s.get(Dataset, second.id).approved_by == "admin2"
def test_approve_refuses_a_dataset_with_parse_errors():
with Session(engine()) as s:
d = register_draft(s, kind="NCM", provider="Siscomex", source_url="https://x", sha256="a" * 64, version_label="v1", record_count=1, parse_errors={"MINIMUM_ENTRIES": "too few"})
try:
approve_draft(s, d.id, admin_name="admin1")
assert False, "expected RegistryError"
except RegistryError as e:
assert "parse errors" in str(e)
assert s.get(Dataset, d.id).status == "DRAFT"
def test_approve_refuses_a_non_draft_dataset():
with Session(engine()) as s:
d = register_draft(s, kind="NCM", provider="Siscomex", source_url="https://x", sha256="a" * 64, version_label="v1", record_count=1)
approve_draft(s, d.id, admin_name="admin1")
try:
approve_draft(s, d.id, admin_name="admin2")
assert False, "expected RegistryError"
except RegistryError as e:
assert "only a DRAFT" in str(e)
def test_reject_never_touches_the_current_vigente():
with Session(engine()) as s:
first = register_draft(s, kind="NCM", provider="Siscomex", source_url="https://x", sha256="a" * 64, version_label="v1", record_count=1)
approve_draft(s, first.id, admin_name="admin1")
bad = register_draft(s, kind="NCM", provider="Siscomex", source_url="https://x", sha256="b" * 64, version_label="v2-bad", record_count=1)
reject_draft(s, bad.id, admin_name="admin2")
assert s.get(Dataset, bad.id).status == "REJECTED"
assert s.get(Dataset, first.id).status == "VIGENTE"
def test_diff_against_vigente_for_the_first_ever_version_of_a_kind():
with Session(engine()) as s:
d = register_draft(s, kind="CEST", provider="Confaz", source_url="https://x", sha256="c" * 64, version_label="v1", record_count=1051)
diff = diff_against_vigente(s, d)
assert diff["is_first_version"] is True and diff["current_vigente"] is None
def test_list_registry_orders_by_kind_then_newest_first():
with Session(engine()) as s:
register_draft(s, kind="NCM", provider="Siscomex", source_url="https://x", sha256="a" * 64, version_label="v1", record_count=1)
register_draft(s, kind="CEST", provider="Confaz", source_url="https://x", sha256="c" * 64, version_label="v1", record_count=1)
rows = list_registry(s)
assert {r.kind for r in rows} == {"NCM", "CEST"}