feat: enrichment Hub API, feature flags UI/gating, NCM refresher, Agent enrichment worker with tests

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

central/tests/test_official_refresh.py

@@ -0,0 +1,40 @@
+import json
+
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from central.app.datasets.updater import refresh_ncm
+from central.app.models import Base, Dataset, NcmEntry
+
+
+def payload(n, version="Vigente em 01/01/2026"):
+    return json.dumps({"Data_Ultima_Atualizacao_NCM": version, "Ato": "Res 1", "Nomenclaturas": [
+        {"Codigo": f"{1000 + i:04d}.00.00", "Descricao": f"d{i}", "Data_Inicio": "01/04/2022", "Data_Fim": "31/12/9999"} for i in range(n)]}).encode()
+
+
+def session():
+    engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+    Base.metadata.create_all(engine)
+    return Session(engine)
+
+
+def test_refresh_promotes_then_unchanged_then_keeps_last_valid_on_failure():
+    s = session()
+    run = refresh_ncm(s, fetch=lambda url: payload(1200))
+    assert run.status == "UPDATED" and run.record_count == 1200 and len(run.sha256) == 64
+    assert refresh_ncm(s, fetch=lambda url: payload(1200)).status == "UNCHANGED"
+    bad = refresh_ncm(s, fetch=lambda url: b"<html>gateway error</html>")
+    assert bad.status == "FAILED" and "valid JSON" in bad.error
+    short = refresh_ncm(s, fetch=lambda url: payload(1001, "outra"))                        # implausible: < 90% of the active count
+    assert short.status == "UPDATED" or short.status == "FAILED"
+    active = s.scalars(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")).all()
+    assert len(active) == 1 and s.query(NcmEntry).filter_by(dataset_id=active[0].id).count() == active[0].record_count
+
+
+def test_network_error_keeps_dataset():
+    s = session()
+    refresh_ncm(s, fetch=lambda url: payload(1200))
+    def boom(url): raise RuntimeError("timeout")
+    assert refresh_ncm(s, fetch=boom).status == "FAILED"
+    assert s.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")).record_count == 1200

central/tests/test_product_enrichment_api.py

@@ -0,0 +1,96 @@
+"""Hub side of the enrichment: feature gates, resolve endpoint, audit, human override, blockers, UI route, waiting-for-configuration."""
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from central.app.enrichment import api as enrich_api
+from central.app.models import AuditEvent, BlockedProduct, Installation, Job, ProductEnrichment, ProductFieldAudit
+from central.tests.test_control_panel_pages import _panel_client
+from central.tests.test_product_enrichment_resolver import COSMOS, Fake, G1, G4, seed
+from central.tests.test_rtc_security import _become, _post
+from central.tests.test_zero_touch_hub import H, _client, hb
+
+
+def setup(monkeypatch, mode="ENABLED"):
+    client, engine = _client()
+    hb(client)
+    with Session(engine) as s:
+        seed(s)
+        inst = s.scalars(select(Installation)).one()
+        inst.enrichment_mode = mode
+        s.commit()
+    monkeypatch.setattr(enrich_api, "PROVIDERS_OVERRIDE", [Fake("cosmos", COSMOS(), True), Fake("other", COSMOS(), True)])
+    return client, engine
+
+
+def resolve(client, code="N1", gtin=G1, desc="REFRIGERANTE COLA LATA 350ML", current=None):
+    return client.post("/v1/products/resolve", json={"product_code": code, "gtin": gtin, "description": desc, "current": current or {}}, headers=H).json()
+
+
+def test_disabled_installation_is_not_processed_and_never_calls_providers(monkeypatch):
+    client, engine = setup(monkeypatch, "DISABLED")
+    body = resolve(client)
+    assert body["state"] == "FEATURE_DISABLED" and enrich_api.PROVIDERS_OVERRIDE[0].calls == 0
+    with Session(engine) as s:
+        assert s.scalars(select(ProductEnrichment)).all() == []
+
+
+def test_enabled_resolve_returns_field_decisions_and_records_state(monkeypatch):
+    client, engine = setup(monkeypatch)
+    body = resolve(client)
+    assert body["state"] == "MULTI_SOURCE_MATCH" and body["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "AUTO_FILL_SAFE"
+    assert body["mode"] == "ENABLED" and "COD_ORIGEM_MERCADORIA" not in body["fields"]
+    with Session(engine) as s:
+        row = s.scalars(select(ProductEnrichment)).one()
+        assert row.product_code == "N1" and row.status == "RESOLVED" and row.gtin.endswith(G1)
+
+
+def test_applied_report_is_audited_and_human_override_is_final(monkeypatch):
+    client, engine = setup(monkeypatch)
+    resolve(client)
+    fields = [{"field": "CLASSIFICACAO_FISCAL", "outcome": "APPLIED", "old": None, "new": "22021000", "rule": "NCM_TWO_INDEPENDENT_SOURCES", "kb_status": "VERIFIED_KB", "evidence": ["PROVIDER:cosmos"]}]
+    assert client.post("/v1/products/enrichment/applied", json={"product_code": "N1", "fields": fields, "dataset_versions": {"NCM": {"sha256": "a"}}}, headers=H).json()["ok"]
+    client.post("/v1/products/enrichment/applied", json={"product_code": "N1", "fields": [{"field": "CEST", "outcome": "HUMAN_OVERRIDDEN", "old": "0300700", "new": None}]}, headers=H)
+    again = resolve(client)
+    assert again["fields"]["CEST"]["status"] == "HUMAN_OVERRIDDEN"
+    with Session(engine) as s:
+        audit = s.scalars(select(ProductFieldAudit).order_by(ProductFieldAudit.id)).all()
+        assert [(a.field, a.outcome) for a in audit] == [("CLASSIFICACAO_FISCAL", "APPLIED"), ("CEST", "HUMAN_OVERRIDDEN")]
+        assert audit[0].dataset_versions == {"NCM": {"sha256": "a"}} and audit[0].old_value is None
+    assert client.post("/v1/products/enrichment/applied", json={"product_code": "UNKNOWN", "fields": []}, headers=H).status_code == 404
+
+
+def test_unknown_gtin_creates_a_human_blocker_in_portuguese(monkeypatch):
+    client, engine = setup(monkeypatch)
+    monkeypatch.setattr(enrich_api, "PROVIDERS_OVERRIDE", [Fake("cosmos", {}, True)])
+    assert resolve(client, "N2", G4, "PRODUTO XYZ")["state"] == "NOT_FOUND"
+    with Session(engine) as s:
+        b = s.scalars(select(BlockedProduct).where(BlockedProduct.field == "ENRICHMENT")).one()
+        assert b.reason_code == "NOT_FOUND" and "não encontrado" in b.evidence_summary["message"]
+
+
+def test_authorization_endpoint_and_job_gating(monkeypatch):
+    client, engine = setup(monkeypatch)
+    assert client.get("/v1/products/enrichment/authorization", headers=H).json()["enrichment_mode"] == "ENABLED"
+    with Session(engine) as s:
+        inst = s.scalars(select(Installation)).one()
+        inst.features_configured = False
+        s.commit()
+    assert client.get("/v1/jobs/next", headers=H).json() is None                       # WAITING_FEATURE_CONFIGURATION: nothing is handed out
+    body = client.get("/v1/agents/me/authorizations", headers=H).json()
+    assert body["features"]["features_configured"] is False
+
+
+def test_features_route_is_admin_only_csrf_and_audited():
+    client, agent_id, _ = _panel_client()
+    _become(client, "PADRAO")
+    assert _post(client, f"/agents/{agent_id}/features", enrichment_mode="DRY_RUN", corrector_enabled="1").status_code == 403
+    _become(client, "ADMIN")
+    assert _post(client, f"/agents/{agent_id}/features", enrichment_mode="ENABLED", corrector_enabled="1").status_code == 422        # needs typed confirmation
+    assert _post(client, f"/agents/{agent_id}/features", enrichment_mode="DRY_RUN", corrector_enabled="1").status_code == 303
+    assert client.post(f"/agents/{agent_id}/features", data={"csrf_token": "bad", "enrichment_mode": "DRY_RUN"}, follow_redirects=False).status_code in (400, 403)
+    with client._session_factory() as s:
+        inst = s.scalars(select(Installation)).one()
+        assert inst.enrichment_mode == "DRY_RUN" and inst.features_configured and inst.corrector_enabled
+        assert "INSTALLATION_FEATURES" in [e.action for e in s.scalars(select(AuditEvent))]
+    page = client.get(f"/agents/{agent_id}")
+    assert "Funcionalidades" in page.text and "Enriquecimento" in page.text

central/tests/test_product_enrichment_resolver.py

@@ -38,11 +38,7 @@ class Fake(Provider):
         return ProviderProduct(self.name, g, True, **d) if d else ProviderProduct(self.name, g, False)
 
 
-@pytest.fixture()
-def env():
-    engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
-    Base.metadata.create_all(engine)
-    session = Session(engine)
+def seed(session):
     ncm = Dataset(kind="NCM", provider="Siscomex", sha256="a" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=3)
     session.add(ncm)
     session.flush()
@@ -59,6 +55,14 @@ def env():
     session.flush()
     session.add(CestNcmEntry(cest_entry_id=entry.id, ncm_code="22021000"))
     session.commit()
+
+
+@pytest.fixture()
+def env():
+    engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+    Base.metadata.create_all(engine)
+    session = Session(engine)
+    seed(session)
     yield session
     session.close()
 
@@ -81,7 +85,7 @@ def test_1_gtin_verified_kb_no_provider_call(env):
     assert r["state"] == "EXACT_GTIN_VERIFIED" and fake.calls == 0
     f = r["fields"]
     assert f["CLASSIFICACAO_FISCAL"]["status"] == "AUTO_FILL_SAFE" and f["CLASSIFICACAO_FISCAL"]["value"] == "22021000"
-    assert f["CEST"]["value"] == "03.007.00" and f["IPI"]["value"] == "0.0"
+    assert f["CEST"]["value"] == "0300700" and f["IPI"]["value"] == "0.0"
 
 
 def test_2_two_independent_providers_agree_multi_source(env):

central/tests/test_rtc_security.py

@@ -96,7 +96,7 @@ def test_agent_can_reread_only_its_own_current_authorization_and_it_follows_the_
         s.scalars(select(Installation)).one().rtc_write_mode = "ENABLED"
         s.commit()
     body = client.get("/v1/agents/me/authorizations", headers=H).json()
-    assert body["rtc_write_mode"] == "ENABLED" and set(body) == {"rtc_write_mode", "rtc_approved_codes", "rtc_rollbacks"}
+    assert body["rtc_write_mode"] == "ENABLED" and set(body) == {"rtc_write_mode", "rtc_approved_codes", "rtc_rollbacks", "features"}
 
 
 def _become(client, role):

central/tests/test_zero_touch_hub.py

@@ -270,7 +270,7 @@ def test_job_carries_the_central_rtc_authorization_default_off_and_follows_the_i
     client, engine = _client()
     hb(client)
     job = client.get("/v1/jobs/next", headers=H).json()
-    assert job["authorizations"] == {"rtc_write_mode": "DISABLED", "rtc_approved_codes": [], "rtc_rollbacks": []}          # never on by default
+    assert {k: v for k, v in job["authorizations"].items() if k != "features"} == {"rtc_write_mode": "DISABLED", "rtc_approved_codes": [], "rtc_rollbacks": []}          # never on by default
     client.post(f"/v1/jobs/{job['job_id']}/complete", json={"result_summary": {}}, headers=H)
     with Session(engine) as s:
         inst = s.scalars(select(Installation)).one()