feat: enrichment Hub API, feature flags UI/gating, NCM refresher, Agent enrichment worker with tests

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-25 18:47:35 -03:00
parent 27f992d9e0
commit 5deebea14e
22 changed files with 1118 additions and 23 deletions
@@ -0,0 +1,201 @@
+"""Agent enrichment: baseline/new-product detection, feature gating, typed write, second run = 0 writes, human override (incl. deleted value)."""
+import sqlite3
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT / "agent" / "src"))
+from gti_fiscal_agent import enrichment as E  # noqa: E402
+
+
+class Conn:
+    """sqlite stand-in for the Firebird connection; close() is a no-op so the worker can 'open' it repeatedly."""
+
+    def __init__(self, db):
+        self.db = db
+
+    def cursor(self):
+        return self.db.cursor()
+
+    def commit(self):
+        self.db.commit()
+
+    def rollback(self):
+        self.db.rollback()
+
+    def close(self):
+        pass
+
+
+class Hub:
+    def __init__(self, features, decisions=None, state="EXACT_GTIN_VERIFIED"):
+        self.features = features
+        self.decisions = decisions or {}
+        self.state = state
+        self.resolve_calls, self.reports, self.overridden = [], [], {}
+
+    def authorizations(self):
+        return {"features": dict(self.features)}
+
+    def resolve_product(self, payload):
+        self.resolve_calls.append(payload)
+        fields = {k: dict(v) for k, v in self.decisions.items()}
+        for f in list(fields):
+            if f in self.overridden.get(payload["product_code"], set()):
+                fields[f] = {"status": "HUMAN_OVERRIDDEN", "value": None}
+            elif payload["current"].get(f):
+                fields[f] = {"status": "ALREADY_SET", "value": None}
+        return {"state": self.state, "fields": fields, "dataset_versions": {"NCM": {"sha256": "abc"}}}
+
+    def report_enrichment(self, payload):
+        self.reports.append(payload)
+        for item in payload["fields"]:
+            if item["outcome"] == "HUMAN_OVERRIDDEN":
+                self.overridden.setdefault(payload["product_code"], set()).add(item["field"])
+        return {"ok": True}
+
+
+ENABLED = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}
+DECISIONS = {"CLASSIFICACAO_FISCAL": {"status": "AUTO_FILL_SAFE", "value": "22021000", "rule": "NCM_KB_VERIFIED", "kb_status": "VERIFIED_KB"},
+             "CEST": {"status": "AUTO_FILL_SAFE", "value": "0300700", "rule": "CEST_OFFICIAL_UNIQUE_MAPPING"},
+             "IPI": {"status": "AUTO_FILL_SAFE", "value": "0.0", "rule": "IPI_TIPI_NUMERIC"}}
+
+
+@pytest.fixture()
+def env(tmp_path):
+    db = sqlite3.connect(":memory:", check_same_thread=False)
+    db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT)")
+    db.executemany("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", [(f"H{i}", f"HIST {i}", "", "keep") for i in range(50)])
+    db.commit()
+    hub = Hub(dict(ENABLED), DECISIONS)
+    store = E.EnrichmentStore(tmp_path / "e.db")
+    now = [1000.0]
+    reproc = []
+    worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=reproc.append, clock=lambda: now[0], sleep=lambda s: None)
+    return db, hub, store, worker, now, reproc
+
+
+def add(db, code, name="REFRIGERANTE COLA LATA 350ML", gtin="7891000100103"):
+    db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", (code, name, gtin, "keep"))
+    db.commit()
+
+
+def row(db, code):
+    return db.execute("SELECT CLASSIFICACAO_FISCAL,CEST,IPI,OTHER FROM C000025 WHERE CODIGO=?", (code,)).fetchone()
+
+
+def test_baseline_never_sends_historical_products(env):
+    db, hub, store, worker, now, _ = env
+    assert worker.step() == []                       # baseline
+    now[0] += 5
+    assert worker.step() == [] and hub.resolve_calls == []
+    assert len(store.known()) == 50
+
+
+def test_new_product_is_enriched_only_in_empty_fields_then_second_run_zero_writes(env):
+    db, hub, store, worker, now, reproc = env
+    worker.step()
+    add(db, "N1")
+    now[0] += 5
+    assert worker.step() == ["N1"]
+    assert row(db, "N1") == ("22021000", "0300700", 0.0, "keep")
+    assert reproc == [["N1"]]
+    assert row(db, "H1") == (None, None, None, "keep")                 # historical untouched
+    assert [r["outcome"] for r in hub.reports[0]["fields"]] == ["APPLIED"] * 3
+    now[0] += 5
+    before = db.total_changes
+    worker.step()
+    assert db.total_changes == before and len(hub.resolve_calls) == 1  # nothing re-processed, 0 writes
+
+
+def test_existing_value_is_never_overwritten(env):
+    db, hub, store, worker, now, _ = env
+    worker.step()
+    add(db, "N2")
+    db.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N2'")
+    db.commit()
+    now[0] += 5
+    worker.step()
+    assert row(db, "N2")[0] == "19059090" and row(db, "N2")[1] == "0300700"
+
+
+def test_dry_run_and_disabled_and_waiting_do_not_write_or_touch_firebird(env):
+    db, hub, store, worker, now, _ = env
+    worker.step()
+    add(db, "N3")
+    for features in ({"features_configured": False}, {"features_configured": True, "enrichment_mode": "DISABLED"}):
+        hub.features = features
+        worker._features_at = -999
+        assert worker.step() == [] and hub.resolve_calls == []
+    hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
+    worker._features_at = -999
+    now[0] += 5
+    worker.step()
+    assert row(db, "N3") == (None, None, None, "keep") and hub.reports[0]["fields"][0]["outcome"] == "DRY_RUN"
+
+
+def test_authorization_revoked_right_before_write_fails_closed(env):
+    db, hub, store, worker, now, _ = env
+    worker.step()
+    add(db, "N4")
+    calls = {"n": 0}
+
+    def flip():
+        calls["n"] += 1
+        return {"features": dict(ENABLED if calls["n"] == 1 else {"features_configured": True, "enrichment_mode": "DISABLED"})}
+    hub.authorizations = flip
+    now[0] += 5
+    worker._features_at = -999
+    worker.step()
+    assert row(db, "N4") == (None, None, None, "keep")
+    assert all(f["outcome"] == "ABORTED" and f["reason"] == "AUTHORIZATION_CHANGED" for f in hub.reports[0]["fields"])
+
+
+def test_human_override_including_deleted_value_is_final(env):
+    db, hub, store, worker, now, _ = env
+    worker.step()
+    add(db, "N5")
+    now[0] += 5
+    worker.step()
+    db.execute("UPDATE C000025 SET CEST='', CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N5'")     # a person deletes the CEST and changes the NCM
+    db.commit()
+    now[0] += 30
+    worker.step()
+    assert store.is_overridden("N5", "CEST") and store.is_overridden("N5", "CLASSIFICACAO_FISCAL") and not store.is_overridden("N5", "IPI")
+    # the description changes a moment later: the worker re-resolves, but overridden fields must NOT be written again
+    db.execute("UPDATE C000025 SET PRODUTO='REFRIGERANTE COLA LATA 350ML ZERO' WHERE CODIGO='N5'")
+    db.commit()
+    now[0] += 30
+    worker.step()
+    assert row(db, "N5")[:2] == ("19059090", "")
+
+
+def test_write_field_allowlist_and_shape():
+    db = sqlite3.connect(":memory:")
+    db.execute("CREATE TABLE C000025(CODIGO TEXT, COD_ORIGEM_MERCADORIA TEXT, CLASSIFICACAO_FISCAL TEXT)")
+    with pytest.raises(ValueError):
+        E.write_field(Conn(db), "1", "COD_ORIGEM_MERCADORIA", "1")
+    with pytest.raises(ValueError):
+        E.write_field(Conn(db), "1", "CCLASSTRIB", "000001")
+    db.execute("INSERT INTO C000025(CODIGO) VALUES('1')")
+    db.commit()
+    assert E.write_field(Conn(db), "1", "CLASSIFICACAO_FISCAL", "123")["reason"] == "VALUE_SHAPE_INVALID"
+
+
+def test_hub_offline_keeps_product_pending_and_retries(env):
+    db, hub, store, worker, now, _ = env
+    worker.step()
+    add(db, "N6")
+
+    def boom(payload):
+        raise ConnectionError("down")
+    good = hub.resolve_product
+    hub.resolve_product = boom
+    now[0] += 5
+    worker.step()
+    assert store.get("N6")["state"] == "RETRY" and row(db, "N6")[0] is None
+    hub.resolve_product = good
+    now[0] += 40
+    assert worker.step() == ["N6"] and row(db, "N6")[0] == "22021000"