Files
GTI_GERENCIADOR_FISCAL/agent/tests/test_enrichment.py
T
gti 7e83d17533 feat: wire field_contract into product enrichment; auto-discover CRT/CONTRIBUINTE_IPI at heartbeat; targeted per-product ERP evidence
field_contract.resolve_product was only exercised by its own unit tests. /v1/products/resolve now calls it for every product whose NCM
resolves, merging CFOP/CST/CSOSN/PIS-COFINS/CST_IPI/SITUACAO_TRIBUTARIA/IAT/IPPT decisions alongside CLASSIFICACAO_FISCAL/CEST/IPI - reusing
the same engine, the same FiscalContextEntry table the blocker-resolution workflow already writes to, no duplicated rules.

Closes three real transport/discovery gaps found while wiring it in:
- Agent heartbeat never sent company_crt (env-var UF only) or company_contribuinte_ipi, so Installation.company_crt stayed permanently empty
  even though company_context.resolve() already proves both from C000004/EMISSOR for every coverage job. Resolved once, cached, retried only
  while unresolved. Migration 0032 adds Installation.company_contribuinte_ipi (NULL=unknown, never coerced to "N").
- The Agent's enrichment payload only ever reported 3 old fields (CLASSIFICACAO_FISCAL/CEST/IPI) as "current", so the Hub could never see the
  product's own CFOP/CST/CSOSN/... nor the classification columns (TIPO/PROD_COZINHA/PRODUTO_SERVICO/CODGRUPO) infer_role needs - separated
  into a read-only CURRENT_REPORT_FIELDS list, the write allowlist (PRODUCT_ENRICHMENT_WRITE_V1) untouched.
- Added a targeted, product_code-scoped evidence query (firebird_reader.read_product_evidence: recipes/composed/purchases/stock, indexed WHERE,
  never a full-table scan) so st_withheld/product_role can be proven for a single ad-hoc resolve, not only during a full sync job.

Verified read-only against the real database (D:\PROJETOS\GTISISTEMAS\Banco\GTI.FDB): CRT=1, CONTRIBUINTE_IPI=N, company_context.resolve()
reaches HIGH confidence; infer_role correctly resolved RESALE/OWN_PRODUCTION for 5/8 sample real products with zero operator questions.

1121 tests passed (tests + central/tests + agent/tests), 0 failed. Version 1.3.7 (single GTI_FISCAL_VERSION shared by Hub and Agent per
central/app/version.py's own design - never independently versioned).
2026-09-27 18:26:20 -03:00

408 lines
18 KiB
Python

"""Agent enrichment: baseline/new-product detection, feature gating, typed write, second run = 0 writes, human override (incl. deleted value)."""
import sqlite3
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "agent" / "src"))
from gti_fiscal_agent import enrichment as E # noqa: E402
class Conn:
"""sqlite stand-in for the Firebird connection; close() is a no-op so the worker can 'open' it repeatedly."""
def __init__(self, db):
self.db = db
def cursor(self):
return self.db.cursor()
def commit(self):
self.db.commit()
def rollback(self):
self.db.rollback()
def close(self):
pass
class Hub:
def __init__(self, features, decisions=None, state="EXACT_GTIN_VERIFIED"):
self.features = features
self.decisions = decisions or {}
self.state = state
self.resolve_calls, self.reports, self.overridden = [], [], {}
def authorizations(self):
return {"features": dict(self.features)}
def resolve_product(self, payload):
self.resolve_calls.append(payload)
fields = {k: dict(v) for k, v in self.decisions.items()}
for f in list(fields):
if f in self.overridden.get(payload["product_code"], set()):
fields[f] = {"status": "HUMAN_OVERRIDDEN", "value": None}
elif payload["current"].get(f) and not E.is_placeholder(f, payload["current"].get(f)):
fields[f] = {"status": "ALREADY_SET", "value": None}
return {"state": self.state, "fields": fields, "dataset_versions": {"NCM": {"sha256": "abc"}}}
def pending_enrichment(self):
codes, self.flagged = list(getattr(self, "flagged", [])), []
return codes
def report_enrichment(self, payload):
self.reports.append(payload)
for item in payload["fields"]:
if item["outcome"] == "HUMAN_OVERRIDDEN":
self.overridden.setdefault(payload["product_code"], set()).add(item["field"])
return {"ok": True}
ENABLED = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}
DECISIONS = {"CLASSIFICACAO_FISCAL": {"status": "AUTO_FILL_SAFE", "value": "22021000", "rule": "NCM_KB_VERIFIED", "kb_status": "VERIFIED_KB"},
"CEST": {"status": "AUTO_FILL_SAFE", "value": "0300700", "rule": "CEST_OFFICIAL_UNIQUE_MAPPING"},
"IPI": {"status": "AUTO_FILL_SAFE", "value": "0.0", "rule": "IPI_TIPI_NUMERIC"}}
@pytest.fixture()
def env(tmp_path):
db = sqlite3.connect(":memory:", check_same_thread=False)
db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT, CODBARRA TEXT, "
+ ", ".join(f"{f} TEXT" for f in E.FISCAL_COMPLETION_FIELDS + E.CLASSIFICATION_CONTEXT_FIELDS) + ")")
db.executemany("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", [(f"H{i}", f"HIST {i}", "", "keep") for i in range(50)])
db.commit()
hub = Hub(dict(ENABLED), DECISIONS)
store = E.EnrichmentStore(tmp_path / "e.db")
now = [1000.0]
reproc = []
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=reproc.append, clock=lambda: now[0], sleep=lambda s: None)
return db, hub, store, worker, now, reproc
def add(db, code, name="REFRIGERANTE COLA LATA 350ML", gtin="7891000100103"):
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", (code, name, gtin, "keep"))
db.commit()
def row(db, code):
return db.execute("SELECT CLASSIFICACAO_FISCAL,CEST,IPI,OTHER FROM C000025 WHERE CODIGO=?", (code,)).fetchone()
def test_baseline_never_sends_historical_products(env):
db, hub, store, worker, now, _ = env
assert worker.step() == [] # baseline
now[0] += 5
assert worker.step() == [] and hub.resolve_calls == []
assert len(store.known()) == 50
def test_new_product_is_enriched_only_in_empty_fields_then_second_run_zero_writes(env):
db, hub, store, worker, now, reproc = env
worker.step()
add(db, "N1")
now[0] += 5
assert worker.step() == ["N1"]
assert row(db, "N1") == ("22021000", "0300700", 0.0, "keep")
assert reproc == [["N1"]]
assert row(db, "H1") == (None, None, None, "keep") # historical untouched
assert [r["outcome"] for r in hub.reports[0]["fields"]] == ["APPLIED"] * 3
now[0] += 5
before = db.total_changes
worker.step()
assert db.total_changes == before and len(hub.resolve_calls) == 1 # nothing re-processed, 0 writes
def test_current_report_includes_fiscal_completion_and_classification_columns(tmp_path):
"""`current` sent to the Hub used to carry only CLASSIFICACAO_FISCAL/CEST/IPI. field_contract.resolve_product (now wired into
/v1/products/resolve) also needs the current CFOP/CST/CSOSN/... values and the classification columns infer_role reads (TIPO/
PROD_COZINHA/PRODUTO_SERVICO/CODGRUPO) - otherwise it can never see what's already filled or auto-discover product_role."""
db = sqlite3.connect(":memory:", check_same_thread=False)
db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT, CODBARRA TEXT, "
+ ", ".join(f"{f} TEXT" for f in E.FISCAL_COMPLETION_FIELDS + E.CLASSIFICATION_CONTEXT_FIELDS) + ")")
db.commit()
hub = Hub(dict(ENABLED), DECISIONS)
store = E.EnrichmentStore(tmp_path / "e.db")
now = [1000.0]
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=lambda c: None, clock=lambda: now[0], sleep=lambda s: None)
worker.step() # baseline
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER,CSOSN,TIPO,PRODUTO_SERVICO) VALUES(?,?,?,?,?,?,?)", ("N9", "PRODUTO X", "7891000100103", "keep", "102", "VENDA", None))
db.commit()
now[0] += 5
worker.step()
sent = hub.resolve_calls[0]["current"]
assert sent["CSOSN"] == "102" and sent["TIPO"] == "VENDA"
for f in E.FISCAL_COMPLETION_FIELDS + E.CLASSIFICATION_CONTEXT_FIELDS:
assert f in sent, f
assert hub.resolve_calls[0]["ev"] == {"recipes": [], "composed": [], "purchases": {}, "stock_entry": {}, "stock_in": {}} # targeted evidence rides along (empty here: sqlite has no RDB$RELATION_FIELDS)
def test_existing_value_is_never_overwritten(env):
db, hub, store, worker, now, _ = env
worker.step()
add(db, "N2")
db.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N2'")
db.commit()
now[0] += 5
worker.step()
assert row(db, "N2")[0] == "19059090" and row(db, "N2")[1] == "0300700"
def test_dry_run_and_disabled_and_waiting_do_not_write_or_touch_firebird(env):
db, hub, store, worker, now, _ = env
worker.step()
for features in ({"features_configured": False}, {"features_configured": True, "enrichment_mode": "DISABLED"}):
hub.features = features
worker._features_at = -999
assert worker.step() == [] and hub.resolve_calls == []
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
worker._features_at = -999
now[0] += 5
worker.step() # re-baseline after the inactive period
add(db, "N3")
now[0] += 5
worker.step()
assert row(db, "N3") == (None, None, None, "keep") and hub.reports[0]["fields"][0]["outcome"] == "DRY_RUN"
def test_authorization_revoked_right_before_write_fails_closed(env):
db, hub, store, worker, now, _ = env
worker.step()
add(db, "N4")
calls = {"n": 0}
def flip():
calls["n"] += 1
return {"features": dict(ENABLED if calls["n"] == 1 else {"features_configured": True, "enrichment_mode": "DISABLED"})}
hub.authorizations = flip
now[0] += 5
worker._features_at = -999
worker.step()
assert row(db, "N4") == (None, None, None, "keep")
assert all(f["outcome"] == "ABORTED" and f["reason"] == "AUTHORIZATION_CHANGED" for f in hub.reports[0]["fields"])
def test_human_override_including_deleted_value_is_final(env):
db, hub, store, worker, now, _ = env
worker.step()
add(db, "N5")
now[0] += 5
worker.step()
db.execute("UPDATE C000025 SET CEST='', CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N5'") # a person deletes the CEST and changes the NCM
db.commit()
now[0] += 30
worker.step()
assert store.is_overridden("N5", "CEST") and store.is_overridden("N5", "CLASSIFICACAO_FISCAL") and not store.is_overridden("N5", "IPI")
# the description changes a moment later: the worker re-resolves, but overridden fields must NOT be written again
db.execute("UPDATE C000025 SET PRODUTO='REFRIGERANTE COLA LATA 350ML ZERO' WHERE CODIGO='N5'")
db.commit()
now[0] += 30
worker.step()
assert row(db, "N5")[:2] == ("19059090", "")
def test_write_field_allowlist_and_shape():
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE C000025(CODIGO TEXT, COD_ORIGEM_MERCADORIA TEXT, CLASSIFICACAO_FISCAL TEXT)")
with pytest.raises(ValueError):
E.write_field(Conn(db), "1", "COD_ORIGEM_MERCADORIA", "1")
with pytest.raises(ValueError):
E.write_field(Conn(db), "1", "CCLASSTRIB", "000001")
db.execute("INSERT INTO C000025(CODIGO) VALUES('1')")
db.commit()
assert E.write_field(Conn(db), "1", "CLASSIFICACAO_FISCAL", "123")["reason"] == "VALUE_SHAPE_INVALID"
def test_hub_offline_keeps_product_pending_and_retries(env):
db, hub, store, worker, now, _ = env
worker.step()
add(db, "N6")
def boom(payload):
raise ConnectionError("down")
good = hub.resolve_product
hub.resolve_product = boom
now[0] += 5
worker.step()
assert store.get("N6")["state"] == "RETRY" and row(db, "N6")[0] is None
hub.resolve_product = good
now[0] += 40
assert worker.step() == ["N6"] and row(db, "N6")[0] == "22021000"
def test_gtin_check_parity_with_the_hub_and_codbarra_is_used():
sys.path.insert(0, str(ROOT))
from central.app.enrichment.gtin import normalize_gtin
for value in ("7891000100103", "7891000100104", "", "0", "12345678", "40170725", "036000291452", "11111111", "78910001001030", "abc", "2000000000008"):
assert E.gtin_check_ok(value) == (normalize_gtin(value)["status"] in ("VALID_GTIN", "RESTRICTED_GTIN")), value
assert E.pick_gtin({"GTIN": "", "CODBARRA": "7891000100103"}) == "7891000100103"
assert E.pick_gtin({"GTIN": "7891000100103", "CODBARRA": "999"}) == "7891000100103"
assert E.pick_gtin({"GTIN": "", "CODBARRA": "123"}) == "123" and E.pick_gtin({}) == ""
def test_products_created_while_inactive_are_not_new_when_reactivated(env):
db, hub, store, worker, now, _ = env
worker.step() # baseline
hub.features = {"features_configured": True, "enrichment_mode": "DISABLED"}
worker._features_at = -999
now[0] += 5
worker.step()
add(db, "W1") # created while the function is off
now[0] += 5
worker.step()
hub.features = dict(ENABLED)
worker._features_at = -999
now[0] += 5
assert worker.step() == [] and hub.resolve_calls == [] # re-baselined: W1 is history now
add(db, "W2")
now[0] += 5
assert worker.step() == ["W2"]
# ---- TECHNICAL_PLACEHOLDER (the SGV refuses a product without NCM: operators type 00000000) ---------------------------------------------------
def add_with_ncm(db, code, ncm, name="COCA-COLA ORIGINAL LATA 350 ML"):
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,CLASSIFICACAO_FISCAL,OTHER) VALUES(?,?,?,?,?)", (code, name, "7891000100103", ncm, "keep"))
db.commit()
def test_placeholder_parity_with_the_hub():
sys.path.insert(0, str(ROOT))
from central.app.enrichment import placeholders as hub
assert {k: set(v) for k, v in E.PLACEHOLDERS.items()} == {k: set(v) for k, v in hub.PRODUCT_ENRICHMENT_PLACEHOLDERS.items()}
for value in ("00000000", "0000.00.00", " 00000000 ", "22021000", "99999999", "0", "SEM NCM", "", None):
assert E.is_placeholder("CLASSIFICACAO_FISCAL", value) == hub.is_placeholder("CLASSIFICACAO_FISCAL", value), value
assert not E.is_placeholder("CEST", "0000000") and not E.is_placeholder("IPI", 0)
def test_placeholder_ncm_is_replaced_dry_run_first_then_enabled_and_second_run_is_zero(env):
db, hub, store, worker, now, _ = env
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
worker._features_at = -999
worker.step() # baseline
add_with_ncm(db, "P1", "00000000")
now[0] += 5
worker.step()
assert row(db, "P1")[0] == "00000000" and hub.reports[-1]["fields"][0]["outcome"] == "DRY_RUN" # DRY_RUN: detected, zero writes
assert hub.resolve_calls[-1]["current"]["CLASSIFICACAO_FISCAL"] == "00000000"
hub.features = dict(ENABLED)
worker._features_at = -999
worker._auth_cache = (None, 0.0)
add_with_ncm(db, "P2", "00000000")
now[0] += 5
worker.step()
assert row(db, "P2") == ("22021000", "0300700", 0.0, "keep")
first = [f for f in hub.reports[-1]["fields"] if f["field"] == "CLASSIFICACAO_FISCAL"][0]
assert first["outcome"] == "APPLIED" and first["old"] == "00000000"
before = db.total_changes
now[0] += 5
worker.step()
now[0] += 60
worker.step()
assert db.total_changes == before # second run: 0 writes
def test_real_or_unlisted_ncm_is_never_replaced(env):
db, hub, store, worker, now, _ = env
worker.step()
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
add_with_ncm(db, code, ncm)
now[0] += 5
worker.step()
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
assert row(db, code)[0] == ncm, code # EXISTING_REAL_VALUE_PRESERVED, even if the KB says otherwise
def test_write_field_guards_the_placeholder_swap():
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)")
db.executemany("INSERT INTO C000025 VALUES(?,?)", [("a", "00000000"), ("b", "22021000"), ("c", None), ("d", "0000.00.00")])
db.commit()
assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "19059090")["reason"] == "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"
assert E.write_field(Conn(db), "c", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
assert E.write_field(Conn(db), "d", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
assert [r[0] for r in db.execute("SELECT CLASSIFICACAO_FISCAL FROM C000025 ORDER BY CODIGO")] == ["19059090", "22021000", "19059090", "19059090"]
# ---- restart / race / multiple products / stale guard / reevaluation / provenance --------------------------------------------------------------
def make_worker(db, hub, store, now):
return E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=lambda codes: None, clock=lambda: now[0], sleep=lambda s: None)
def test_product_created_while_agent_is_down_is_found_after_restart(env):
db, hub, store, worker, now, _ = env
worker.step()
add(db, "D1") # created while the Agent is stopped / restarting
worker2 = make_worker(db, hub, store, now) # new process, same persisted state
now[0] += 5
assert worker2.step() == ["D1"] and row(db, "D1")[0] == "22021000"
def test_many_products_between_polls_and_a_deleted_one(env):
db, hub, store, worker, now, _ = env
worker.step()
for i in range(5):
add(db, f"M{i}")
db.execute("DELETE FROM C000025 WHERE CODIGO='M2'")
db.commit()
add(db, "M9")
now[0] += 5
done = worker.step()
assert sorted(done) == ["M0", "M1", "M3", "M4", "M9"] and all(row(db, c)[0] == "22021000" for c in done)
def test_reevaluation_flagged_by_the_hub_retries_an_unresolved_product(env):
db, hub, store, worker, now, _ = env
worker.step()
hub.decisions = {} # first attempt: nothing known yet
add(db, "U1")
now[0] += 5
worker.step()
assert row(db, "U1")[0] is None and store.get("U1")["state"] == "DONE"
hub.decisions = dict(DECISIONS) # new evidence (KB decision) + 'Reavaliar'/'Reprocessar'
hub.flagged = ["U1"]
now[0] += 30
assert "U1" in worker.step() and row(db, "U1")[0] == "22021000"
now[0] += 30
before = db.total_changes
worker.step()
assert db.total_changes == before
def test_stale_write_is_refused_when_a_person_changed_the_value_after_the_decision(env):
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)")
db.execute("INSERT INTO C000025 VALUES('a','19059090')")
db.execute("INSERT INTO C000025 VALUES('b','00000000')")
db.commit()
assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "22021000", observed="00000000", guard_observed=True)["reason"] == "STALE_CONFLICT"
assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "22021000", observed="00000000", guard_observed=True)["outcome"] == "APPLIED"
def test_placeholder_after_a_human_override_is_not_filled_again(env):
db, hub, store, worker, now, _ = env
worker.step()
add_with_ncm(db, "HP1", "00000000") # born with the placeholder: eligible
now[0] += 5
worker.step()
assert row(db, "HP1")[0] == "22021000"
db.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL='00000000' WHERE CODIGO='HP1'") # a person puts the SAME value back: a human decision now
db.commit()
now[0] += 30
worker.step()
assert store.is_overridden("HP1", "CLASSIFICACAO_FISCAL")
hub.flagged = ["HP1"]
now[0] += 30
worker.step()
assert row(db, "HP1")[0] == "00000000" # provenance matters: same value, different meaning
def test_automatic_mode_needs_no_sync_button_or_restart(env):
db, hub, store, worker, now, _ = env
worker.step()
add(db, "A1")
now[0] += 3
assert worker.step() == ["A1"] and row(db, "A1")[0] == "22021000" # save -> next poll -> written