gustavosantos/GTI_GERENCIADOR_FISCAL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
efc8b54bc711ac267e815cec07d362b6ba54d0b8
GTI_GERENCIADOR_FISCAL/agent/tests/test_enrichment.py
T
gtiandClaude Sonnet 5 1abb7e516e fix: product_role never infers OWN_PRODUCTION from a single COMPOSICAO occurrence 
Missão 4 / Bloco A.1 (prioridade máxima): central.app.field_contract.infer_role
treated ANY C000032.TIPO=COMPOSICAO row for a product as proof of own
production, with no ratio check - a beer sold 19 times and used once as a
recipe component (Agent 2, real data) was classified OWN_PRODUCTION from
that single occurrence, proposing to change its CFOP and IPPT.

New deterministic threshold in field_contract.py: COMPOSICAO must be a
majority (>=70%) of that product's own C000032 movement history, with at
least 5 total events, to count as own-production evidence. Below 30% the
evidence is discarded entirely (falls through to other role evidence,
e.g. TIPO=VENDA -> RESALE). Between 30% and 70% (with enough events to be
meaningful) is genuinely conflicting evidence -> UNKNOWN, never guessed
either way. RECEITA_INGREDIENTES/PROD_COZINHA-based evidence (a direct
recipe-table linkage, not a statistical ratio) is untouched.

agent/src/gti_fiscal_agent/firebird_reader.py now counts COMPOSICAO vs other
C000032 rows per product (read_product_evidence: single indexed query;
read_evidence: one grouped query for the whole catalog) into a new
ev["composed_ratio"] key, alongside the existing "composed" set (kept for
other, non-role consumers like coverage.py). central/app/enrichment/api.py's
sanitize_product_evidence and fiscal_completion.py's empty-ev fallback both
carry the new key through.

New tests (central/tests/test_field_contract.py) cover the 5 real cases named
in the mission: the beer (1 vs 19, 5% - discarded), a second beer (3 vs 4,
42.9% - MIXED/UNKNOWN), the liqueur (34 vs 22, 60.7% - still MIXED/UNKNOWN,
below the 70% bar), the juice with PROD_COZINHA=1+RECEITA_INGREDIENTES
(unaffected, stays OWN_PRODUCTION), and the tilapia/corvina fish (no
composed evidence at all, stays RESALE) - plus a strong 90% majority case
and an insufficient-sample (<5 events) case. A new end-to-end test in
agent/tests/test_cfop_ippt_auto_apply.py (committed separately, see the
feature/cfop-ippt-write branch) proves the weak-evidence beer never reaches
select_candidates as a write candidate, independent of any write flag.

Re-running the dry-run over Agent 2's real database (GTI.FDB, 527 products)
after this fix: AUTO_FILL_SAFE CFOP/IPPT cells drop from 12 to 3 (the two
fish CFOP fills + the juice's IPPT, all with strong, non-statistical
evidence) - docs/runs/GTI_dryrun_universe_summary_after_A1.json. This
corrects the Missão 3 report's 12/9(downgraded)/4(sustained) manual
judgment call with a deterministic, tested engine rule.

No schema/migration change. Zero Firebird writes (read-only analysis only).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-28 10:53:10 -03:00

408 lines
18 KiB
Python
Raw Blame History

"""Agent enrichment: baseline/new-product detection, feature gating, typed write, second run = 0 writes, human override (incl. deleted value)."""
import sqlite3
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "agent" / "src"))
from gti_fiscal_agent import enrichment as E # noqa: E402
class Conn:
"""sqlite stand-in for the Firebird connection; close() is a no-op so the worker can 'open' it repeatedly."""
def __init__(self, db):
self.db = db
def cursor(self):
return self.db.cursor()
def commit(self):
self.db.commit()
def rollback(self):
self.db.rollback()
def close(self):
pass
class Hub:
def __init__(self, features, decisions=None, state="EXACT_GTIN_VERIFIED"):
self.features = features
self.decisions = decisions or {}
self.state = state
self.resolve_calls, self.reports, self.overridden = [], [], {}
def authorizations(self):
return {"features": dict(self.features)}
def resolve_product(self, payload):
self.resolve_calls.append(payload)
fields = {k: dict(v) for k, v in self.decisions.items()}
for f in list(fields):
if f in self.overridden.get(payload["product_code"], set()):
fields[f] = {"status": "HUMAN_OVERRIDDEN", "value": None}
elif payload["current"].get(f) and not E.is_placeholder(f, payload["current"].get(f)):
fields[f] = {"status": "ALREADY_SET", "value": None}
return {"state": self.state, "fields": fields, "dataset_versions": {"NCM": {"sha256": "abc"}}}
def pending_enrichment(self):
codes, self.flagged = list(getattr(self, "flagged", [])), []
return codes
def report_enrichment(self, payload):
self.reports.append(payload)
for item in payload["fields"]:
if item["outcome"] == "HUMAN_OVERRIDDEN":
self.overridden.setdefault(payload["product_code"], set()).add(item["field"])
return {"ok": True}
ENABLED = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}
DECISIONS = {"CLASSIFICACAO_FISCAL": {"status": "AUTO_FILL_SAFE", "value": "22021000", "rule": "NCM_KB_VERIFIED", "kb_status": "VERIFIED_KB"},
"CEST": {"status": "AUTO_FILL_SAFE", "value": "0300700", "rule": "CEST_OFFICIAL_UNIQUE_MAPPING"},
"IPI": {"status": "AUTO_FILL_SAFE", "value": "0.0", "rule": "IPI_TIPI_NUMERIC"}}
@pytest.fixture()
def env(tmp_path):
db = sqlite3.connect(":memory:", check_same_thread=False)
db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT, CODBARRA TEXT, "
+ ", ".join(f"{f} TEXT" for f in E.FISCAL_COMPLETION_FIELDS + E.CLASSIFICATION_CONTEXT_FIELDS) + ")")
db.executemany("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", [(f"H{i}", f"HIST {i}", "", "keep") for i in range(50)])
db.commit()
hub = Hub(dict(ENABLED), DECISIONS)
store = E.EnrichmentStore(tmp_path / "e.db")
now = [1000.0]
reproc = []
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=reproc.append, clock=lambda: now[0], sleep=lambda s: None)
return db, hub, store, worker, now, reproc
def add(db, code, name="REFRIGERANTE COLA LATA 350ML", gtin="7891000100103"):
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", (code, name, gtin, "keep"))
db.commit()
def row(db, code):
return db.execute("SELECT CLASSIFICACAO_FISCAL,CEST,IPI,OTHER FROM C000025 WHERE CODIGO=?", (code,)).fetchone()
def test_baseline_never_sends_historical_products(env):
db, hub, store, worker, now, _ = env
assert worker.step() == [] # baseline
now[0] += 5
assert worker.step() == [] and hub.resolve_calls == []
assert len(store.known()) == 50
def test_new_product_is_enriched_only_in_empty_fields_then_second_run_zero_writes(env):
db, hub, store, worker, now, reproc = env
worker.step()
add(db, "N1")
now[0] += 5
assert worker.step() == ["N1"]
assert row(db, "N1") == ("22021000", "0300700", 0.0, "keep")
assert reproc == [["N1"]]
assert row(db, "H1") == (None, None, None, "keep") # historical untouched
assert [r["outcome"] for r in hub.reports[0]["fields"]] == ["APPLIED"] * 3
now[0] += 5
before = db.total_changes
worker.step()
assert db.total_changes == before and len(hub.resolve_calls) == 1 # nothing re-processed, 0 writes
def test_current_report_includes_fiscal_completion_and_classification_columns(tmp_path):
"""`current` sent to the Hub used to carry only CLASSIFICACAO_FISCAL/CEST/IPI. field_contract.resolve_product (now wired into
/v1/products/resolve) also needs the current CFOP/CST/CSOSN/... values and the classification columns infer_role reads (TIPO/
PROD_COZINHA/PRODUTO_SERVICO/CODGRUPO) - otherwise it can never see what's already filled or auto-discover product_role."""
db = sqlite3.connect(":memory:", check_same_thread=False)
db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT, CODBARRA TEXT, "
+ ", ".join(f"{f} TEXT" for f in E.FISCAL_COMPLETION_FIELDS + E.CLASSIFICATION_CONTEXT_FIELDS) + ")")
db.commit()
hub = Hub(dict(ENABLED), DECISIONS)
store = E.EnrichmentStore(tmp_path / "e.db")
now = [1000.0]
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=lambda c: None, clock=lambda: now[0], sleep=lambda s: None)
worker.step() # baseline
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER,CSOSN,TIPO,PRODUTO_SERVICO) VALUES(?,?,?,?,?,?,?)", ("N9", "PRODUTO X", "7891000100103", "keep", "102", "VENDA", None))
db.commit()
now[0] += 5
worker.step()
sent = hub.resolve_calls[0]["current"]
assert sent["CSOSN"] == "102" and sent["TIPO"] == "VENDA"
for f in E.FISCAL_COMPLETION_FIELDS + E.CLASSIFICATION_CONTEXT_FIELDS:
assert f in sent, f
assert hub.resolve_calls[0]["ev"] == {"recipes": [], "composed": [], "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}} # targeted evidence rides along (empty here: sqlite has no RDB$RELATION_FIELDS)
def test_existing_value_is_never_overwritten(env):
db, hub, store, worker, now, _ = env
worker.step()
add(db, "N2")
db.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N2'")
db.commit()
now[0] += 5
worker.step()
assert row(db, "N2")[0] == "19059090" and row(db, "N2")[1] == "0300700"
def test_dry_run_and_disabled_and_waiting_do_not_write_or_touch_firebird(env):
db, hub, store, worker, now, _ = env
worker.step()
for features in ({"features_configured": False}, {"features_configured": True, "enrichment_mode": "DISABLED"}):
hub.features = features
worker._features_at = -999
assert worker.step() == [] and hub.resolve_calls == []
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
worker._features_at = -999
now[0] += 5
worker.step() # re-baseline after the inactive period
add(db, "N3")
now[0] += 5
worker.step()
assert row(db, "N3") == (None, None, None, "keep") and hub.reports[0]["fields"][0]["outcome"] == "DRY_RUN"
def test_authorization_revoked_right_before_write_fails_closed(env):
db, hub, store, worker, now, _ = env
worker.step()
add(db, "N4")
calls = {"n": 0}
def flip():
calls["n"] += 1
return {"features": dict(ENABLED if calls["n"] == 1 else {"features_configured": True, "enrichment_mode": "DISABLED"})}
hub.authorizations = flip
now[0] += 5
worker._features_at = -999
worker.step()
assert row(db, "N4") == (None, None, None, "keep")
assert all(f["outcome"] == "ABORTED" and f["reason"] == "AUTHORIZATION_CHANGED" for f in hub.reports[0]["fields"])
def test_human_override_including_deleted_value_is_final(env):
db, hub, store, worker, now, _ = env
worker.step()
add(db, "N5")
now[0] += 5
worker.step()
db.execute("UPDATE C000025 SET CEST='', CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N5'") # a person deletes the CEST and changes the NCM
db.commit()
now[0] += 30
worker.step()
assert store.is_overridden("N5", "CEST") and store.is_overridden("N5", "CLASSIFICACAO_FISCAL") and not store.is_overridden("N5", "IPI")
# the description changes a moment later: the worker re-resolves, but overridden fields must NOT be written again
db.execute("UPDATE C000025 SET PRODUTO='REFRIGERANTE COLA LATA 350ML ZERO' WHERE CODIGO='N5'")
db.commit()
now[0] += 30
worker.step()
assert row(db, "N5")[:2] == ("19059090", "")
def test_write_field_allowlist_and_shape():
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE C000025(CODIGO TEXT, COD_ORIGEM_MERCADORIA TEXT, CLASSIFICACAO_FISCAL TEXT)")
with pytest.raises(ValueError):
E.write_field(Conn(db), "1", "COD_ORIGEM_MERCADORIA", "1")
with pytest.raises(ValueError):
E.write_field(Conn(db), "1", "CCLASSTRIB", "000001")
db.execute("INSERT INTO C000025(CODIGO) VALUES('1')")
db.commit()
assert E.write_field(Conn(db), "1", "CLASSIFICACAO_FISCAL", "123")["reason"] == "VALUE_SHAPE_INVALID"
def test_hub_offline_keeps_product_pending_and_retries(env):
db, hub, store, worker, now, _ = env
worker.step()
add(db, "N6")
def boom(payload):
raise ConnectionError("down")
good = hub.resolve_product
hub.resolve_product = boom
now[0] += 5
worker.step()
assert store.get("N6")["state"] == "RETRY" and row(db, "N6")[0] is None
hub.resolve_product = good
now[0] += 40
assert worker.step() == ["N6"] and row(db, "N6")[0] == "22021000"
def test_gtin_check_parity_with_the_hub_and_codbarra_is_used():
sys.path.insert(0, str(ROOT))
from central.app.enrichment.gtin import normalize_gtin
for value in ("7891000100103", "7891000100104", "", "0", "12345678", "40170725", "036000291452", "11111111", "78910001001030", "abc", "2000000000008"):
assert E.gtin_check_ok(value) == (normalize_gtin(value)["status"] in ("VALID_GTIN", "RESTRICTED_GTIN")), value
assert E.pick_gtin({"GTIN": "", "CODBARRA": "7891000100103"}) == "7891000100103"
assert E.pick_gtin({"GTIN": "7891000100103", "CODBARRA": "999"}) == "7891000100103"
assert E.pick_gtin({"GTIN": "", "CODBARRA": "123"}) == "123" and E.pick_gtin({}) == ""
def test_products_created_while_inactive_are_not_new_when_reactivated(env):
db, hub, store, worker, now, _ = env
worker.step() # baseline
hub.features = {"features_configured": True, "enrichment_mode": "DISABLED"}
worker._features_at = -999
now[0] += 5
worker.step()
add(db, "W1") # created while the function is off
now[0] += 5
worker.step()
hub.features = dict(ENABLED)
worker._features_at = -999
now[0] += 5
assert worker.step() == [] and hub.resolve_calls == [] # re-baselined: W1 is history now
add(db, "W2")
now[0] += 5
assert worker.step() == ["W2"]
# ---- TECHNICAL_PLACEHOLDER (the SGV refuses a product without NCM: operators type 00000000) ---------------------------------------------------
def add_with_ncm(db, code, ncm, name="COCA-COLA ORIGINAL LATA 350 ML"):
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,CLASSIFICACAO_FISCAL,OTHER) VALUES(?,?,?,?,?)", (code, name, "7891000100103", ncm, "keep"))
db.commit()
def test_placeholder_parity_with_the_hub():
sys.path.insert(0, str(ROOT))
from central.app.enrichment import placeholders as hub
assert {k: set(v) for k, v in E.PLACEHOLDERS.items()} == {k: set(v) for k, v in hub.PRODUCT_ENRICHMENT_PLACEHOLDERS.items()}
for value in ("00000000", "0000.00.00", " 00000000 ", "22021000", "99999999", "0", "SEM NCM", "", None):
assert E.is_placeholder("CLASSIFICACAO_FISCAL", value) == hub.is_placeholder("CLASSIFICACAO_FISCAL", value), value
assert not E.is_placeholder("CEST", "0000000") and not E.is_placeholder("IPI", 0)
def test_placeholder_ncm_is_replaced_dry_run_first_then_enabled_and_second_run_is_zero(env):
db, hub, store, worker, now, _ = env
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
worker._features_at = -999
worker.step() # baseline
add_with_ncm(db, "P1", "00000000")
now[0] += 5
worker.step()
assert row(db, "P1")[0] == "00000000" and hub.reports[-1]["fields"][0]["outcome"] == "DRY_RUN" # DRY_RUN: detected, zero writes
assert hub.resolve_calls[-1]["current"]["CLASSIFICACAO_FISCAL"] == "00000000"
hub.features = dict(ENABLED)
worker._features_at = -999
worker._auth_cache = (None, 0.0)
add_with_ncm(db, "P2", "00000000")
now[0] += 5
worker.step()
assert row(db, "P2") == ("22021000", "0300700", 0.0, "keep")
first = [f for f in hub.reports[-1]["fields"] if f["field"] == "CLASSIFICACAO_FISCAL"][0]
assert first["outcome"] == "APPLIED" and first["old"] == "00000000"
before = db.total_changes
now[0] += 5
worker.step()
now[0] += 60
worker.step()
assert db.total_changes == before # second run: 0 writes
def test_real_or_unlisted_ncm_is_never_replaced(env):
db, hub, store, worker, now, _ = env
worker.step()
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
add_with_ncm(db, code, ncm)
now[0] += 5
worker.step()
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
assert row(db, code)[0] == ncm, code # EXISTING_REAL_VALUE_PRESERVED, even if the KB says otherwise
def test_write_field_guards_the_placeholder_swap():
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)")
db.executemany("INSERT INTO C000025 VALUES(?,?)", [("a", "00000000"), ("b", "22021000"), ("c", None), ("d", "0000.00.00")])
db.commit()
assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "19059090")["reason"] == "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"
assert E.write_field(Conn(db), "c", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
assert E.write_field(Conn(db), "d", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
assert [r[0] for r in db.execute("SELECT CLASSIFICACAO_FISCAL FROM C000025 ORDER BY CODIGO")] == ["19059090", "22021000", "19059090", "19059090"]
# ---- restart / race / multiple products / stale guard / reevaluation / provenance --------------------------------------------------------------
def make_worker(db, hub, store, now):
return E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=lambda codes: None, clock=lambda: now[0], sleep=lambda s: None)
def test_product_created_while_agent_is_down_is_found_after_restart(env):
db, hub, store, worker, now, _ = env
worker.step()
add(db, "D1") # created while the Agent is stopped / restarting
worker2 = make_worker(db, hub, store, now) # new process, same persisted state
now[0] += 5
assert worker2.step() == ["D1"] and row(db, "D1")[0] == "22021000"
def test_many_products_between_polls_and_a_deleted_one(env):
db, hub, store, worker, now, _ = env
worker.step()
for i in range(5):
add(db, f"M{i}")
db.execute("DELETE FROM C000025 WHERE CODIGO='M2'")
db.commit()
add(db, "M9")
now[0] += 5
done = worker.step()
assert sorted(done) == ["M0", "M1", "M3", "M4", "M9"] and all(row(db, c)[0] == "22021000" for c in done)
def test_reevaluation_flagged_by_the_hub_retries_an_unresolved_product(env):
db, hub, store, worker, now, _ = env
worker.step()
hub.decisions = {} # first attempt: nothing known yet
add(db, "U1")
now[0] += 5
worker.step()
assert row(db, "U1")[0] is None and store.get("U1")["state"] == "DONE"
hub.decisions = dict(DECISIONS) # new evidence (KB decision) + 'Reavaliar'/'Reprocessar'
hub.flagged = ["U1"]
now[0] += 30
assert "U1" in worker.step() and row(db, "U1")[0] == "22021000"
now[0] += 30
before = db.total_changes
worker.step()
assert db.total_changes == before
def test_stale_write_is_refused_when_a_person_changed_the_value_after_the_decision(env):
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)")
db.execute("INSERT INTO C000025 VALUES('a','19059090')")
db.execute("INSERT INTO C000025 VALUES('b','00000000')")
db.commit()
assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "22021000", observed="00000000", guard_observed=True)["reason"] == "STALE_CONFLICT"
assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "22021000", observed="00000000", guard_observed=True)["outcome"] == "APPLIED"
def test_placeholder_after_a_human_override_is_not_filled_again(env):
db, hub, store, worker, now, _ = env
worker.step()
add_with_ncm(db, "HP1", "00000000") # born with the placeholder: eligible
now[0] += 5
worker.step()
assert row(db, "HP1")[0] == "22021000"
db.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL='00000000' WHERE CODIGO='HP1'") # a person puts the SAME value back: a human decision now
db.commit()
now[0] += 30
worker.step()
assert store.is_overridden("HP1", "CLASSIFICACAO_FISCAL")
hub.flagged = ["HP1"]
now[0] += 30
worker.step()
assert row(db, "HP1")[0] == "00000000" # provenance matters: same value, different meaning
def test_automatic_mode_needs_no_sync_button_or_restart(env):
db, hub, store, worker, now, _ = env
worker.step()
add(db, "A1")
now[0] += 3
assert worker.step() == ["A1"] and row(db, "A1")[0] == "22021000" # save -> next poll -> written
Reference in New Issue View Git Blame Copy Permalink