298 lines
13 KiB
Python
298 lines
13 KiB
Python
"""Agent enrichment: baseline/new-product detection, feature gating, typed write, second run = 0 writes, human override (incl. deleted value)."""
|
|
import sqlite3
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
|
from gti_fiscal_agent import enrichment as E # noqa: E402
|
|
|
|
|
|
class Conn:
|
|
"""sqlite stand-in for the Firebird connection; close() is a no-op so the worker can 'open' it repeatedly."""
|
|
|
|
def __init__(self, db):
|
|
self.db = db
|
|
|
|
def cursor(self):
|
|
return self.db.cursor()
|
|
|
|
def commit(self):
|
|
self.db.commit()
|
|
|
|
def rollback(self):
|
|
self.db.rollback()
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
|
|
class Hub:
|
|
def __init__(self, features, decisions=None, state="EXACT_GTIN_VERIFIED"):
|
|
self.features = features
|
|
self.decisions = decisions or {}
|
|
self.state = state
|
|
self.resolve_calls, self.reports, self.overridden = [], [], {}
|
|
|
|
def authorizations(self):
|
|
return {"features": dict(self.features)}
|
|
|
|
def resolve_product(self, payload):
|
|
self.resolve_calls.append(payload)
|
|
fields = {k: dict(v) for k, v in self.decisions.items()}
|
|
for f in list(fields):
|
|
if f in self.overridden.get(payload["product_code"], set()):
|
|
fields[f] = {"status": "HUMAN_OVERRIDDEN", "value": None}
|
|
elif payload["current"].get(f) and not E.is_placeholder(f, payload["current"].get(f)):
|
|
fields[f] = {"status": "ALREADY_SET", "value": None}
|
|
return {"state": self.state, "fields": fields, "dataset_versions": {"NCM": {"sha256": "abc"}}}
|
|
|
|
def report_enrichment(self, payload):
|
|
self.reports.append(payload)
|
|
for item in payload["fields"]:
|
|
if item["outcome"] == "HUMAN_OVERRIDDEN":
|
|
self.overridden.setdefault(payload["product_code"], set()).add(item["field"])
|
|
return {"ok": True}
|
|
|
|
|
|
ENABLED = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}
|
|
DECISIONS = {"CLASSIFICACAO_FISCAL": {"status": "AUTO_FILL_SAFE", "value": "22021000", "rule": "NCM_KB_VERIFIED", "kb_status": "VERIFIED_KB"},
|
|
"CEST": {"status": "AUTO_FILL_SAFE", "value": "0300700", "rule": "CEST_OFFICIAL_UNIQUE_MAPPING"},
|
|
"IPI": {"status": "AUTO_FILL_SAFE", "value": "0.0", "rule": "IPI_TIPI_NUMERIC"}}
|
|
|
|
|
|
@pytest.fixture()
|
|
def env(tmp_path):
|
|
db = sqlite3.connect(":memory:", check_same_thread=False)
|
|
db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT, CODBARRA TEXT)")
|
|
db.executemany("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", [(f"H{i}", f"HIST {i}", "", "keep") for i in range(50)])
|
|
db.commit()
|
|
hub = Hub(dict(ENABLED), DECISIONS)
|
|
store = E.EnrichmentStore(tmp_path / "e.db")
|
|
now = [1000.0]
|
|
reproc = []
|
|
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=reproc.append, clock=lambda: now[0], sleep=lambda s: None)
|
|
return db, hub, store, worker, now, reproc
|
|
|
|
|
|
def add(db, code, name="REFRIGERANTE COLA LATA 350ML", gtin="7891000100103"):
|
|
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", (code, name, gtin, "keep"))
|
|
db.commit()
|
|
|
|
|
|
def row(db, code):
|
|
return db.execute("SELECT CLASSIFICACAO_FISCAL,CEST,IPI,OTHER FROM C000025 WHERE CODIGO=?", (code,)).fetchone()
|
|
|
|
|
|
def test_baseline_never_sends_historical_products(env):
|
|
db, hub, store, worker, now, _ = env
|
|
assert worker.step() == [] # baseline
|
|
now[0] += 5
|
|
assert worker.step() == [] and hub.resolve_calls == []
|
|
assert len(store.known()) == 50
|
|
|
|
|
|
def test_new_product_is_enriched_only_in_empty_fields_then_second_run_zero_writes(env):
|
|
db, hub, store, worker, now, reproc = env
|
|
worker.step()
|
|
add(db, "N1")
|
|
now[0] += 5
|
|
assert worker.step() == ["N1"]
|
|
assert row(db, "N1") == ("22021000", "0300700", 0.0, "keep")
|
|
assert reproc == [["N1"]]
|
|
assert row(db, "H1") == (None, None, None, "keep") # historical untouched
|
|
assert [r["outcome"] for r in hub.reports[0]["fields"]] == ["APPLIED"] * 3
|
|
now[0] += 5
|
|
before = db.total_changes
|
|
worker.step()
|
|
assert db.total_changes == before and len(hub.resolve_calls) == 1 # nothing re-processed, 0 writes
|
|
|
|
|
|
def test_existing_value_is_never_overwritten(env):
|
|
db, hub, store, worker, now, _ = env
|
|
worker.step()
|
|
add(db, "N2")
|
|
db.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N2'")
|
|
db.commit()
|
|
now[0] += 5
|
|
worker.step()
|
|
assert row(db, "N2")[0] == "19059090" and row(db, "N2")[1] == "0300700"
|
|
|
|
|
|
def test_dry_run_and_disabled_and_waiting_do_not_write_or_touch_firebird(env):
|
|
db, hub, store, worker, now, _ = env
|
|
worker.step()
|
|
for features in ({"features_configured": False}, {"features_configured": True, "enrichment_mode": "DISABLED"}):
|
|
hub.features = features
|
|
worker._features_at = -999
|
|
assert worker.step() == [] and hub.resolve_calls == []
|
|
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
|
|
worker._features_at = -999
|
|
now[0] += 5
|
|
worker.step() # re-baseline after the inactive period
|
|
add(db, "N3")
|
|
now[0] += 5
|
|
worker.step()
|
|
assert row(db, "N3") == (None, None, None, "keep") and hub.reports[0]["fields"][0]["outcome"] == "DRY_RUN"
|
|
|
|
|
|
def test_authorization_revoked_right_before_write_fails_closed(env):
|
|
db, hub, store, worker, now, _ = env
|
|
worker.step()
|
|
add(db, "N4")
|
|
calls = {"n": 0}
|
|
|
|
def flip():
|
|
calls["n"] += 1
|
|
return {"features": dict(ENABLED if calls["n"] == 1 else {"features_configured": True, "enrichment_mode": "DISABLED"})}
|
|
hub.authorizations = flip
|
|
now[0] += 5
|
|
worker._features_at = -999
|
|
worker.step()
|
|
assert row(db, "N4") == (None, None, None, "keep")
|
|
assert all(f["outcome"] == "ABORTED" and f["reason"] == "AUTHORIZATION_CHANGED" for f in hub.reports[0]["fields"])
|
|
|
|
|
|
def test_human_override_including_deleted_value_is_final(env):
|
|
db, hub, store, worker, now, _ = env
|
|
worker.step()
|
|
add(db, "N5")
|
|
now[0] += 5
|
|
worker.step()
|
|
db.execute("UPDATE C000025 SET CEST='', CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N5'") # a person deletes the CEST and changes the NCM
|
|
db.commit()
|
|
now[0] += 30
|
|
worker.step()
|
|
assert store.is_overridden("N5", "CEST") and store.is_overridden("N5", "CLASSIFICACAO_FISCAL") and not store.is_overridden("N5", "IPI")
|
|
# the description changes a moment later: the worker re-resolves, but overridden fields must NOT be written again
|
|
db.execute("UPDATE C000025 SET PRODUTO='REFRIGERANTE COLA LATA 350ML ZERO' WHERE CODIGO='N5'")
|
|
db.commit()
|
|
now[0] += 30
|
|
worker.step()
|
|
assert row(db, "N5")[:2] == ("19059090", "")
|
|
|
|
|
|
def test_write_field_allowlist_and_shape():
|
|
db = sqlite3.connect(":memory:")
|
|
db.execute("CREATE TABLE C000025(CODIGO TEXT, COD_ORIGEM_MERCADORIA TEXT, CLASSIFICACAO_FISCAL TEXT)")
|
|
with pytest.raises(ValueError):
|
|
E.write_field(Conn(db), "1", "COD_ORIGEM_MERCADORIA", "1")
|
|
with pytest.raises(ValueError):
|
|
E.write_field(Conn(db), "1", "CCLASSTRIB", "000001")
|
|
db.execute("INSERT INTO C000025(CODIGO) VALUES('1')")
|
|
db.commit()
|
|
assert E.write_field(Conn(db), "1", "CLASSIFICACAO_FISCAL", "123")["reason"] == "VALUE_SHAPE_INVALID"
|
|
|
|
|
|
def test_hub_offline_keeps_product_pending_and_retries(env):
|
|
db, hub, store, worker, now, _ = env
|
|
worker.step()
|
|
add(db, "N6")
|
|
|
|
def boom(payload):
|
|
raise ConnectionError("down")
|
|
good = hub.resolve_product
|
|
hub.resolve_product = boom
|
|
now[0] += 5
|
|
worker.step()
|
|
assert store.get("N6")["state"] == "RETRY" and row(db, "N6")[0] is None
|
|
hub.resolve_product = good
|
|
now[0] += 40
|
|
assert worker.step() == ["N6"] and row(db, "N6")[0] == "22021000"
|
|
|
|
|
|
def test_gtin_check_parity_with_the_hub_and_codbarra_is_used():
|
|
sys.path.insert(0, str(ROOT))
|
|
from central.app.enrichment.gtin import normalize_gtin
|
|
for value in ("7891000100103", "7891000100104", "", "0", "12345678", "40170725", "036000291452", "11111111", "78910001001030", "abc", "2000000000008"):
|
|
assert E.gtin_check_ok(value) == (normalize_gtin(value)["status"] in ("VALID_GTIN", "RESTRICTED_GTIN")), value
|
|
assert E.pick_gtin({"GTIN": "", "CODBARRA": "7891000100103"}) == "7891000100103"
|
|
assert E.pick_gtin({"GTIN": "7891000100103", "CODBARRA": "999"}) == "7891000100103"
|
|
assert E.pick_gtin({"GTIN": "", "CODBARRA": "123"}) == "123" and E.pick_gtin({}) == ""
|
|
|
|
|
|
def test_products_created_while_inactive_are_not_new_when_reactivated(env):
|
|
db, hub, store, worker, now, _ = env
|
|
worker.step() # baseline
|
|
hub.features = {"features_configured": True, "enrichment_mode": "DISABLED"}
|
|
worker._features_at = -999
|
|
now[0] += 5
|
|
worker.step()
|
|
add(db, "W1") # created while the function is off
|
|
now[0] += 5
|
|
worker.step()
|
|
hub.features = dict(ENABLED)
|
|
worker._features_at = -999
|
|
now[0] += 5
|
|
assert worker.step() == [] and hub.resolve_calls == [] # re-baselined: W1 is history now
|
|
add(db, "W2")
|
|
now[0] += 5
|
|
assert worker.step() == ["W2"]
|
|
|
|
|
|
# ---- TECHNICAL_PLACEHOLDER (the SGV refuses a product without NCM: operators type 00000000) ---------------------------------------------------
|
|
def add_with_ncm(db, code, ncm, name="COCA-COLA ORIGINAL LATA 350 ML"):
|
|
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,CLASSIFICACAO_FISCAL,OTHER) VALUES(?,?,?,?,?)", (code, name, "7891000100103", ncm, "keep"))
|
|
db.commit()
|
|
|
|
|
|
def test_placeholder_parity_with_the_hub():
|
|
sys.path.insert(0, str(ROOT))
|
|
from central.app.enrichment import placeholders as hub
|
|
assert {k: set(v) for k, v in E.PLACEHOLDERS.items()} == {k: set(v) for k, v in hub.PRODUCT_ENRICHMENT_PLACEHOLDERS.items()}
|
|
for value in ("00000000", "0000.00.00", " 00000000 ", "22021000", "99999999", "0", "SEM NCM", "", None):
|
|
assert E.is_placeholder("CLASSIFICACAO_FISCAL", value) == hub.is_placeholder("CLASSIFICACAO_FISCAL", value), value
|
|
assert not E.is_placeholder("CEST", "0000000") and not E.is_placeholder("IPI", 0)
|
|
|
|
|
|
def test_placeholder_ncm_is_replaced_dry_run_first_then_enabled_and_second_run_is_zero(env):
|
|
db, hub, store, worker, now, _ = env
|
|
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
|
|
worker._features_at = -999
|
|
worker.step() # baseline
|
|
add_with_ncm(db, "P1", "00000000")
|
|
now[0] += 5
|
|
worker.step()
|
|
assert row(db, "P1")[0] == "00000000" and hub.reports[-1]["fields"][0]["outcome"] == "DRY_RUN" # DRY_RUN: detected, zero writes
|
|
assert hub.resolve_calls[-1]["current"]["CLASSIFICACAO_FISCAL"] == "00000000"
|
|
hub.features = dict(ENABLED)
|
|
worker._features_at = -999
|
|
worker._auth_cache = (None, 0.0)
|
|
add_with_ncm(db, "P2", "00000000")
|
|
now[0] += 5
|
|
worker.step()
|
|
assert row(db, "P2") == ("22021000", "0300700", 0.0, "keep")
|
|
first = [f for f in hub.reports[-1]["fields"] if f["field"] == "CLASSIFICACAO_FISCAL"][0]
|
|
assert first["outcome"] == "APPLIED" and first["old"] == "00000000"
|
|
before = db.total_changes
|
|
now[0] += 5
|
|
worker.step()
|
|
now[0] += 60
|
|
worker.step()
|
|
assert db.total_changes == before # second run: 0 writes
|
|
|
|
|
|
def test_real_or_unlisted_ncm_is_never_replaced(env):
|
|
db, hub, store, worker, now, _ = env
|
|
worker.step()
|
|
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
|
|
add_with_ncm(db, code, ncm)
|
|
now[0] += 5
|
|
worker.step()
|
|
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
|
|
assert row(db, code)[0] == ncm, code # EXISTING_REAL_VALUE_PRESERVED, even if the KB says otherwise
|
|
|
|
|
|
def test_write_field_guards_the_placeholder_swap():
|
|
db = sqlite3.connect(":memory:")
|
|
db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)")
|
|
db.executemany("INSERT INTO C000025 VALUES(?,?)", [("a", "00000000"), ("b", "22021000"), ("c", None), ("d", "0000.00.00")])
|
|
db.commit()
|
|
assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
|
assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "19059090")["reason"] == "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"
|
|
assert E.write_field(Conn(db), "c", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
|
assert E.write_field(Conn(db), "d", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
|
assert [r[0] for r in db.execute("SELECT CLASSIFICACAO_FISCAL FROM C000025 ORDER BY CODIGO")] == ["19059090", "22021000", "19059090", "19059090"]
|