feat: visible per-product enrichment status, flagged re-evaluation (panel/Reprocessar/Sincronizar/KB decision), operator NCM decision, deterministic name-only match, stale-write guard; 1.3.5
This commit is contained in:
gti
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "gti-fiscal-agent"
|
||||
version = "1.3.4"
|
||||
version = "1.3.5"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = ["httpx>=0.27,<1"]
|
||||
|
||||
|
||||
@@ -107,7 +107,7 @@ class Config:
|
||||
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
|
||||
# RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib.
|
||||
self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB)))
|
||||
self.agent_version = env.get("AGENT_VERSION", "1.3.4")
|
||||
self.agent_version = env.get("AGENT_VERSION", "1.3.5")
|
||||
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
# Pilot write authorization: both must be explicitly set by the installer for
|
||||
@@ -349,7 +349,7 @@ def execute_job(job, config, executor, state, env):
|
||||
"fiscal_context": fiscal_context,
|
||||
"rtc_auto_apply": rtc_apply,
|
||||
"fiscal_mode": getattr(config, "fiscal_mode", None),
|
||||
"release_version": env.get("AGENT_VERSION") or "1.3.4",
|
||||
"release_version": env.get("AGENT_VERSION") or "1.3.5",
|
||||
"products_total": len(products),
|
||||
"products_changed": len(to_process),
|
||||
"products_processed": summary["processed"],
|
||||
@@ -520,7 +520,7 @@ def run(config, env, *, max_iterations=None, stopper=None):
|
||||
def enforce_startup_preflight(env):
|
||||
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
|
||||
from gti_fiscal_agent.preflight import run_preflight
|
||||
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.4")
|
||||
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.5")
|
||||
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
|
||||
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
|
||||
if report["PREFLIGHT"] != "PASS":
|
||||
|
||||
@@ -45,6 +45,10 @@ class HubClient:
|
||||
"""Asks the Hub to resolve a NEW product (the Agent never contacts external providers). Bounded: the Hub answers within its own time budget."""
|
||||
response = httpx.post(f"{self.url}/v1/products/resolve", json=payload, headers=self._headers(), timeout=20)
|
||||
response.raise_for_status(); return response.json()
|
||||
def pending_enrichment(self):
|
||||
"""Codes the Hub flagged for a new evaluation (panel button, Reprocessar/Sincronizar, KB decision)."""
|
||||
response = httpx.get(f"{self.url}/v1/products/enrichment/pending", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json().get("codes") or []
|
||||
def report_enrichment(self, payload):
|
||||
response = _resilient(lambda: httpx.post(f"{self.url}/v1/products/enrichment/applied", json=payload, headers=self._headers(), timeout=30))
|
||||
response.raise_for_status(); return response.json()
|
||||
|
||||
@@ -29,6 +29,7 @@ READ_FIELDS = ("CODIGO", "PRODUTO", "GTIN", "CODBARRA") + ALLOWED_FIELDS #
|
||||
POLL_SECONDS = 2.0
|
||||
SAFETY_DIFF_SECONDS = 60.0
|
||||
OVERRIDE_CHECK_SECONDS = 20.0
|
||||
PENDING_POLL_SECONDS = 10.0 # cheap GET: products the Hub flagged for a new evaluation
|
||||
WATCH_WINDOW_SECONDS = 120.0 # a fresh product is re-read for a while: the operator may add the EAN seconds after the first save
|
||||
RETRY_BACKOFF = (30, 60, 120, 300, 600, 1800)
|
||||
|
||||
@@ -170,7 +171,7 @@ def read_rows(conn, codes) -> dict[str, dict]:
|
||||
return out
|
||||
|
||||
|
||||
def write_field(conn, code: str, field: str, value) -> dict:
|
||||
def write_field(conn, code: str, field: str, value, observed=None, guard_observed: bool = False) -> dict:
|
||||
"""PRODUCT_ENRICHMENT_WRITE_V1: one field, one short transaction, only if the field is still empty. Returns {outcome, reason?}."""
|
||||
if field not in ALLOWED_FIELDS:
|
||||
raise ValueError(f"field {field!r} is outside the PRODUCT_ENRICHMENT_WRITE_V1 allowlist")
|
||||
@@ -188,6 +189,9 @@ def write_field(conn, code: str, field: str, value) -> dict:
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND"}
|
||||
raw = rows[0][0]
|
||||
if guard_observed and _text(raw) != _text(observed): # STALE_CONFLICT: the value changed between the Hub decision and now (e.g. a person typed something) -> zero write
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "STALE_CONFLICT"}
|
||||
if not writable_now(field, raw):
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"}
|
||||
@@ -235,6 +239,7 @@ class EnrichmentWorker:
|
||||
self._last_override = 0.0
|
||||
self._auth_cache = (None, 0.0)
|
||||
self._count_ms = 0.0
|
||||
self._last_pending = 0.0
|
||||
self._wconn = None
|
||||
self.metrics = {"detected": 0, "resolved": 0, "fields_written": 0, "detect_latency_ms": []}
|
||||
|
||||
@@ -372,7 +377,7 @@ class EnrichmentWorker:
|
||||
if mode == "DRY_RUN":
|
||||
reports.append({**base, "outcome": "DRY_RUN", "reason": "DRY_RUN_NO_WRITE"})
|
||||
continue
|
||||
outcome = self._write_safely(code, field, value)
|
||||
outcome = self._write_safely(code, field, value, observed=current.get(field))
|
||||
reports.append({**base, "outcome": outcome["outcome"], "reason": outcome.get("reason")})
|
||||
if outcome["outcome"] == "APPLIED":
|
||||
written[field] = value
|
||||
@@ -391,14 +396,14 @@ class EnrichmentWorker:
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("reprocess request for %s failed", code)
|
||||
|
||||
def _write_safely(self, code, field, value) -> dict:
|
||||
def _write_safely(self, code, field, value, observed=None) -> dict:
|
||||
"""Per field: fresh central authorization (fail closed) + guarded typed UPDATE. write_field re-reads the field INSIDE its own transaction and refuses when anything is there."""
|
||||
if not self._fresh_enabled():
|
||||
return {"outcome": "ABORTED", "reason": "AUTHORIZATION_CHANGED"}
|
||||
try:
|
||||
if self._wconn is None:
|
||||
self._wconn = self._open(self.write_conn)
|
||||
return write_field(self._wconn, code, field, value)
|
||||
return write_field(self._wconn, code, field, value, observed=observed, guard_observed=True)
|
||||
except Exception as error: # noqa: BLE001 - one field failing never breaks the others or the SGV
|
||||
self._close_write()
|
||||
return {"outcome": "FAILED", "reason": type(error).__name__}
|
||||
@@ -411,6 +416,26 @@ class EnrichmentWorker:
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
def reevaluate_flagged(self) -> list[str]:
|
||||
"""'Reavaliar' in the panel, Reprocessar/Sincronizar and KB decisions: re-run the normal resolve for tracked, not-yet-enriched products (KB/cache first, the Hub decides about providers)."""
|
||||
try:
|
||||
codes = self.hub.pending_enrichment()
|
||||
except Exception: # noqa: BLE001 - Hub offline: try again next cycle
|
||||
return []
|
||||
done = []
|
||||
for code in codes[:25]:
|
||||
item = self.store.get(code)
|
||||
if item is None:
|
||||
self.store.enqueue(code, self.clock())
|
||||
item = self.store.get(code)
|
||||
try:
|
||||
self.process(item)
|
||||
except Exception as error: # noqa: BLE001
|
||||
self._close_write()
|
||||
logger.warning("re-evaluation of %s failed: %s", code, type(error).__name__)
|
||||
done.append(code)
|
||||
return done
|
||||
|
||||
# ---- human override watch --------------------------------------------------------------------------------------------------------------------------
|
||||
def check_overrides(self):
|
||||
tracked = self.store.tracked_written()
|
||||
@@ -468,6 +493,9 @@ class EnrichmentWorker:
|
||||
if row and f"{pick_gtin(row)}|{_text(row['PRODUTO'])}" != item.get("snapshot"):
|
||||
self.process(item)
|
||||
done.append(item["code"])
|
||||
if self.clock() - self._last_pending >= PENDING_POLL_SECONDS:
|
||||
self._last_pending = self.clock()
|
||||
done += self.reevaluate_flagged()
|
||||
if self.clock() - self._last_override >= OVERRIDE_CHECK_SECONDS:
|
||||
self._last_override = self.clock()
|
||||
self.check_overrides()
|
||||
|
||||
@@ -9,7 +9,7 @@ import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_VERSION_DEFAULT = "1.3.4"
|
||||
AGENT_VERSION_DEFAULT = "1.3.5"
|
||||
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
|
||||
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
|
||||
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
|
||||
|
||||
@@ -49,6 +49,10 @@ class Hub:
|
||||
fields[f] = {"status": "ALREADY_SET", "value": None}
|
||||
return {"state": self.state, "fields": fields, "dataset_versions": {"NCM": {"sha256": "abc"}}}
|
||||
|
||||
def pending_enrichment(self):
|
||||
codes, self.flagged = list(getattr(self, "flagged", [])), []
|
||||
return codes
|
||||
|
||||
def report_enrichment(self, payload):
|
||||
self.reports.append(payload)
|
||||
for item in payload["fields"]:
|
||||
@@ -295,3 +299,84 @@ def test_write_field_guards_the_placeholder_swap():
|
||||
assert E.write_field(Conn(db), "c", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
||||
assert E.write_field(Conn(db), "d", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
||||
assert [r[0] for r in db.execute("SELECT CLASSIFICACAO_FISCAL FROM C000025 ORDER BY CODIGO")] == ["19059090", "22021000", "19059090", "19059090"]
|
||||
|
||||
|
||||
# ---- restart / race / multiple products / stale guard / reevaluation / provenance --------------------------------------------------------------
|
||||
def make_worker(db, hub, store, now):
|
||||
return E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=lambda codes: None, clock=lambda: now[0], sleep=lambda s: None)
|
||||
|
||||
|
||||
def test_product_created_while_agent_is_down_is_found_after_restart(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "D1") # created while the Agent is stopped / restarting
|
||||
worker2 = make_worker(db, hub, store, now) # new process, same persisted state
|
||||
now[0] += 5
|
||||
assert worker2.step() == ["D1"] and row(db, "D1")[0] == "22021000"
|
||||
|
||||
|
||||
def test_many_products_between_polls_and_a_deleted_one(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
for i in range(5):
|
||||
add(db, f"M{i}")
|
||||
db.execute("DELETE FROM C000025 WHERE CODIGO='M2'")
|
||||
db.commit()
|
||||
add(db, "M9")
|
||||
now[0] += 5
|
||||
done = worker.step()
|
||||
assert sorted(done) == ["M0", "M1", "M3", "M4", "M9"] and all(row(db, c)[0] == "22021000" for c in done)
|
||||
|
||||
|
||||
def test_reevaluation_flagged_by_the_hub_retries_an_unresolved_product(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
hub.decisions = {} # first attempt: nothing known yet
|
||||
add(db, "U1")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "U1")[0] is None and store.get("U1")["state"] == "DONE"
|
||||
hub.decisions = dict(DECISIONS) # new evidence (KB decision) + 'Reavaliar'/'Reprocessar'
|
||||
hub.flagged = ["U1"]
|
||||
now[0] += 30
|
||||
assert "U1" in worker.step() and row(db, "U1")[0] == "22021000"
|
||||
now[0] += 30
|
||||
before = db.total_changes
|
||||
worker.step()
|
||||
assert db.total_changes == before
|
||||
|
||||
|
||||
def test_stale_write_is_refused_when_a_person_changed_the_value_after_the_decision(env):
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)")
|
||||
db.execute("INSERT INTO C000025 VALUES('a','19059090')")
|
||||
db.execute("INSERT INTO C000025 VALUES('b','00000000')")
|
||||
db.commit()
|
||||
assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "22021000", observed="00000000", guard_observed=True)["reason"] == "STALE_CONFLICT"
|
||||
assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "22021000", observed="00000000", guard_observed=True)["outcome"] == "APPLIED"
|
||||
|
||||
|
||||
def test_placeholder_after_a_human_override_is_not_filled_again(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add_with_ncm(db, "HP1", "00000000") # born with the placeholder: eligible
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "HP1")[0] == "22021000"
|
||||
db.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL='00000000' WHERE CODIGO='HP1'") # a person puts the SAME value back: a human decision now
|
||||
db.commit()
|
||||
now[0] += 30
|
||||
worker.step()
|
||||
assert store.is_overridden("HP1", "CLASSIFICACAO_FISCAL")
|
||||
hub.flagged = ["HP1"]
|
||||
now[0] += 30
|
||||
worker.step()
|
||||
assert row(db, "HP1")[0] == "00000000" # provenance matters: same value, different meaning
|
||||
|
||||
|
||||
def test_automatic_mode_needs_no_sync_button_or_restart(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "A1")
|
||||
now[0] += 3
|
||||
assert worker.step() == ["A1"] and row(db, "A1")[0] == "22021000" # save -> next poll -> written
|
||||
|
||||
Reference in New Issue
Block a user