gustavosantos/GTI_GERENCIADOR_FISCAL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: product enrichment observability, lab KB study, benchmark, copy + local-Hub E2E, docs; 1.3.0

Browse Source 
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
2026-09-25 19:26:13 -03:00
co-authored by Claude Sonnet 5
parent 5deebea14e
commit 4fc83a33d5
38 changed files with 1282 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/pyproject.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "gti-fiscal-agent"
version = "1.2.5"
version = "1.3.0"
requires-python = ">=3.10"
dependencies = ["httpx>=0.27,<1"]
agent/scripts/run_agent_service.py
+3 -3
View File
@@ -107,7 +107,7 @@ class Config:
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
# RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib.
self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB)))
self.agent_version = env.get("AGENT_VERSION", "1.2.5")
self.agent_version = env.get("AGENT_VERSION", "1.3.0")
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
# Pilot write authorization: both must be explicitly set by the installer for
@@ -349,7 +349,7 @@ def execute_job(job, config, executor, state, env):
"fiscal_context": fiscal_context,
"rtc_auto_apply": rtc_apply,
"fiscal_mode": getattr(config, "fiscal_mode", None),
"release_version": env.get("AGENT_VERSION") or "1.2.5",
"release_version": env.get("AGENT_VERSION") or "1.3.0",
"products_total": len(products),
"products_changed": len(to_process),
"products_processed": summary["processed"],
@@ -520,7 +520,7 @@ def run(config, env, *, max_iterations=None, stopper=None):
def enforce_startup_preflight(env):
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
from gti_fiscal_agent.preflight import run_preflight
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.2.5")
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.0")
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
if report["PREFLIGHT"] != "PASS":
agent/src/gti_fiscal_agent/enrichment.py
+67 -24
View File
@@ -22,8 +22,8 @@ logger = logging.getLogger("gti_fiscal_agent.enrichment")
CAPABILITY = "PRODUCT_ENRICHMENT_WRITE_V1"
ALLOWED_FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI") # explicit allowlist; anything else raises before touching SQL
READ_FIELDS = ("CODIGO", "PRODUTO", "GTIN") + ALLOWED_FIELDS
POLL_SECONDS = 3.0
READ_FIELDS = ("CODIGO", "PRODUTO", "GTIN", "CODBARRA") + ALLOWED_FIELDS # the SGV operator types the EAN in CODBARRA; GTIN (fiscal) is mostly empty
POLL_SECONDS = 2.0
SAFETY_DIFF_SECONDS = 60.0
OVERRIDE_CHECK_SECONDS = 20.0
WATCH_WINDOW_SECONDS = 120.0 # a fresh product is re-read for a while: the operator may add the EAN seconds after the first save
@@ -112,6 +112,21 @@ def _digits(value) -> str:
return "".join(ch for ch in _text(value) if ch.isdigit())
def gtin_check_ok(value) -> bool:
"""Same acceptance as central.app.enrichment.gtin (8/12/13/14 digits + check digit); parity is asserted by a test. Only used to pick WHICH column to send: the Hub decides."""
d = _digits(value)
if len(d) not in (8, 12, 13, 14) or len(set(d)) == 1:
return False
total = sum(int(x) * (3 if i % 2 == 0 else 1) for i, x in enumerate(reversed(d[:-1])))
return (10 - total % 10) % 10 == int(d[-1])
def pick_gtin(row) -> str:
"""First VALID of GTIN / CODBARRA; otherwise whatever was typed (the Hub classifies it INVALID_GTIN/MISSING_GTIN without calling any provider)."""
candidates = [_text(row.get("GTIN")), _text(row.get("CODBARRA"))]
return next((c for c in candidates if gtin_check_ok(c)), next((c for c in candidates if c), ""))
def read_codes(conn) -> set[str]:
cur = conn.cursor()
cur.execute("SELECT CODIGO FROM C000025")
@@ -202,8 +217,20 @@ class EnrichmentWorker:
self._last_diff = 0.0
self._last_override = 0.0
self._auth_cache = (None, 0.0)
self._count_ms = 0.0
self._wconn = None
self.metrics = {"detected": 0, "resolved": 0, "fields_written": 0, "detect_latency_ms": []}
def _open(self, factory, attempts=3):
"""Firebird can drop a connection under load ('connection lost'): retry briefly instead of aborting the product."""
for attempt in range(attempts):
try:
return factory()
except Exception: # noqa: BLE001
if attempt == attempts - 1:
raise
self.sleep(0.3 * (attempt + 1))
# ---- feature gate ------------------------------------------------------------------------------------------------------------------------------
def features(self, max_age=10.0) -> dict:
if self._features is None or self.clock() - self._features_at > max_age:
@@ -234,7 +261,7 @@ class EnrichmentWorker:
# ---- detection ---------------------------------------------------------------------------------------------------------------------------------
def detect(self) -> list[str]:
conn = self.read_conn()
conn = self._open(self.read_conn)
try:
if self.store.meta("baseline_done") != "1":
codes = read_codes(conn)
@@ -245,7 +272,9 @@ class EnrichmentWorker:
self._last_count, self._last_diff = len(codes), self.clock()
return []
started = self.clock()
t0 = time.monotonic()
count = read_count(conn)
self._count_ms = (time.monotonic() - t0) * 1000
moved = self._last_count is None or count != self._last_count
if not moved and self.clock() - self._last_diff < SAFETY_DIFF_SECONDS:
return []
@@ -265,7 +294,7 @@ class EnrichmentWorker:
# ---- one product ---------------------------------------------------------------------------------------------------------------------------------
def process(self, item: dict):
code, mode = item["code"], self.mode()
conn = self.read_conn()
conn = self._open(self.read_conn)
try:
rows = read_rows(conn, [code])
finally:
@@ -274,14 +303,14 @@ class EnrichmentWorker:
if row is None:
self.store.update(code, state="GONE")
return
snapshot = f"{_text(row['GTIN'])}|{_text(row['PRODUTO'])}"
snapshot = f"{pick_gtin(row)}|{_text(row['PRODUTO'])}"
if mode not in ("DRY_RUN", "ENABLED"):
self.store.update(code, state="SKIPPED_FEATURE", snapshot=snapshot)
return
attempts = int(item["attempts"] or 0) + 1
current = {f: row[f] for f in ALLOWED_FIELDS}
try:
result = self.hub.resolve_product({"product_code": code, "gtin": _text(row["GTIN"]), "description": _text(row["PRODUTO"]), "current": {k: (None if v is None else str(v)) for k, v in current.items()}})
result = self.hub.resolve_product({"product_code": code, "gtin": pick_gtin(row), "description": _text(row["PRODUTO"]), "current": {k: (None if v is None else str(v)) for k, v in current.items()}})
except Exception as error: # noqa: BLE001 - Hub offline / slow: the product stays pending, the SGV is unaffected
delay = RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)]
self.store.update(code, state="RETRY", attempts=attempts, next_attempt_at=self.clock() + delay, last_state=f"HUB_ERROR:{type(error).__name__}")
@@ -322,6 +351,8 @@ class EnrichmentWorker:
if outcome["outcome"] == "APPLIED":
written[field] = value
self.metrics["fields_written"] += 1
self.store.update(code, written=written) # remembered at once: a crash right after a write still leaves it protected/tracked
self._close_write()
self.store.update(code, state="DRY_RUN_DONE" if mode == "DRY_RUN" else "DONE", attempts=attempts, snapshot=snapshot, written=written, last_state=state, next_attempt_at=None)
if reports:
try:
@@ -335,29 +366,31 @@ class EnrichmentWorker:
logger.warning("reprocess request for %s failed", code)
def _write_safely(self, code, field, value) -> dict:
"""Per field: fresh central authorization (fail closed) + guarded typed UPDATE. write_field re-reads the field INSIDE its own transaction and refuses when anything is there."""
if not self._fresh_enabled():
return {"outcome": "ABORTED", "reason": "AUTHORIZATION_CHANGED"}
conn = self.read_conn()
try:
fresh = read_rows(conn, [code]).get(code)
finally:
conn.close()
if fresh is None:
return {"outcome": "ABORTED", "reason": "PRODUCT_NOT_FOUND"}
if _text(fresh[field]):
return {"outcome": "ABORTED", "reason": "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"}
wconn = self.write_conn()
try:
return write_field(wconn, code, field, value)
finally:
wconn.close()
if self._wconn is None:
self._wconn = self._open(self.write_conn)
return write_field(self._wconn, code, field, value)
except Exception as error: # noqa: BLE001 - one field failing never breaks the others or the SGV
self._close_write()
return {"outcome": "FAILED", "reason": type(error).__name__}
def _close_write(self):
conn, self._wconn = self._wconn, None
if conn is not None:
try:
conn.close()
except Exception: # noqa: BLE001
pass
# ---- human override watch --------------------------------------------------------------------------------------------------------------------------
def check_overrides(self):
tracked = self.store.tracked_written()
if not tracked:
return
conn = self.read_conn()
conn = self._open(self.read_conn)
try:
rows = read_rows(conn, [t["code"] for t in tracked])
finally:
@@ -388,17 +421,23 @@ class EnrichmentWorker:
self.detect()
done = []
for item in self.store.due(self.clock()):
self.process(item)
try:
self.process(item)
except Exception as error: # noqa: BLE001 - keep the product retryable, never lose the batch
self._close_write()
attempts = int(item["attempts"] or 0) + 1
self.store.update(item["code"], state="RETRY", attempts=attempts, next_attempt_at=self.clock() + RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)], last_state=f"ERROR:{type(error).__name__}")
logger.warning("enrichment of %s failed: %s", item["code"], type(error).__name__)
done.append(item["code"])
for item in self.store.watching(self.clock()): # operator completes the record (EAN typed a moment later): re-resolve once per change
rows = None
conn = self.read_conn()
conn = self._open(self.read_conn)
try:
rows = read_rows(conn, [item["code"]])
finally:
conn.close()
row = (rows or {}).get(item["code"])
if row and f"{_text(row['GTIN'])}|{_text(row['PRODUTO'])}" != item.get("snapshot"):
if row and f"{pick_gtin(row)}|{_text(row['PRODUTO'])}" != item.get("snapshot"):
self.process(item)
done.append(item["code"])
if self.clock() - self._last_override >= OVERRIDE_CHECK_SECONDS:
@@ -406,10 +445,14 @@ class EnrichmentWorker:
self.check_overrides()
return done
def poll_seconds(self) -> float:
"""Adaptive: the SGV is never loaded by more than ~2.5% of one query-time. 3 s on small catalogs, longer when COUNT(*) is slow (huge catalogs); never above 30 s."""
return min(30.0, max(POLL_SECONDS, 40 * self._count_ms / 1000))
def run_forever(self, stopper):
while not stopper.stop:
try:
self.step()
except Exception: # noqa: BLE001 - the enrichment thread must never die or affect the SGV
logger.exception("enrichment step failed")
self.sleep(POLL_SECONDS)
self.sleep(self.poll_seconds())
agent/src/gti_fiscal_agent/preflight.py
+1 -1
View File
@@ -9,7 +9,7 @@ import os
import sqlite3
from pathlib import Path
AGENT_VERSION_DEFAULT = "1.2.5"
AGENT_VERSION_DEFAULT = "1.3.0"
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
agent/tests/test_enrichment.py
+11 -1
View File
@@ -66,7 +66,7 @@ DECISIONS = {"CLASSIFICACAO_FISCAL": {"status": "AUTO_FILL_SAFE", "value": "2202
@pytest.fixture()
def env(tmp_path):
db = sqlite3.connect(":memory:", check_same_thread=False)
db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT)")
db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT, CODBARRA TEXT)")
db.executemany("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", [(f"H{i}", f"HIST {i}", "", "keep") for i in range(50)])
db.commit()
hub = Hub(dict(ENABLED), DECISIONS)
@@ -199,3 +199,13 @@ def test_hub_offline_keeps_product_pending_and_retries(env):
hub.resolve_product = good
now[0] += 40
assert worker.step() == ["N6"] and row(db, "N6")[0] == "22021000"
def test_gtin_check_parity_with_the_hub_and_codbarra_is_used():
sys.path.insert(0, str(ROOT))
from central.app.enrichment.gtin import normalize_gtin
for value in ("7891000100103", "7891000100104", "", "0", "12345678", "40170725", "036000291452", "11111111", "78910001001030", "abc", "2000000000008"):
assert E.gtin_check_ok(value) == (normalize_gtin(value)["status"] in ("VALID_GTIN", "RESTRICTED_GTIN")), value
assert E.pick_gtin({"GTIN": "", "CODBARRA": "7891000100103"}) == "7891000100103"
assert E.pick_gtin({"GTIN": "7891000100103", "CODBARRA": "999"}) == "7891000100103"
assert E.pick_gtin({"GTIN": "", "CODBARRA": "123"}) == "123" and E.pick_gtin({}) == ""