gustavosantos/GTI_GERENCIADOR_FISCAL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: product enrichment observability, lab KB study, benchmark, copy + local-Hub E2E, docs; 1.3.0

Browse Source 
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
2026-09-25 19:26:13 -03:00
co-authored by Claude Sonnet 5
parent 5deebea14e
commit 4fc83a33d5
38 changed files with 1282 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/src/gti_fiscal_agent/enrichment.py
+67 -24
View File
@@ -22,8 +22,8 @@ logger = logging.getLogger("gti_fiscal_agent.enrichment")
CAPABILITY = "PRODUCT_ENRICHMENT_WRITE_V1"
ALLOWED_FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI") # explicit allowlist; anything else raises before touching SQL
READ_FIELDS = ("CODIGO", "PRODUTO", "GTIN") + ALLOWED_FIELDS
POLL_SECONDS = 3.0
READ_FIELDS = ("CODIGO", "PRODUTO", "GTIN", "CODBARRA") + ALLOWED_FIELDS # the SGV operator types the EAN in CODBARRA; GTIN (fiscal) is mostly empty
POLL_SECONDS = 2.0
SAFETY_DIFF_SECONDS = 60.0
OVERRIDE_CHECK_SECONDS = 20.0
WATCH_WINDOW_SECONDS = 120.0 # a fresh product is re-read for a while: the operator may add the EAN seconds after the first save
@@ -112,6 +112,21 @@ def _digits(value) -> str:
return "".join(ch for ch in _text(value) if ch.isdigit())
def gtin_check_ok(value) -> bool:
"""Same acceptance as central.app.enrichment.gtin (8/12/13/14 digits + check digit); parity is asserted by a test. Only used to pick WHICH column to send: the Hub decides."""
d = _digits(value)
if len(d) not in (8, 12, 13, 14) or len(set(d)) == 1:
return False
total = sum(int(x) * (3 if i % 2 == 0 else 1) for i, x in enumerate(reversed(d[:-1])))
return (10 - total % 10) % 10 == int(d[-1])
def pick_gtin(row) -> str:
"""First VALID of GTIN / CODBARRA; otherwise whatever was typed (the Hub classifies it INVALID_GTIN/MISSING_GTIN without calling any provider)."""
candidates = [_text(row.get("GTIN")), _text(row.get("CODBARRA"))]
return next((c for c in candidates if gtin_check_ok(c)), next((c for c in candidates if c), ""))
def read_codes(conn) -> set[str]:
cur = conn.cursor()
cur.execute("SELECT CODIGO FROM C000025")
@@ -202,8 +217,20 @@ class EnrichmentWorker:
self._last_diff = 0.0
self._last_override = 0.0
self._auth_cache = (None, 0.0)
self._count_ms = 0.0
self._wconn = None
self.metrics = {"detected": 0, "resolved": 0, "fields_written": 0, "detect_latency_ms": []}
def _open(self, factory, attempts=3):
"""Firebird can drop a connection under load ('connection lost'): retry briefly instead of aborting the product."""
for attempt in range(attempts):
try:
return factory()
except Exception: # noqa: BLE001
if attempt == attempts - 1:
raise
self.sleep(0.3 * (attempt + 1))
# ---- feature gate ------------------------------------------------------------------------------------------------------------------------------
def features(self, max_age=10.0) -> dict:
if self._features is None or self.clock() - self._features_at > max_age:
@@ -234,7 +261,7 @@ class EnrichmentWorker:
# ---- detection ---------------------------------------------------------------------------------------------------------------------------------
def detect(self) -> list[str]:
conn = self.read_conn()
conn = self._open(self.read_conn)
try:
if self.store.meta("baseline_done") != "1":
codes = read_codes(conn)
@@ -245,7 +272,9 @@ class EnrichmentWorker:
self._last_count, self._last_diff = len(codes), self.clock()
return []
started = self.clock()
t0 = time.monotonic()
count = read_count(conn)
self._count_ms = (time.monotonic() - t0) * 1000
moved = self._last_count is None or count != self._last_count
if not moved and self.clock() - self._last_diff < SAFETY_DIFF_SECONDS:
return []
@@ -265,7 +294,7 @@ class EnrichmentWorker:
# ---- one product ---------------------------------------------------------------------------------------------------------------------------------
def process(self, item: dict):
code, mode = item["code"], self.mode()
conn = self.read_conn()
conn = self._open(self.read_conn)
try:
rows = read_rows(conn, [code])
finally:
@@ -274,14 +303,14 @@ class EnrichmentWorker:
if row is None:
self.store.update(code, state="GONE")
return
snapshot = f"{_text(row['GTIN'])}|{_text(row['PRODUTO'])}"
snapshot = f"{pick_gtin(row)}|{_text(row['PRODUTO'])}"
if mode not in ("DRY_RUN", "ENABLED"):
self.store.update(code, state="SKIPPED_FEATURE", snapshot=snapshot)
return
attempts = int(item["attempts"] or 0) + 1
current = {f: row[f] for f in ALLOWED_FIELDS}
try:
result = self.hub.resolve_product({"product_code": code, "gtin": _text(row["GTIN"]), "description": _text(row["PRODUTO"]), "current": {k: (None if v is None else str(v)) for k, v in current.items()}})
result = self.hub.resolve_product({"product_code": code, "gtin": pick_gtin(row), "description": _text(row["PRODUTO"]), "current": {k: (None if v is None else str(v)) for k, v in current.items()}})
except Exception as error: # noqa: BLE001 - Hub offline / slow: the product stays pending, the SGV is unaffected
delay = RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)]
self.store.update(code, state="RETRY", attempts=attempts, next_attempt_at=self.clock() + delay, last_state=f"HUB_ERROR:{type(error).__name__}")
@@ -322,6 +351,8 @@ class EnrichmentWorker:
if outcome["outcome"] == "APPLIED":
written[field] = value
self.metrics["fields_written"] += 1
self.store.update(code, written=written) # remembered at once: a crash right after a write still leaves it protected/tracked
self._close_write()
self.store.update(code, state="DRY_RUN_DONE" if mode == "DRY_RUN" else "DONE", attempts=attempts, snapshot=snapshot, written=written, last_state=state, next_attempt_at=None)
if reports:
try:
@@ -335,29 +366,31 @@ class EnrichmentWorker:
logger.warning("reprocess request for %s failed", code)
def _write_safely(self, code, field, value) -> dict:
"""Per field: fresh central authorization (fail closed) + guarded typed UPDATE. write_field re-reads the field INSIDE its own transaction and refuses when anything is there."""
if not self._fresh_enabled():
return {"outcome": "ABORTED", "reason": "AUTHORIZATION_CHANGED"}
conn = self.read_conn()
try:
fresh = read_rows(conn, [code]).get(code)
finally:
conn.close()
if fresh is None:
return {"outcome": "ABORTED", "reason": "PRODUCT_NOT_FOUND"}
if _text(fresh[field]):
return {"outcome": "ABORTED", "reason": "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"}
wconn = self.write_conn()
try:
return write_field(wconn, code, field, value)
finally:
wconn.close()
if self._wconn is None:
self._wconn = self._open(self.write_conn)
return write_field(self._wconn, code, field, value)
except Exception as error: # noqa: BLE001 - one field failing never breaks the others or the SGV
self._close_write()
return {"outcome": "FAILED", "reason": type(error).__name__}
def _close_write(self):
conn, self._wconn = self._wconn, None
if conn is not None:
try:
conn.close()
except Exception: # noqa: BLE001
pass
# ---- human override watch --------------------------------------------------------------------------------------------------------------------------
def check_overrides(self):
tracked = self.store.tracked_written()
if not tracked:
return
conn = self.read_conn()
conn = self._open(self.read_conn)
try:
rows = read_rows(conn, [t["code"] for t in tracked])
finally:
@@ -388,17 +421,23 @@ class EnrichmentWorker:
self.detect()
done = []
for item in self.store.due(self.clock()):
self.process(item)
try:
self.process(item)
except Exception as error: # noqa: BLE001 - keep the product retryable, never lose the batch
self._close_write()
attempts = int(item["attempts"] or 0) + 1
self.store.update(item["code"], state="RETRY", attempts=attempts, next_attempt_at=self.clock() + RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)], last_state=f"ERROR:{type(error).__name__}")
logger.warning("enrichment of %s failed: %s", item["code"], type(error).__name__)
done.append(item["code"])
for item in self.store.watching(self.clock()): # operator completes the record (EAN typed a moment later): re-resolve once per change
rows = None
conn = self.read_conn()
conn = self._open(self.read_conn)
try:
rows = read_rows(conn, [item["code"]])
finally:
conn.close()
row = (rows or {}).get(item["code"])
if row and f"{_text(row['GTIN'])}|{_text(row['PRODUTO'])}" != item.get("snapshot"):
if row and f"{pick_gtin(row)}|{_text(row['PRODUTO'])}" != item.get("snapshot"):
self.process(item)
done.append(item["code"])
if self.clock() - self._last_override >= OVERRIDE_CHECK_SECONDS:
@@ -406,10 +445,14 @@ class EnrichmentWorker:
self.check_overrides()
return done
def poll_seconds(self) -> float:
"""Adaptive: the SGV is never loaded by more than ~2.5% of one query-time. 3 s on small catalogs, longer when COUNT(*) is slow (huge catalogs); never above 30 s."""
return min(30.0, max(POLL_SECONDS, 40 * self._count_ms / 1000))
def run_forever(self, stopper):
while not stopper.stop:
try:
self.step()
except Exception: # noqa: BLE001 - the enrichment thread must never die or affect the SGV
logger.exception("enrichment step failed")
self.sleep(POLL_SECONDS)
self.sleep(self.poll_seconds())
agent/src/gti_fiscal_agent/preflight.py
+1 -1
View File
@@ -9,7 +9,7 @@ import os
import sqlite3
from pathlib import Path
AGENT_VERSION_DEFAULT = "1.2.5"
AGENT_VERSION_DEFAULT = "1.3.0"
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")