515 lines
26 KiB
Python
515 lines
26 KiB
Python
"""New-product enrichment on the Agent: detect NEW rows of C000025, ask the Hub, write ONLY safe empty fields.
|
|
|
|
Safety model (defense in depth; nothing here is switchable locally - the Hub owns the per-installation feature flags):
|
|
- Feature gate: nothing touches Firebird unless the Hub says features_configured AND enrichment_mode != DISABLED. DRY_RUN never opens a write connection.
|
|
- Scope: only products that appear AFTER the baseline (taken when the function is first active). The historical catalog is never sent anywhere.
|
|
- No trigger, no table, no procedure in the SGV database: detection is one `SELECT COUNT(*)` per poll (+ a code diff only when the count moved, and every 60 s as a safety net).
|
|
- Typed capability PRODUCT_ENRICHMENT_WRITE_V1: ALLOWED_FIELDS is an explicit allowlist on C000025; the only statement is a fixed UPDATE ... WHERE CODIGO=? AND (field IS NULL OR TRIM(field)=''),
|
|
one short transaction per field, followed by a re-read. COD_ORIGEM_MERCADORIA and cClassTrib are NOT in the list (cClassTrib stays with the existing RTC gates).
|
|
- Right before EVERY field write: re-read the row, re-check the Hub authorization (fail closed), skip when a human already touched the field.
|
|
- Human overrides: after writing, the Agent watches the value; anything different from what it wrote (including a deleted value) marks the field HUMAN_OVERRIDDEN forever, is reported to the Hub
|
|
and is never written again by the enrichment (nor re-suggested by the Hub).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import sqlite3
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
logger = logging.getLogger("gti_fiscal_agent.enrichment")
|
|
|
|
CAPABILITY = "PRODUCT_ENRICHMENT_WRITE_V1"
|
|
ALLOWED_FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI")
|
|
# TECHNICAL_PLACEHOLDER: the SGV refuses a product without NCM, so operators type a filler. It means "not classified yet" (not a human decision) and may be replaced.
|
|
# Explicit per-field allowlist, identical to central/app/enrichment/placeholders.py (parity test). Nothing else is ever overwritten.
|
|
PLACEHOLDERS = {"CLASSIFICACAO_FISCAL": frozenset({"00000000"})} # explicit allowlist; anything else raises before touching SQL
|
|
READ_FIELDS = ("CODIGO", "PRODUTO", "GTIN", "CODBARRA") + ALLOWED_FIELDS # the SGV operator types the EAN in CODBARRA; GTIN (fiscal) is mostly empty
|
|
POLL_SECONDS = 2.0
|
|
SAFETY_DIFF_SECONDS = 60.0
|
|
OVERRIDE_CHECK_SECONDS = 20.0
|
|
PENDING_POLL_SECONDS = 10.0 # cheap GET: products the Hub flagged for a new evaluation
|
|
WATCH_WINDOW_SECONDS = 120.0 # a fresh product is re-read for a while: the operator may add the EAN seconds after the first save
|
|
RETRY_BACKOFF = (30, 60, 120, 300, 600, 1800)
|
|
|
|
|
|
def _now() -> float:
|
|
return time.time()
|
|
|
|
|
|
def _text(value) -> str:
|
|
return "" if value is None else str(value).strip()
|
|
|
|
|
|
class EnrichmentStore:
|
|
"""Own SQLite connection (the worker runs in its own thread)."""
|
|
|
|
def __init__(self, path):
|
|
self.db = sqlite3.connect(str(path), timeout=10, check_same_thread=False)
|
|
self.db.row_factory = sqlite3.Row
|
|
self.db.executescript("""
|
|
CREATE TABLE IF NOT EXISTS enrich_meta(k TEXT PRIMARY KEY, v TEXT);
|
|
CREATE TABLE IF NOT EXISTS enrich_known(code TEXT PRIMARY KEY);
|
|
CREATE TABLE IF NOT EXISTS enrich_products(
|
|
code TEXT PRIMARY KEY, state TEXT NOT NULL, detected_at REAL NOT NULL, next_attempt_at REAL, attempts INTEGER NOT NULL DEFAULT 0,
|
|
snapshot TEXT, written TEXT NOT NULL DEFAULT '{}', overridden TEXT NOT NULL DEFAULT '{}', last_state TEXT);
|
|
""")
|
|
self.db.commit()
|
|
|
|
def meta(self, key, default=None):
|
|
row = self.db.execute("SELECT v FROM enrich_meta WHERE k=?", (key,)).fetchone()
|
|
return row["v"] if row else default
|
|
|
|
def set_meta(self, key, value):
|
|
self.db.execute("INSERT INTO enrich_meta(k,v) VALUES(?,?) ON CONFLICT(k) DO UPDATE SET v=excluded.v", (key, str(value)))
|
|
self.db.commit()
|
|
|
|
def known(self) -> set[str]:
|
|
return {r["code"] for r in self.db.execute("SELECT code FROM enrich_known")}
|
|
|
|
def add_known(self, codes):
|
|
self.db.executemany("INSERT OR IGNORE INTO enrich_known(code) VALUES(?)", [(c,) for c in codes])
|
|
self.db.commit()
|
|
|
|
def enqueue(self, code, now=None):
|
|
now = _now() if now is None else now
|
|
self.db.execute("INSERT OR IGNORE INTO enrich_products(code,state,detected_at,next_attempt_at) VALUES(?,?,?,?)", (code, "PENDING", now, now))
|
|
self.db.commit()
|
|
|
|
def get(self, code):
|
|
row = self.db.execute("SELECT * FROM enrich_products WHERE code=?", (code,)).fetchone()
|
|
return dict(row) if row else None
|
|
|
|
def due(self, now):
|
|
return [dict(r) for r in self.db.execute("SELECT * FROM enrich_products WHERE state IN ('PENDING','RETRY') AND next_attempt_at<=? ORDER BY detected_at LIMIT 20", (now,))]
|
|
|
|
def watching(self, now):
|
|
return [dict(r) for r in self.db.execute("SELECT * FROM enrich_products WHERE state IN ('DONE','DRY_RUN_DONE') AND detected_at>=?", (now - WATCH_WINDOW_SECONDS,))]
|
|
|
|
def tracked_written(self):
|
|
return [dict(r) for r in self.db.execute("SELECT * FROM enrich_products WHERE written != '{}'")]
|
|
|
|
def update(self, code, **values):
|
|
for key in ("written", "overridden"):
|
|
if key in values and not isinstance(values[key], str):
|
|
values[key] = json.dumps(values[key])
|
|
sets = ",".join(f"{k}=?" for k in values)
|
|
self.db.execute(f"UPDATE enrich_products SET {sets} WHERE code=?", (*values.values(), code))
|
|
self.db.commit()
|
|
|
|
def is_overridden(self, code, field) -> bool:
|
|
row = self.get(code)
|
|
return bool(row and json.loads(row["overridden"] or "{}").get(field))
|
|
|
|
|
|
def is_human_overridden(store_path, code, field) -> bool:
|
|
"""Used by every automatic writer of the Agent (the Fiscal Corrector included) before touching a field the enrichment tracks."""
|
|
try:
|
|
return EnrichmentStore(store_path).is_overridden(str(code), field)
|
|
except Exception: # noqa: BLE001 - no state means nothing is protected yet
|
|
return False
|
|
|
|
|
|
# ---- Firebird access (injected: `read_conn()` returns a READ-ONLY connection, `write_conn()` a write-capable one, only ever opened by ENABLED mode) ----------------------------------------
|
|
def is_placeholder(field, value) -> bool:
|
|
text = _text(value)
|
|
text = "".join(ch for ch in text if ch.isdigit()) if field == "CLASSIFICACAO_FISCAL" else text
|
|
return text in PLACEHOLDERS.get(field, frozenset())
|
|
|
|
|
|
def writable_now(field, value) -> bool:
|
|
return not _text(value) or is_placeholder(field, value)
|
|
|
|
|
|
def _digits(value) -> str:
|
|
return "".join(ch for ch in _text(value) if ch.isdigit())
|
|
|
|
|
|
def gtin_check_ok(value) -> bool:
|
|
"""Same acceptance as central.app.enrichment.gtin (8/12/13/14 digits + check digit); parity is asserted by a test. Only used to pick WHICH column to send: the Hub decides."""
|
|
d = _digits(value)
|
|
if len(d) not in (8, 12, 13, 14) or len(set(d)) == 1:
|
|
return False
|
|
total = sum(int(x) * (3 if i % 2 == 0 else 1) for i, x in enumerate(reversed(d[:-1])))
|
|
return (10 - total % 10) % 10 == int(d[-1])
|
|
|
|
|
|
def pick_gtin(row) -> str:
|
|
"""First VALID of GTIN / CODBARRA; otherwise whatever was typed (the Hub classifies it INVALID_GTIN/MISSING_GTIN without calling any provider)."""
|
|
candidates = [_text(row.get("GTIN")), _text(row.get("CODBARRA"))]
|
|
return next((c for c in candidates if gtin_check_ok(c)), next((c for c in candidates if c), ""))
|
|
|
|
|
|
def read_codes(conn) -> set[str]:
|
|
cur = conn.cursor()
|
|
cur.execute("SELECT CODIGO FROM C000025")
|
|
codes = {_text(r[0]) for r in cur.fetchall()}
|
|
conn.commit()
|
|
return codes
|
|
|
|
|
|
def read_count(conn) -> int:
|
|
cur = conn.cursor()
|
|
cur.execute("SELECT COUNT(*) FROM C000025")
|
|
n = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return n
|
|
|
|
|
|
def read_rows(conn, codes) -> dict[str, dict]:
|
|
out = {}
|
|
codes = list(codes)
|
|
for start in range(0, len(codes), 100):
|
|
chunk = codes[start:start + 100]
|
|
cur = conn.cursor()
|
|
cur.execute(f"SELECT {','.join(READ_FIELDS)} FROM C000025 WHERE CODIGO IN ({','.join('?' for _ in chunk)})", tuple(chunk))
|
|
for row in cur.fetchall():
|
|
rec = dict(zip(READ_FIELDS, row))
|
|
out[_text(rec["CODIGO"])] = rec
|
|
conn.commit()
|
|
return out
|
|
|
|
|
|
def write_field(conn, code: str, field: str, value, observed=None, guard_observed: bool = False) -> dict:
|
|
"""PRODUCT_ENRICHMENT_WRITE_V1: one field, one short transaction, only if the field is still empty. Returns {outcome, reason?}."""
|
|
if field not in ALLOWED_FIELDS:
|
|
raise ValueError(f"field {field!r} is outside the PRODUCT_ENRICHMENT_WRITE_V1 allowlist")
|
|
if field == "IPI":
|
|
value = float(value)
|
|
else:
|
|
value = _digits(value)
|
|
if not value or (field == "CLASSIFICACAO_FISCAL" and len(value) != 8) or (field == "CEST" and len(value) != 7):
|
|
return {"outcome": "ABORTED", "reason": "VALUE_SHAPE_INVALID"}
|
|
cur = conn.cursor()
|
|
try:
|
|
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
|
rows = cur.fetchall()
|
|
if len(rows) != 1:
|
|
conn.rollback()
|
|
return {"outcome": "ABORTED", "reason": "PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND"}
|
|
raw = rows[0][0]
|
|
if guard_observed and _text(raw) != _text(observed): # STALE_CONFLICT: the value changed between the Hub decision and now (e.g. a person typed something) -> zero write
|
|
conn.rollback()
|
|
return {"outcome": "ABORTED", "reason": "STALE_CONFLICT"}
|
|
if not writable_now(field, raw):
|
|
conn.rollback()
|
|
return {"outcome": "ABORTED", "reason": "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"}
|
|
if _text(raw): # technical placeholder: compare-and-swap on the exact value just read
|
|
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND TRIM({field}) = ?", (value, code, _text(raw)))
|
|
else:
|
|
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND ({field} IS NULL OR TRIM({field}) = '')", (value, code))
|
|
if cur.rowcount != 1:
|
|
conn.rollback()
|
|
return {"outcome": "ABORTED", "reason": "UPDATE_ROWCOUNT_NOT_ONE"}
|
|
conn.commit()
|
|
cur = conn.cursor()
|
|
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
|
after = cur.fetchone()[0]
|
|
conn.commit()
|
|
same = (float(after) == value) if field == "IPI" and after is not None else _text(after) == value
|
|
if not same:
|
|
return {"outcome": "FAILED", "reason": "POST_WRITE_VERIFICATION_FAILED"}
|
|
return {"outcome": "APPLIED", "value": value, "rowcount": 1, "replaced_placeholder": _text(raw) if _text(raw) else None}
|
|
except Exception as error: # noqa: BLE001
|
|
try:
|
|
conn.rollback()
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
return {"outcome": "FAILED", "reason": type(error).__name__}
|
|
|
|
|
|
def _same_value(field, current, written) -> bool:
|
|
if field == "IPI":
|
|
try:
|
|
return current is not None and _text(current) != "" and abs(float(current) - float(written)) < 1e-9
|
|
except (TypeError, ValueError):
|
|
return False
|
|
return _text(current) == _text(written) and _text(current) != ""
|
|
|
|
|
|
class EnrichmentWorker:
|
|
def __init__(self, *, hub, store: EnrichmentStore, read_conn, write_conn, create_reprocess=None, clock=_now, sleep=time.sleep):
|
|
self.hub, self.store, self.read_conn, self.write_conn = hub, store, read_conn, write_conn
|
|
self.create_reprocess, self.clock, self.sleep = create_reprocess, clock, sleep
|
|
self._features = None
|
|
self._features_at = 0.0
|
|
self._last_count = None
|
|
self._last_diff = 0.0
|
|
self._last_override = 0.0
|
|
self._auth_cache = (None, 0.0)
|
|
self._count_ms = 0.0
|
|
self._last_pending = 0.0
|
|
self._wconn = None
|
|
self.metrics = {"detected": 0, "resolved": 0, "fields_written": 0, "detect_latency_ms": []}
|
|
|
|
def _open(self, factory, attempts=3):
|
|
"""Firebird can drop a connection under load ('connection lost'): retry briefly instead of aborting the product."""
|
|
for attempt in range(attempts):
|
|
try:
|
|
return factory()
|
|
except Exception: # noqa: BLE001
|
|
if attempt == attempts - 1:
|
|
raise
|
|
self.sleep(0.3 * (attempt + 1))
|
|
|
|
# ---- feature gate ------------------------------------------------------------------------------------------------------------------------------
|
|
def features(self, max_age=10.0) -> dict:
|
|
if self._features is None or self.clock() - self._features_at > max_age:
|
|
try:
|
|
auth = self.hub.authorizations()
|
|
self._features = (auth or {}).get("features") or {}
|
|
except Exception: # noqa: BLE001 - Hub offline: keep the last known gate, never invent one
|
|
self._features = self._features or {}
|
|
self._features_at = self.clock()
|
|
return self._features
|
|
|
|
def mode(self) -> str:
|
|
f = self.features()
|
|
if not f.get("features_configured"):
|
|
return "WAITING_FEATURE_CONFIGURATION"
|
|
return f.get("enrichment_mode") or "DISABLED"
|
|
|
|
def _fresh_enabled(self) -> bool:
|
|
"""Fail-closed authorization read right before a write (cached ~3 s)."""
|
|
auth, at = self._auth_cache
|
|
if auth is None or self.clock() - at > 3.0:
|
|
try:
|
|
auth = self.hub.authorizations().get("features") or {}
|
|
except Exception: # noqa: BLE001
|
|
return False
|
|
self._auth_cache = (auth, self.clock())
|
|
return bool(auth.get("features_configured")) and auth.get("enrichment_mode") == "ENABLED"
|
|
|
|
# ---- detection ---------------------------------------------------------------------------------------------------------------------------------
|
|
def detect(self) -> list[str]:
|
|
conn = self._open(self.read_conn)
|
|
try:
|
|
if self.store.meta("baseline_done") != "1":
|
|
codes = read_codes(conn)
|
|
self.store.add_known(codes) # historical catalog: known, never sent
|
|
self.store.set_meta("baseline_done", "1")
|
|
self.store.set_meta("baseline_at", datetime.now(timezone.utc).isoformat())
|
|
self.store.set_meta("baseline_count", len(codes))
|
|
self._last_count, self._last_diff = len(codes), self.clock()
|
|
return []
|
|
if self.store.meta("rebaseline") == "1":
|
|
# the function was inactive (DISABLED / WAITING_FEATURE_CONFIGURATION) for a while: whatever appeared meanwhile is NOT "new" for the enrichment
|
|
# (a long-disabled installation must never send a burst of products to the providers when it is switched on again)
|
|
codes = read_codes(conn)
|
|
self.store.add_known(codes)
|
|
self.store.set_meta("rebaseline", "0")
|
|
self._last_count, self._last_diff = len(codes), self.clock()
|
|
return []
|
|
started = self.clock()
|
|
t0 = time.monotonic()
|
|
count = read_count(conn)
|
|
self._count_ms = (time.monotonic() - t0) * 1000
|
|
moved = self._last_count is None or count != self._last_count
|
|
if not moved and self.clock() - self._last_diff < SAFETY_DIFF_SECONDS:
|
|
return []
|
|
codes = read_codes(conn)
|
|
self._last_count, self._last_diff = len(codes), self.clock()
|
|
new = sorted(codes - self.store.known())
|
|
self.store.add_known(new)
|
|
for code in new:
|
|
self.store.enqueue(code, self.clock())
|
|
if new:
|
|
self.metrics["detected"] += len(new)
|
|
self.metrics["detect_latency_ms"].append(int((self.clock() - started) * 1000))
|
|
return new
|
|
finally:
|
|
conn.close()
|
|
|
|
# ---- one product ---------------------------------------------------------------------------------------------------------------------------------
|
|
def process(self, item: dict):
|
|
code, mode = item["code"], self.mode()
|
|
conn = self._open(self.read_conn)
|
|
try:
|
|
rows = read_rows(conn, [code])
|
|
finally:
|
|
conn.close()
|
|
row = rows.get(code)
|
|
if row is None:
|
|
self.store.update(code, state="GONE")
|
|
return
|
|
snapshot = f"{pick_gtin(row)}|{_text(row['PRODUTO'])}"
|
|
if mode not in ("DRY_RUN", "ENABLED"):
|
|
self.store.update(code, state="SKIPPED_FEATURE", snapshot=snapshot)
|
|
return
|
|
attempts = int(item["attempts"] or 0) + 1
|
|
current = {f: row[f] for f in ALLOWED_FIELDS}
|
|
try:
|
|
result = self.hub.resolve_product({"product_code": code, "gtin": pick_gtin(row), "description": _text(row["PRODUTO"]), "current": {k: (None if v is None else str(v)) for k, v in current.items()}})
|
|
except Exception as error: # noqa: BLE001 - Hub offline / slow: the product stays pending, the SGV is unaffected
|
|
delay = RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)]
|
|
self.store.update(code, state="RETRY", attempts=attempts, next_attempt_at=self.clock() + delay, last_state=f"HUB_ERROR:{type(error).__name__}")
|
|
return
|
|
self.metrics["resolved"] += 1
|
|
state = result.get("state")
|
|
if state == "FEATURE_DISABLED":
|
|
self.store.update(code, state="SKIPPED_FEATURE", snapshot=snapshot, last_state=state)
|
|
return
|
|
if state in ("PENDING_PROVIDER_QUOTA", "PENDING_PROVIDERS_OFFLINE"):
|
|
retry_at = result.get("retry_at")
|
|
due = self.clock() + RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)]
|
|
if retry_at:
|
|
try:
|
|
due = max(due, datetime.fromisoformat(retry_at).timestamp())
|
|
except ValueError:
|
|
pass
|
|
self.store.update(code, state="RETRY", attempts=attempts, next_attempt_at=due, last_state=state, snapshot=snapshot)
|
|
return
|
|
fields = result.get("fields") or {}
|
|
reports = []
|
|
written = json.loads(item.get("written") or "{}")
|
|
overridden = json.loads(item.get("overridden") or "{}")
|
|
for field in ALLOWED_FIELDS:
|
|
decision = fields.get(field) or {}
|
|
status, value = decision.get("status"), decision.get("value")
|
|
placeholder = _text(current.get(field)) if is_placeholder(field, current.get(field)) else None
|
|
base = {"field": field, "rule": decision.get("rule"), "kb_status": decision.get("kb_status"), "evidence": decision.get("evidence") or [], "old": placeholder, "new": value}
|
|
if status == "HUMAN_OVERRIDDEN" or overridden.get(field):
|
|
reports.append({**base, "outcome": "HUMAN_OVERRIDDEN", "reason": "HUMAN_OVERRIDDEN", "new": None})
|
|
continue
|
|
if status != "AUTO_FILL_SAFE" or value in (None, ""):
|
|
continue
|
|
if mode == "DRY_RUN":
|
|
reports.append({**base, "outcome": "DRY_RUN", "reason": "DRY_RUN_NO_WRITE"})
|
|
continue
|
|
outcome = self._write_safely(code, field, value, observed=current.get(field))
|
|
reports.append({**base, "outcome": outcome["outcome"], "reason": outcome.get("reason")})
|
|
if outcome["outcome"] == "APPLIED":
|
|
written[field] = value
|
|
self.metrics["fields_written"] += 1
|
|
self.store.update(code, written=written) # remembered at once: a crash right after a write still leaves it protected/tracked
|
|
self._close_write()
|
|
self.store.update(code, state="DRY_RUN_DONE" if mode == "DRY_RUN" else "DONE", attempts=attempts, snapshot=snapshot, written=written, last_state=state, next_attempt_at=None)
|
|
if reports:
|
|
try:
|
|
self.hub.report_enrichment({"product_code": code, "fields": reports, "dataset_versions": result.get("dataset_versions") or {}})
|
|
except Exception: # noqa: BLE001 - audit delivery is retried on the next cycle via last_state
|
|
self.store.update(code, last_state="AUDIT_PENDING")
|
|
if written and self.create_reprocess is not None and mode == "ENABLED":
|
|
try:
|
|
self.create_reprocess([code]) # cClassTrib is decided by the existing RTC gates, never here
|
|
except Exception: # noqa: BLE001
|
|
logger.warning("reprocess request for %s failed", code)
|
|
|
|
def _write_safely(self, code, field, value, observed=None) -> dict:
|
|
"""Per field: fresh central authorization (fail closed) + guarded typed UPDATE. write_field re-reads the field INSIDE its own transaction and refuses when anything is there."""
|
|
if not self._fresh_enabled():
|
|
return {"outcome": "ABORTED", "reason": "AUTHORIZATION_CHANGED"}
|
|
try:
|
|
if self._wconn is None:
|
|
self._wconn = self._open(self.write_conn)
|
|
return write_field(self._wconn, code, field, value, observed=observed, guard_observed=True)
|
|
except Exception as error: # noqa: BLE001 - one field failing never breaks the others or the SGV
|
|
self._close_write()
|
|
return {"outcome": "FAILED", "reason": type(error).__name__}
|
|
|
|
def _close_write(self):
|
|
conn, self._wconn = self._wconn, None
|
|
if conn is not None:
|
|
try:
|
|
conn.close()
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
|
|
def reevaluate_flagged(self) -> list[str]:
|
|
"""'Reavaliar' in the panel, Reprocessar/Sincronizar and KB decisions: re-run the normal resolve for tracked, not-yet-enriched products (KB/cache first, the Hub decides about providers)."""
|
|
try:
|
|
codes = self.hub.pending_enrichment()
|
|
except Exception: # noqa: BLE001 - Hub offline: try again next cycle
|
|
return []
|
|
done = []
|
|
for code in codes[:25]:
|
|
item = self.store.get(code)
|
|
if item is None:
|
|
self.store.enqueue(code, self.clock())
|
|
item = self.store.get(code)
|
|
try:
|
|
self.process(item)
|
|
except Exception as error: # noqa: BLE001
|
|
self._close_write()
|
|
logger.warning("re-evaluation of %s failed: %s", code, type(error).__name__)
|
|
done.append(code)
|
|
return done
|
|
|
|
# ---- human override watch --------------------------------------------------------------------------------------------------------------------------
|
|
def check_overrides(self):
|
|
tracked = self.store.tracked_written()
|
|
if not tracked:
|
|
return
|
|
conn = self._open(self.read_conn)
|
|
try:
|
|
rows = read_rows(conn, [t["code"] for t in tracked])
|
|
finally:
|
|
conn.close()
|
|
for item in tracked:
|
|
row = rows.get(item["code"])
|
|
written, overridden = json.loads(item["written"]), json.loads(item["overridden"] or "{}")
|
|
new = []
|
|
for field, value in written.items():
|
|
if overridden.get(field) or row is None:
|
|
continue
|
|
if not _same_value(field, row[field], value): # different value OR blanked/deleted: a person decided, forever
|
|
overridden[field] = True
|
|
new.append({"field": field, "outcome": "HUMAN_OVERRIDDEN", "reason": "VALUE_CHANGED_AFTER_WRITE", "old": value, "new": None})
|
|
if new:
|
|
self.store.update(item["code"], overridden=overridden)
|
|
try:
|
|
self.hub.report_enrichment({"product_code": item["code"], "fields": new})
|
|
except Exception: # noqa: BLE001 - local protection is already in place; the Hub copy follows on the next cycle
|
|
logger.warning("override report for %s failed", item["code"])
|
|
|
|
# ---- loop ----------------------------------------------------------------------------------------------------------------------------------------
|
|
def step(self):
|
|
"""One iteration; returns the list of processed product codes."""
|
|
mode = self.mode()
|
|
if mode not in ("DRY_RUN", "ENABLED"):
|
|
if self.store.meta("baseline_done") == "1":
|
|
self.store.set_meta("rebaseline", "1")
|
|
return [] # WAITING_FEATURE_CONFIGURATION / DISABLED: zero Firebird activity
|
|
self.detect()
|
|
done = []
|
|
for item in self.store.due(self.clock()):
|
|
try:
|
|
self.process(item)
|
|
except Exception as error: # noqa: BLE001 - keep the product retryable, never lose the batch
|
|
self._close_write()
|
|
attempts = int(item["attempts"] or 0) + 1
|
|
self.store.update(item["code"], state="RETRY", attempts=attempts, next_attempt_at=self.clock() + RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)], last_state=f"ERROR:{type(error).__name__}")
|
|
logger.warning("enrichment of %s failed: %s", item["code"], type(error).__name__)
|
|
done.append(item["code"])
|
|
for item in self.store.watching(self.clock()): # operator completes the record (EAN typed a moment later): re-resolve once per change
|
|
rows = None
|
|
conn = self._open(self.read_conn)
|
|
try:
|
|
rows = read_rows(conn, [item["code"]])
|
|
finally:
|
|
conn.close()
|
|
row = (rows or {}).get(item["code"])
|
|
if row and f"{pick_gtin(row)}|{_text(row['PRODUTO'])}" != item.get("snapshot"):
|
|
self.process(item)
|
|
done.append(item["code"])
|
|
if self.clock() - self._last_pending >= PENDING_POLL_SECONDS:
|
|
self._last_pending = self.clock()
|
|
done += self.reevaluate_flagged()
|
|
if self.clock() - self._last_override >= OVERRIDE_CHECK_SECONDS:
|
|
self._last_override = self.clock()
|
|
self.check_overrides()
|
|
return done
|
|
|
|
def poll_seconds(self) -> float:
|
|
"""Adaptive: the SGV is never loaded by more than ~2.5% of one query-time. 3 s on small catalogs, longer when COUNT(*) is slow (huge catalogs); never above 30 s."""
|
|
return min(30.0, max(POLL_SECONDS, 40 * self._count_ms / 1000))
|
|
|
|
def run_forever(self, stopper):
|
|
while not stopper.stop:
|
|
try:
|
|
self.step()
|
|
except Exception: # noqa: BLE001 - the enrichment thread must never die or affect the SGV
|
|
logger.exception("enrichment step failed")
|
|
self.sleep(self.poll_seconds())
|