feat: enrichment Hub API, feature flags UI/gating, NCM refresher, Agent enrichment worker with tests
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
Claude Sonnet 5
@@ -36,6 +36,7 @@ from gti_fiscal_agent.pipeline import run_pipeline # noqa: E402
|
||||
from gti_fiscal_agent.runner import REFERENCE_SCHEMA_V1, product_fingerprint, to_hub_product # noqa: E402
|
||||
from central.app.operational_state import classify_product_state, summarize_operational_states # noqa: E402
|
||||
from gti_fiscal_agent.state import AgentState # noqa: E402
|
||||
from gti_fiscal_agent import enrichment as _enrichment # noqa: E402
|
||||
|
||||
_MAX_BACKOFF_SECONDS = 60
|
||||
SYNC_JOB_KINDS = ("INITIAL_SYNC", "SYNC_INCREMENTAL", "REPROCESS_ALL") # read-only ERP jobs that refresh coverage
|
||||
@@ -409,6 +410,17 @@ def resolve_installation_id(config, env, state, client):
|
||||
return state.installation_id(str(installation_id))
|
||||
|
||||
|
||||
def _start_enrichment(config, env, client, stopper):
|
||||
"""New-product enrichment worker (own thread, own SQLite file). It gates ITSELF on the Hub's per-installation features: DISABLED / not configured = zero Firebird activity."""
|
||||
if not hasattr(client, "resolve_product"):
|
||||
return None
|
||||
store = _enrichment.EnrichmentStore(Path(config.state_db).with_name("enrichment_state.db"))
|
||||
worker = _enrichment.EnrichmentWorker(hub=client, store=store, read_conn=lambda: _firebird_reader._connect(env, True), write_conn=lambda: open_write_connection(env, config.database_path),
|
||||
create_reprocess=lambda codes: client.create_job("REPROCESS_PRODUCTS", {"codigos": list(codes), "reason": "ENRICHMENT"}))
|
||||
threading.Thread(target=worker.run_forever, args=(stopper,), name="product-enrichment", daemon=True).start()
|
||||
return worker
|
||||
|
||||
|
||||
def run(config, env, *, max_iterations=None, stopper=None):
|
||||
"""`stopper` defaults to a signal-based GracefulStop (SIGINT/SIGTERM), which
|
||||
only works from the main thread. A host that manages its own lifecycle
|
||||
@@ -420,6 +432,8 @@ def run(config, env, *, max_iterations=None, stopper=None):
|
||||
client = HubClient(config.hub_url, config.agent_token)
|
||||
installation_id = resolve_installation_id(config, env, state, client)
|
||||
stopper = stopper if stopper is not None else GracefulStop()
|
||||
if max_iterations is None:
|
||||
_start_enrichment(config, env, client, stopper)
|
||||
|
||||
backoff = 1
|
||||
was_down = False
|
||||
@@ -436,7 +450,7 @@ def run(config, env, *, max_iterations=None, stopper=None):
|
||||
"agent_version": config.agent_version, "last_job_id": last_job_id, "display_name": env.get("AGENT_DISPLAY_NAME", ""),
|
||||
"customer_code": env.get("CUSTOMER_CODE"), "hostname": env.get("AGENT_HOSTNAME") or os.environ.get("COMPUTERNAME"),
|
||||
"windows_version": env.get("WINDOWS_VERSION"), "company_code": config.company_id, "company_uf": config.company_uf,
|
||||
"database_path": config.database_path, "capabilities": {"heartbeat": True, "jobs": True, "local_logs": True, "RTC_CCLASSTRIB_WRITE_V1": True},
|
||||
"database_path": config.database_path, "capabilities": {"heartbeat": True, "jobs": True, "local_logs": True, "RTC_CCLASSTRIB_WRITE_V1": True, "PRODUCT_ENRICHMENT_WRITE_V1": True},
|
||||
"fiscal_engine_version": env.get("FISCAL_ENGINE_VERSION", config.agent_version), "dataset_versions": {},
|
||||
}
|
||||
if hasattr(client, "heartbeat_metadata"):
|
||||
|
||||
@@ -41,6 +41,13 @@ class HubClient:
|
||||
response.raise_for_status(); return response.json()
|
||||
def identity(self): return self.get_json("/v1/agents/me/identity")
|
||||
def authorizations(self): return self.get_json("/v1/agents/me/authorizations")
|
||||
def resolve_product(self, payload):
|
||||
"""Asks the Hub to resolve a NEW product (the Agent never contacts external providers). Bounded: the Hub answers within its own time budget."""
|
||||
response = httpx.post(f"{self.url}/v1/products/resolve", json=payload, headers=self._headers(), timeout=20)
|
||||
response.raise_for_status(); return response.json()
|
||||
def report_enrichment(self, payload):
|
||||
response = _resilient(lambda: httpx.post(f"{self.url}/v1/products/enrichment/applied", json=payload, headers=self._headers(), timeout=30))
|
||||
response.raise_for_status(); return response.json()
|
||||
def health(self):
|
||||
response = httpx.get(f"{self.url}/health", timeout=10)
|
||||
return response.status_code == 200
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
"""New-product enrichment on the Agent: detect NEW rows of C000025, ask the Hub, write ONLY safe empty fields.
|
||||
|
||||
Safety model (defense in depth; nothing here is switchable locally - the Hub owns the per-installation feature flags):
|
||||
- Feature gate: nothing touches Firebird unless the Hub says features_configured AND enrichment_mode != DISABLED. DRY_RUN never opens a write connection.
|
||||
- Scope: only products that appear AFTER the baseline (taken when the function is first active). The historical catalog is never sent anywhere.
|
||||
- No trigger, no table, no procedure in the SGV database: detection is one `SELECT COUNT(*)` per poll (+ a code diff only when the count moved, and every 60 s as a safety net).
|
||||
- Typed capability PRODUCT_ENRICHMENT_WRITE_V1: ALLOWED_FIELDS is an explicit allowlist on C000025; the only statement is a fixed UPDATE ... WHERE CODIGO=? AND (field IS NULL OR TRIM(field)=''),
|
||||
one short transaction per field, followed by a re-read. COD_ORIGEM_MERCADORIA and cClassTrib are NOT in the list (cClassTrib stays with the existing RTC gates).
|
||||
- Right before EVERY field write: re-read the row, re-check the Hub authorization (fail closed), skip when a human already touched the field.
|
||||
- Human overrides: after writing, the Agent watches the value; anything different from what it wrote (including a deleted value) marks the field HUMAN_OVERRIDDEN forever, is reported to the Hub
|
||||
and is never written again by the enrichment (nor re-suggested by the Hub).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_agent.enrichment")
|
||||
|
||||
CAPABILITY = "PRODUCT_ENRICHMENT_WRITE_V1"
|
||||
ALLOWED_FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI") # explicit allowlist; anything else raises before touching SQL
|
||||
READ_FIELDS = ("CODIGO", "PRODUTO", "GTIN") + ALLOWED_FIELDS
|
||||
POLL_SECONDS = 3.0
|
||||
SAFETY_DIFF_SECONDS = 60.0
|
||||
OVERRIDE_CHECK_SECONDS = 20.0
|
||||
WATCH_WINDOW_SECONDS = 120.0 # a fresh product is re-read for a while: the operator may add the EAN seconds after the first save
|
||||
RETRY_BACKOFF = (30, 60, 120, 300, 600, 1800)
|
||||
|
||||
|
||||
def _now() -> float:
|
||||
return time.time()
|
||||
|
||||
|
||||
def _text(value) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
class EnrichmentStore:
|
||||
"""Own SQLite connection (the worker runs in its own thread)."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.db = sqlite3.connect(str(path), timeout=10, check_same_thread=False)
|
||||
self.db.row_factory = sqlite3.Row
|
||||
self.db.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS enrich_meta(k TEXT PRIMARY KEY, v TEXT);
|
||||
CREATE TABLE IF NOT EXISTS enrich_known(code TEXT PRIMARY KEY);
|
||||
CREATE TABLE IF NOT EXISTS enrich_products(
|
||||
code TEXT PRIMARY KEY, state TEXT NOT NULL, detected_at REAL NOT NULL, next_attempt_at REAL, attempts INTEGER NOT NULL DEFAULT 0,
|
||||
snapshot TEXT, written TEXT NOT NULL DEFAULT '{}', overridden TEXT NOT NULL DEFAULT '{}', last_state TEXT);
|
||||
""")
|
||||
self.db.commit()
|
||||
|
||||
def meta(self, key, default=None):
|
||||
row = self.db.execute("SELECT v FROM enrich_meta WHERE k=?", (key,)).fetchone()
|
||||
return row["v"] if row else default
|
||||
|
||||
def set_meta(self, key, value):
|
||||
self.db.execute("INSERT INTO enrich_meta(k,v) VALUES(?,?) ON CONFLICT(k) DO UPDATE SET v=excluded.v", (key, str(value)))
|
||||
self.db.commit()
|
||||
|
||||
def known(self) -> set[str]:
|
||||
return {r["code"] for r in self.db.execute("SELECT code FROM enrich_known")}
|
||||
|
||||
def add_known(self, codes):
|
||||
self.db.executemany("INSERT OR IGNORE INTO enrich_known(code) VALUES(?)", [(c,) for c in codes])
|
||||
self.db.commit()
|
||||
|
||||
def enqueue(self, code, now=None):
|
||||
now = _now() if now is None else now
|
||||
self.db.execute("INSERT OR IGNORE INTO enrich_products(code,state,detected_at,next_attempt_at) VALUES(?,?,?,?)", (code, "PENDING", now, now))
|
||||
self.db.commit()
|
||||
|
||||
def get(self, code):
|
||||
row = self.db.execute("SELECT * FROM enrich_products WHERE code=?", (code,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def due(self, now):
|
||||
return [dict(r) for r in self.db.execute("SELECT * FROM enrich_products WHERE state IN ('PENDING','RETRY') AND next_attempt_at<=? ORDER BY detected_at LIMIT 20", (now,))]
|
||||
|
||||
def watching(self, now):
|
||||
return [dict(r) for r in self.db.execute("SELECT * FROM enrich_products WHERE state IN ('DONE','DRY_RUN_DONE') AND detected_at>=?", (now - WATCH_WINDOW_SECONDS,))]
|
||||
|
||||
def tracked_written(self):
|
||||
return [dict(r) for r in self.db.execute("SELECT * FROM enrich_products WHERE written != '{}'")]
|
||||
|
||||
def update(self, code, **values):
|
||||
for key in ("written", "overridden"):
|
||||
if key in values and not isinstance(values[key], str):
|
||||
values[key] = json.dumps(values[key])
|
||||
sets = ",".join(f"{k}=?" for k in values)
|
||||
self.db.execute(f"UPDATE enrich_products SET {sets} WHERE code=?", (*values.values(), code))
|
||||
self.db.commit()
|
||||
|
||||
def is_overridden(self, code, field) -> bool:
|
||||
row = self.get(code)
|
||||
return bool(row and json.loads(row["overridden"] or "{}").get(field))
|
||||
|
||||
|
||||
def is_human_overridden(store_path, code, field) -> bool:
|
||||
"""Used by every automatic writer of the Agent (the Fiscal Corrector included) before touching a field the enrichment tracks."""
|
||||
try:
|
||||
return EnrichmentStore(store_path).is_overridden(str(code), field)
|
||||
except Exception: # noqa: BLE001 - no state means nothing is protected yet
|
||||
return False
|
||||
|
||||
|
||||
# ---- Firebird access (injected: `read_conn()` returns a READ-ONLY connection, `write_conn()` a write-capable one, only ever opened by ENABLED mode) ----------------------------------------
|
||||
def _digits(value) -> str:
|
||||
return "".join(ch for ch in _text(value) if ch.isdigit())
|
||||
|
||||
|
||||
def read_codes(conn) -> set[str]:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT CODIGO FROM C000025")
|
||||
codes = {_text(r[0]) for r in cur.fetchall()}
|
||||
conn.commit()
|
||||
return codes
|
||||
|
||||
|
||||
def read_count(conn) -> int:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT COUNT(*) FROM C000025")
|
||||
n = int(cur.fetchone()[0])
|
||||
conn.commit()
|
||||
return n
|
||||
|
||||
|
||||
def read_rows(conn, codes) -> dict[str, dict]:
|
||||
out = {}
|
||||
codes = list(codes)
|
||||
for start in range(0, len(codes), 100):
|
||||
chunk = codes[start:start + 100]
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {','.join(READ_FIELDS)} FROM C000025 WHERE CODIGO IN ({','.join('?' for _ in chunk)})", tuple(chunk))
|
||||
for row in cur.fetchall():
|
||||
rec = dict(zip(READ_FIELDS, row))
|
||||
out[_text(rec["CODIGO"])] = rec
|
||||
conn.commit()
|
||||
return out
|
||||
|
||||
|
||||
def write_field(conn, code: str, field: str, value) -> dict:
|
||||
"""PRODUCT_ENRICHMENT_WRITE_V1: one field, one short transaction, only if the field is still empty. Returns {outcome, reason?}."""
|
||||
if field not in ALLOWED_FIELDS:
|
||||
raise ValueError(f"field {field!r} is outside the PRODUCT_ENRICHMENT_WRITE_V1 allowlist")
|
||||
if field == "IPI":
|
||||
value = float(value)
|
||||
else:
|
||||
value = _digits(value)
|
||||
if not value or (field == "CLASSIFICACAO_FISCAL" and len(value) != 8) or (field == "CEST" and len(value) != 7):
|
||||
return {"outcome": "ABORTED", "reason": "VALUE_SHAPE_INVALID"}
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
rows = cur.fetchall()
|
||||
if len(rows) != 1:
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND"}
|
||||
if _text(rows[0][0]):
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"}
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND ({field} IS NULL OR TRIM({field}) = '')", (value, code))
|
||||
if cur.rowcount != 1:
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "UPDATE_ROWCOUNT_NOT_ONE"}
|
||||
conn.commit()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
after = cur.fetchone()[0]
|
||||
conn.commit()
|
||||
same = (float(after) == value) if field == "IPI" and after is not None else _text(after) == value
|
||||
if not same:
|
||||
return {"outcome": "FAILED", "reason": "POST_WRITE_VERIFICATION_FAILED"}
|
||||
return {"outcome": "APPLIED", "value": value, "rowcount": 1}
|
||||
except Exception as error: # noqa: BLE001
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return {"outcome": "FAILED", "reason": type(error).__name__}
|
||||
|
||||
|
||||
def _same_value(field, current, written) -> bool:
|
||||
if field == "IPI":
|
||||
try:
|
||||
return current is not None and _text(current) != "" and abs(float(current) - float(written)) < 1e-9
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return _text(current) == _text(written) and _text(current) != ""
|
||||
|
||||
|
||||
class EnrichmentWorker:
|
||||
def __init__(self, *, hub, store: EnrichmentStore, read_conn, write_conn, create_reprocess=None, clock=_now, sleep=time.sleep):
|
||||
self.hub, self.store, self.read_conn, self.write_conn = hub, store, read_conn, write_conn
|
||||
self.create_reprocess, self.clock, self.sleep = create_reprocess, clock, sleep
|
||||
self._features = None
|
||||
self._features_at = 0.0
|
||||
self._last_count = None
|
||||
self._last_diff = 0.0
|
||||
self._last_override = 0.0
|
||||
self._auth_cache = (None, 0.0)
|
||||
self.metrics = {"detected": 0, "resolved": 0, "fields_written": 0, "detect_latency_ms": []}
|
||||
|
||||
# ---- feature gate ------------------------------------------------------------------------------------------------------------------------------
|
||||
def features(self, max_age=10.0) -> dict:
|
||||
if self._features is None or self.clock() - self._features_at > max_age:
|
||||
try:
|
||||
auth = self.hub.authorizations()
|
||||
self._features = (auth or {}).get("features") or {}
|
||||
except Exception: # noqa: BLE001 - Hub offline: keep the last known gate, never invent one
|
||||
self._features = self._features or {}
|
||||
self._features_at = self.clock()
|
||||
return self._features
|
||||
|
||||
def mode(self) -> str:
|
||||
f = self.features()
|
||||
if not f.get("features_configured"):
|
||||
return "WAITING_FEATURE_CONFIGURATION"
|
||||
return f.get("enrichment_mode") or "DISABLED"
|
||||
|
||||
def _fresh_enabled(self) -> bool:
|
||||
"""Fail-closed authorization read right before a write (cached ~3 s)."""
|
||||
auth, at = self._auth_cache
|
||||
if auth is None or self.clock() - at > 3.0:
|
||||
try:
|
||||
auth = self.hub.authorizations().get("features") or {}
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
self._auth_cache = (auth, self.clock())
|
||||
return bool(auth.get("features_configured")) and auth.get("enrichment_mode") == "ENABLED"
|
||||
|
||||
# ---- detection ---------------------------------------------------------------------------------------------------------------------------------
|
||||
def detect(self) -> list[str]:
|
||||
conn = self.read_conn()
|
||||
try:
|
||||
if self.store.meta("baseline_done") != "1":
|
||||
codes = read_codes(conn)
|
||||
self.store.add_known(codes) # historical catalog: known, never sent
|
||||
self.store.set_meta("baseline_done", "1")
|
||||
self.store.set_meta("baseline_at", datetime.now(timezone.utc).isoformat())
|
||||
self.store.set_meta("baseline_count", len(codes))
|
||||
self._last_count, self._last_diff = len(codes), self.clock()
|
||||
return []
|
||||
started = self.clock()
|
||||
count = read_count(conn)
|
||||
moved = self._last_count is None or count != self._last_count
|
||||
if not moved and self.clock() - self._last_diff < SAFETY_DIFF_SECONDS:
|
||||
return []
|
||||
codes = read_codes(conn)
|
||||
self._last_count, self._last_diff = len(codes), self.clock()
|
||||
new = sorted(codes - self.store.known())
|
||||
self.store.add_known(new)
|
||||
for code in new:
|
||||
self.store.enqueue(code, self.clock())
|
||||
if new:
|
||||
self.metrics["detected"] += len(new)
|
||||
self.metrics["detect_latency_ms"].append(int((self.clock() - started) * 1000))
|
||||
return new
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# ---- one product ---------------------------------------------------------------------------------------------------------------------------------
|
||||
def process(self, item: dict):
|
||||
code, mode = item["code"], self.mode()
|
||||
conn = self.read_conn()
|
||||
try:
|
||||
rows = read_rows(conn, [code])
|
||||
finally:
|
||||
conn.close()
|
||||
row = rows.get(code)
|
||||
if row is None:
|
||||
self.store.update(code, state="GONE")
|
||||
return
|
||||
snapshot = f"{_text(row['GTIN'])}|{_text(row['PRODUTO'])}"
|
||||
if mode not in ("DRY_RUN", "ENABLED"):
|
||||
self.store.update(code, state="SKIPPED_FEATURE", snapshot=snapshot)
|
||||
return
|
||||
attempts = int(item["attempts"] or 0) + 1
|
||||
current = {f: row[f] for f in ALLOWED_FIELDS}
|
||||
try:
|
||||
result = self.hub.resolve_product({"product_code": code, "gtin": _text(row["GTIN"]), "description": _text(row["PRODUTO"]), "current": {k: (None if v is None else str(v)) for k, v in current.items()}})
|
||||
except Exception as error: # noqa: BLE001 - Hub offline / slow: the product stays pending, the SGV is unaffected
|
||||
delay = RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)]
|
||||
self.store.update(code, state="RETRY", attempts=attempts, next_attempt_at=self.clock() + delay, last_state=f"HUB_ERROR:{type(error).__name__}")
|
||||
return
|
||||
self.metrics["resolved"] += 1
|
||||
state = result.get("state")
|
||||
if state == "FEATURE_DISABLED":
|
||||
self.store.update(code, state="SKIPPED_FEATURE", snapshot=snapshot, last_state=state)
|
||||
return
|
||||
if state in ("PENDING_PROVIDER_QUOTA", "PENDING_PROVIDERS_OFFLINE"):
|
||||
retry_at = result.get("retry_at")
|
||||
due = self.clock() + RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)]
|
||||
if retry_at:
|
||||
try:
|
||||
due = max(due, datetime.fromisoformat(retry_at).timestamp())
|
||||
except ValueError:
|
||||
pass
|
||||
self.store.update(code, state="RETRY", attempts=attempts, next_attempt_at=due, last_state=state, snapshot=snapshot)
|
||||
return
|
||||
fields = result.get("fields") or {}
|
||||
reports = []
|
||||
written = json.loads(item.get("written") or "{}")
|
||||
overridden = json.loads(item.get("overridden") or "{}")
|
||||
for field in ALLOWED_FIELDS:
|
||||
decision = fields.get(field) or {}
|
||||
status, value = decision.get("status"), decision.get("value")
|
||||
base = {"field": field, "rule": decision.get("rule"), "kb_status": decision.get("kb_status"), "evidence": decision.get("evidence") or [], "old": None, "new": value}
|
||||
if status == "HUMAN_OVERRIDDEN" or overridden.get(field):
|
||||
reports.append({**base, "outcome": "HUMAN_OVERRIDDEN", "reason": "HUMAN_OVERRIDDEN", "new": None})
|
||||
continue
|
||||
if status != "AUTO_FILL_SAFE" or value in (None, ""):
|
||||
continue
|
||||
if mode == "DRY_RUN":
|
||||
reports.append({**base, "outcome": "DRY_RUN", "reason": "DRY_RUN_NO_WRITE"})
|
||||
continue
|
||||
outcome = self._write_safely(code, field, value)
|
||||
reports.append({**base, "outcome": outcome["outcome"], "reason": outcome.get("reason")})
|
||||
if outcome["outcome"] == "APPLIED":
|
||||
written[field] = value
|
||||
self.metrics["fields_written"] += 1
|
||||
self.store.update(code, state="DRY_RUN_DONE" if mode == "DRY_RUN" else "DONE", attempts=attempts, snapshot=snapshot, written=written, last_state=state, next_attempt_at=None)
|
||||
if reports:
|
||||
try:
|
||||
self.hub.report_enrichment({"product_code": code, "fields": reports, "dataset_versions": result.get("dataset_versions") or {}})
|
||||
except Exception: # noqa: BLE001 - audit delivery is retried on the next cycle via last_state
|
||||
self.store.update(code, last_state="AUDIT_PENDING")
|
||||
if written and self.create_reprocess is not None and mode == "ENABLED":
|
||||
try:
|
||||
self.create_reprocess([code]) # cClassTrib is decided by the existing RTC gates, never here
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("reprocess request for %s failed", code)
|
||||
|
||||
def _write_safely(self, code, field, value) -> dict:
|
||||
if not self._fresh_enabled():
|
||||
return {"outcome": "ABORTED", "reason": "AUTHORIZATION_CHANGED"}
|
||||
conn = self.read_conn()
|
||||
try:
|
||||
fresh = read_rows(conn, [code]).get(code)
|
||||
finally:
|
||||
conn.close()
|
||||
if fresh is None:
|
||||
return {"outcome": "ABORTED", "reason": "PRODUCT_NOT_FOUND"}
|
||||
if _text(fresh[field]):
|
||||
return {"outcome": "ABORTED", "reason": "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"}
|
||||
wconn = self.write_conn()
|
||||
try:
|
||||
return write_field(wconn, code, field, value)
|
||||
finally:
|
||||
wconn.close()
|
||||
|
||||
# ---- human override watch --------------------------------------------------------------------------------------------------------------------------
|
||||
def check_overrides(self):
|
||||
tracked = self.store.tracked_written()
|
||||
if not tracked:
|
||||
return
|
||||
conn = self.read_conn()
|
||||
try:
|
||||
rows = read_rows(conn, [t["code"] for t in tracked])
|
||||
finally:
|
||||
conn.close()
|
||||
for item in tracked:
|
||||
row = rows.get(item["code"])
|
||||
written, overridden = json.loads(item["written"]), json.loads(item["overridden"] or "{}")
|
||||
new = []
|
||||
for field, value in written.items():
|
||||
if overridden.get(field) or row is None:
|
||||
continue
|
||||
if not _same_value(field, row[field], value): # different value OR blanked/deleted: a person decided, forever
|
||||
overridden[field] = True
|
||||
new.append({"field": field, "outcome": "HUMAN_OVERRIDDEN", "reason": "VALUE_CHANGED_AFTER_WRITE", "old": value, "new": None})
|
||||
if new:
|
||||
self.store.update(item["code"], overridden=overridden)
|
||||
try:
|
||||
self.hub.report_enrichment({"product_code": item["code"], "fields": new})
|
||||
except Exception: # noqa: BLE001 - local protection is already in place; the Hub copy follows on the next cycle
|
||||
logger.warning("override report for %s failed", item["code"])
|
||||
|
||||
# ---- loop ----------------------------------------------------------------------------------------------------------------------------------------
|
||||
def step(self):
|
||||
"""One iteration; returns the list of processed product codes."""
|
||||
mode = self.mode()
|
||||
if mode not in ("DRY_RUN", "ENABLED"):
|
||||
return [] # WAITING_FEATURE_CONFIGURATION / DISABLED: zero Firebird activity
|
||||
self.detect()
|
||||
done = []
|
||||
for item in self.store.due(self.clock()):
|
||||
self.process(item)
|
||||
done.append(item["code"])
|
||||
for item in self.store.watching(self.clock()): # operator completes the record (EAN typed a moment later): re-resolve once per change
|
||||
rows = None
|
||||
conn = self.read_conn()
|
||||
try:
|
||||
rows = read_rows(conn, [item["code"]])
|
||||
finally:
|
||||
conn.close()
|
||||
row = (rows or {}).get(item["code"])
|
||||
if row and f"{_text(row['GTIN'])}|{_text(row['PRODUTO'])}" != item.get("snapshot"):
|
||||
self.process(item)
|
||||
done.append(item["code"])
|
||||
if self.clock() - self._last_override >= OVERRIDE_CHECK_SECONDS:
|
||||
self._last_override = self.clock()
|
||||
self.check_overrides()
|
||||
return done
|
||||
|
||||
def run_forever(self, stopper):
|
||||
while not stopper.stop:
|
||||
try:
|
||||
self.step()
|
||||
except Exception: # noqa: BLE001 - the enrichment thread must never die or affect the SGV
|
||||
logger.exception("enrichment step failed")
|
||||
self.sleep(POLL_SECONDS)
|
||||
@@ -0,0 +1,201 @@
|
||||
"""Agent enrichment: baseline/new-product detection, feature gating, typed write, second run = 0 writes, human override (incl. deleted value)."""
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent import enrichment as E # noqa: E402
|
||||
|
||||
|
||||
class Conn:
|
||||
"""sqlite stand-in for the Firebird connection; close() is a no-op so the worker can 'open' it repeatedly."""
|
||||
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
def cursor(self):
|
||||
return self.db.cursor()
|
||||
|
||||
def commit(self):
|
||||
self.db.commit()
|
||||
|
||||
def rollback(self):
|
||||
self.db.rollback()
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class Hub:
|
||||
def __init__(self, features, decisions=None, state="EXACT_GTIN_VERIFIED"):
|
||||
self.features = features
|
||||
self.decisions = decisions or {}
|
||||
self.state = state
|
||||
self.resolve_calls, self.reports, self.overridden = [], [], {}
|
||||
|
||||
def authorizations(self):
|
||||
return {"features": dict(self.features)}
|
||||
|
||||
def resolve_product(self, payload):
|
||||
self.resolve_calls.append(payload)
|
||||
fields = {k: dict(v) for k, v in self.decisions.items()}
|
||||
for f in list(fields):
|
||||
if f in self.overridden.get(payload["product_code"], set()):
|
||||
fields[f] = {"status": "HUMAN_OVERRIDDEN", "value": None}
|
||||
elif payload["current"].get(f):
|
||||
fields[f] = {"status": "ALREADY_SET", "value": None}
|
||||
return {"state": self.state, "fields": fields, "dataset_versions": {"NCM": {"sha256": "abc"}}}
|
||||
|
||||
def report_enrichment(self, payload):
|
||||
self.reports.append(payload)
|
||||
for item in payload["fields"]:
|
||||
if item["outcome"] == "HUMAN_OVERRIDDEN":
|
||||
self.overridden.setdefault(payload["product_code"], set()).add(item["field"])
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
ENABLED = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}
|
||||
DECISIONS = {"CLASSIFICACAO_FISCAL": {"status": "AUTO_FILL_SAFE", "value": "22021000", "rule": "NCM_KB_VERIFIED", "kb_status": "VERIFIED_KB"},
|
||||
"CEST": {"status": "AUTO_FILL_SAFE", "value": "0300700", "rule": "CEST_OFFICIAL_UNIQUE_MAPPING"},
|
||||
"IPI": {"status": "AUTO_FILL_SAFE", "value": "0.0", "rule": "IPI_TIPI_NUMERIC"}}
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def env(tmp_path):
|
||||
db = sqlite3.connect(":memory:", check_same_thread=False)
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT)")
|
||||
db.executemany("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", [(f"H{i}", f"HIST {i}", "", "keep") for i in range(50)])
|
||||
db.commit()
|
||||
hub = Hub(dict(ENABLED), DECISIONS)
|
||||
store = E.EnrichmentStore(tmp_path / "e.db")
|
||||
now = [1000.0]
|
||||
reproc = []
|
||||
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=reproc.append, clock=lambda: now[0], sleep=lambda s: None)
|
||||
return db, hub, store, worker, now, reproc
|
||||
|
||||
|
||||
def add(db, code, name="REFRIGERANTE COLA LATA 350ML", gtin="7891000100103"):
|
||||
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", (code, name, gtin, "keep"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def row(db, code):
|
||||
return db.execute("SELECT CLASSIFICACAO_FISCAL,CEST,IPI,OTHER FROM C000025 WHERE CODIGO=?", (code,)).fetchone()
|
||||
|
||||
|
||||
def test_baseline_never_sends_historical_products(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
assert worker.step() == [] # baseline
|
||||
now[0] += 5
|
||||
assert worker.step() == [] and hub.resolve_calls == []
|
||||
assert len(store.known()) == 50
|
||||
|
||||
|
||||
def test_new_product_is_enriched_only_in_empty_fields_then_second_run_zero_writes(env):
|
||||
db, hub, store, worker, now, reproc = env
|
||||
worker.step()
|
||||
add(db, "N1")
|
||||
now[0] += 5
|
||||
assert worker.step() == ["N1"]
|
||||
assert row(db, "N1") == ("22021000", "0300700", 0.0, "keep")
|
||||
assert reproc == [["N1"]]
|
||||
assert row(db, "H1") == (None, None, None, "keep") # historical untouched
|
||||
assert [r["outcome"] for r in hub.reports[0]["fields"]] == ["APPLIED"] * 3
|
||||
now[0] += 5
|
||||
before = db.total_changes
|
||||
worker.step()
|
||||
assert db.total_changes == before and len(hub.resolve_calls) == 1 # nothing re-processed, 0 writes
|
||||
|
||||
|
||||
def test_existing_value_is_never_overwritten(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N2")
|
||||
db.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N2'")
|
||||
db.commit()
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "N2")[0] == "19059090" and row(db, "N2")[1] == "0300700"
|
||||
|
||||
|
||||
def test_dry_run_and_disabled_and_waiting_do_not_write_or_touch_firebird(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N3")
|
||||
for features in ({"features_configured": False}, {"features_configured": True, "enrichment_mode": "DISABLED"}):
|
||||
hub.features = features
|
||||
worker._features_at = -999
|
||||
assert worker.step() == [] and hub.resolve_calls == []
|
||||
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
|
||||
worker._features_at = -999
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "N3") == (None, None, None, "keep") and hub.reports[0]["fields"][0]["outcome"] == "DRY_RUN"
|
||||
|
||||
|
||||
def test_authorization_revoked_right_before_write_fails_closed(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N4")
|
||||
calls = {"n": 0}
|
||||
|
||||
def flip():
|
||||
calls["n"] += 1
|
||||
return {"features": dict(ENABLED if calls["n"] == 1 else {"features_configured": True, "enrichment_mode": "DISABLED"})}
|
||||
hub.authorizations = flip
|
||||
now[0] += 5
|
||||
worker._features_at = -999
|
||||
worker.step()
|
||||
assert row(db, "N4") == (None, None, None, "keep")
|
||||
assert all(f["outcome"] == "ABORTED" and f["reason"] == "AUTHORIZATION_CHANGED" for f in hub.reports[0]["fields"])
|
||||
|
||||
|
||||
def test_human_override_including_deleted_value_is_final(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N5")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
db.execute("UPDATE C000025 SET CEST='', CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N5'") # a person deletes the CEST and changes the NCM
|
||||
db.commit()
|
||||
now[0] += 30
|
||||
worker.step()
|
||||
assert store.is_overridden("N5", "CEST") and store.is_overridden("N5", "CLASSIFICACAO_FISCAL") and not store.is_overridden("N5", "IPI")
|
||||
# the description changes a moment later: the worker re-resolves, but overridden fields must NOT be written again
|
||||
db.execute("UPDATE C000025 SET PRODUTO='REFRIGERANTE COLA LATA 350ML ZERO' WHERE CODIGO='N5'")
|
||||
db.commit()
|
||||
now[0] += 30
|
||||
worker.step()
|
||||
assert row(db, "N5")[:2] == ("19059090", "")
|
||||
|
||||
|
||||
def test_write_field_allowlist_and_shape():
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT, COD_ORIGEM_MERCADORIA TEXT, CLASSIFICACAO_FISCAL TEXT)")
|
||||
with pytest.raises(ValueError):
|
||||
E.write_field(Conn(db), "1", "COD_ORIGEM_MERCADORIA", "1")
|
||||
with pytest.raises(ValueError):
|
||||
E.write_field(Conn(db), "1", "CCLASSTRIB", "000001")
|
||||
db.execute("INSERT INTO C000025(CODIGO) VALUES('1')")
|
||||
db.commit()
|
||||
assert E.write_field(Conn(db), "1", "CLASSIFICACAO_FISCAL", "123")["reason"] == "VALUE_SHAPE_INVALID"
|
||||
|
||||
|
||||
def test_hub_offline_keeps_product_pending_and_retries(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N6")
|
||||
|
||||
def boom(payload):
|
||||
raise ConnectionError("down")
|
||||
good = hub.resolve_product
|
||||
hub.resolve_product = boom
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert store.get("N6")["state"] == "RETRY" and row(db, "N6")[0] is None
|
||||
hub.resolve_product = good
|
||||
now[0] += 40
|
||||
assert worker.step() == ["N6"] and row(db, "N6")[0] == "22021000"
|
||||
Reference in New Issue
Block a user