"""COPY-ONLY E2E of the new-product enrichment on a REAL Firebird (disposable copy restored from its own .fbk). python scripts/enrichment_copy_e2e.py [--products 12] The Agent code under test is the real one (gti_fiscal_agent.enrichment: real fdb connections, real typed UPDATE, real threads/timing). The Hub side is the real resolver run in-process on SQLite with the official NCM dataset and fake providers (no internet). Proves on a real Firebird: detection latency P50/P95 (product INSERTed by an "operator" -> fields written), only-empty fields written, other columns + every other table untouched, historical rows untouched, second run = 0 writes, DRY_RUN = 0 writes, DISABLED / WAITING = 0 Firebird activity, human override incl. deleted value, Hub offline -> pending -> recovers, authorization revoked -> 0 writes. """ import json import os import statistics import sys import threading import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src")) from dotenv import load_dotenv # noqa: E402 load_dotenv(ROOT / ".env") import rtc_copy_write_e2e as base # noqa: E402 from gti_fiscal_agent import enrichment as E # noqa: E402 from sqlalchemy import create_engine # noqa: E402 from sqlalchemy.orm import Session # noqa: E402 from sqlalchemy.pool import StaticPool # noqa: E402 from central.app.datasets.ncm import parse_ncm_payload # noqa: E402 from central.app.datasets.service import DatasetService # noqa: E402 from central.app.enrichment import kb # noqa: E402 from central.app.enrichment.providers import Provider, ProviderProduct # noqa: E402 from central.app.enrichment.quota import ProviderGateway # noqa: E402 from central.app.enrichment.resolver import resolve # noqa: E402 from central.app.models import Base, CestEntry, CestNcmEntry, Dataset, TipiEntry # noqa: E402 from datetime import datetime, timezone # noqa: E402 import httpx # noqa: E402 def gtin(body12): total = sum(int(d) * (3 if i % 2 == 0 else 1) for i, d in enumerate(reversed(body12))) return body12 + str((10 - total % 10) % 10) class Fake(Provider): def __init__(self, name, ncm): super().__init__(client=httpx.Client()) self.name, self.ncm, self.provides_ncm = name, ncm, True def lookup_gtin(self, g): return ProviderProduct(self.name, g, True, title="PRODUTO DE TESTE ESPECIFICO 350ML", ncm=self.ncm) class InProcessHub: """The real Hub resolver, in-process. `online=False` simulates the Hub being unreachable.""" def __init__(self): engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) Base.metadata.create_all(engine) self.session = Session(engine) DatasetService(self.session).import_payload(parse_ncm_payload((ROOT / "scratch_enrich" / "ncm_siscomex.json").read_bytes()), "official", datetime.now(timezone.utc)) tipi = Dataset(kind="TIPI_IPI", provider="RFB", sha256="b" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=1) cest = Dataset(kind="CEST", provider="CONFAZ", sha256="c" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=1) self.session.add_all([tipi, cest]); self.session.flush() self.session.add(TipiEntry(dataset_id=tipi.id, ncm="22021000", ex=None, description="x", ipi_rate=0.0, ipi_rate_text="0", source_row=1)) e = CestEntry(dataset_id=cest.id, code="03.007.00", description="PRODUTO DE TESTE ESPECIFICO 350ML", annex="I") self.session.add(e); self.session.flush(); self.session.add(CestNcmEntry(cest_entry_id=e.id, ncm_code="22021000")); self.session.commit() self.providers = [Fake("cosmos", "22021000"), Fake("second", "22021000")] self.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"} self.online, self.reports, self.overridden, self.calls = True, [], {}, 0 self.lock = threading.Lock() def authorizations(self): if not self.online: raise ConnectionError("hub offline") return {"features": dict(self.features)} def resolve_product(self, payload): if not self.online: raise ConnectionError("hub offline") with self.lock: self.calls += 1 r = resolve(self.session, ProviderGateway(self.session, self.providers), gtin_raw=payload["gtin"], description=payload["description"], current=payload["current"]) for f in self.overridden.get(payload["product_code"], ()): r["fields"][f] = {"status": "HUMAN_OVERRIDDEN", "value": None} r["retry_at"] = None return r def report_enrichment(self, payload): with self.lock: self.reports.append(payload) for it in payload["fields"]: if it["outcome"] == "HUMAN_OVERRIDDEN": self.overridden.setdefault(payload["product_code"], set()).add(it["field"]) return {"ok": True} _orig_connect = base.connect def _connect_retry(rw): """Same brief retry the Agent applies: local XNET connections can report 'connection lost' when several threads connect at the same instant.""" for attempt in range(6): try: return _orig_connect(rw) except Exception: # noqa: BLE001 if attempt == 5: raise time.sleep(0.2 * (attempt + 1)) base.connect = _connect_retry def insert_product(code, name, ean): conn = base.connect(True); cur = conn.cursor() cur.execute("INSERT INTO C000025 (CODIGO, PRODUTO, CODBARRA) VALUES (?, ?, ?)", (code, name, ean)) conn.commit(); conn.close() def row(code): conn = base.connect(False); cur = conn.cursor() cur.execute("SELECT CLASSIFICACAO_FISCAL, CEST, IPI FROM C000025 WHERE CODIGO=?", (code,)) r = cur.fetchone(); conn.close() return tuple(None if v is None else (str(v).strip() if not isinstance(v, float) else v) for v in r) if r else None def side_hashes(): """C000058 (ERP change journal) and C000100 (stock) are maintained by the SGV's OWN triggers on C000025 INSERT/UPDATE. Only rows of the test products (code 'Z...') may differ.""" import hashlib conn = base.connect(False); cur = conn.cursor(); out = {} for table, col in (("C000058", "CODREGISTRO"), ("C000100", "CODPRODUTO")): cur.execute(f"SELECT * FROM {table} WHERE {col} NOT STARTING WITH 'Z'") h = hashlib.sha256() n = 0 for r in cur.fetchall(): h.update(repr(r).encode("utf-8", "replace")); n += 1 out[table] = f"{n}:{h.hexdigest()[:16]}" conn.close() return out def main(): n = int(sys.argv[sys.argv.index("--products") + 1]) if "--products" in sys.argv else 12 checks = {} base.restore_copy() before_tables, _ = base.baseline() before_side = side_hashes() hub = InProcessHub() for suffix in ("", "-wal", "-shm"): try: (ROOT / "scratch_enrich" / f"enrich_e2e.db{suffix}").unlink() except FileNotFoundError: pass store = E.EnrichmentStore(ROOT / "scratch_enrich" / "enrich_e2e.db") worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: base.connect(False), write_conn=lambda: base.connect(True), create_reprocess=lambda codes: None) class Stop: stop = False stopper = Stop() thread = threading.Thread(target=worker.run_forever, args=(stopper,), daemon=True) # --- WAITING / DISABLED: zero Firebird activity, baseline NOT taken hub.features = {"features_configured": False} thread.start(); time.sleep(4) checks["WAITING_NO_BASELINE_NO_CALLS"] = store.meta("baseline_done") is None and hub.calls == 0 hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "DISABLED"}; worker._features_at = -999; time.sleep(4) checks["DISABLED_NO_BASELINE_NO_CALLS"] = store.meta("baseline_done") is None and hub.calls == 0 # --- DRY_RUN: baseline, detect, resolve, report, zero writes hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "DRY_RUN"}; worker._features_at = -999; time.sleep(5) checks["BASELINE_TAKEN_ON_ACTIVATION"] = store.meta("baseline_done") == "1" hist = len(store.known()) insert_product("ZD001", "PRODUTO DE TESTE ESPECIFICO 350ML", gtin("789200000001")); time.sleep(8) checks["DRY_RUN_ZERO_WRITES"] = row("ZD001") == (None, None, None) and hub.reports and hub.reports[-1]["fields"][0]["outcome"] == "DRY_RUN" # --- ENABLED: N products inserted like an operator would, latency measured insert -> written hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}; worker._features_at = -999; worker._auth_cache = (None, 0.0) latencies = [] for i in range(n): code, ean = f"ZE{i:03d}", gtin(f"78930000{i:04d}") t0 = time.monotonic() insert_product(code, "PRODUTO DE TESTE ESPECIFICO 350ML", ean) while time.monotonic() - t0 < 20 and row(code) == (None, None, None): time.sleep(0.1) latencies.append(round(time.monotonic() - t0, 2)) time.sleep(0.5) ok_rows = [row(f"ZE{i:03d}") for i in range(n)] checks["FIELDS_WRITTEN"] = all(r and r[0] == "22021000" and r[1] == "0300700" and r[2] == 0.0 for r in ok_rows) time.sleep(6) writes_before = sum(1 for rep in hub.reports for f in rep["fields"] if f["outcome"] == "APPLIED") calls_before = hub.calls time.sleep(8) # a "second run": nothing pending, nothing re-written checks["SECOND_RUN_ZERO_WRITES"] = writes_before == sum(1 for rep in hub.reports for f in rep["fields"] if f["outcome"] == "APPLIED") and hub.calls - calls_before <= 0 checks["HISTORICAL_KNOWN_COUNT"] = hist > 100 # --- human override incl. deleted value conn = base.connect(True); cur = conn.cursor() cur.execute("UPDATE C000025 SET CEST = NULL WHERE CODIGO = 'ZE000'"); cur.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL = '19059090' WHERE CODIGO = 'ZE001'"); conn.commit(); conn.close() time.sleep(E.OVERRIDE_CHECK_SECONDS + 6) checks["OVERRIDE_DELETED_VALUE"] = store.is_overridden("ZE000", "CEST") and store.is_overridden("ZE001", "CLASSIFICACAO_FISCAL") checks["OVERRIDE_REPORTED_TO_HUB"] = "CEST" in hub.overridden.get("ZE000", set()) # ... and a later description edit does NOT bring the value back conn = base.connect(True); cur = conn.cursor(); cur.execute("UPDATE C000025 SET PRODUTO = 'PRODUTO DE TESTE ESPECIFICO 350ML ZERO' WHERE CODIGO = 'ZE000'"); conn.commit(); conn.close() time.sleep(10) checks["OVERRIDE_NEVER_REWRITTEN"] = row("ZE000")[1] in (None, "") and row("ZE001")[0] == "19059090" # --- Hub offline -> pending -> recovers ; authorization revoked -> 0 writes hub.online = False insert_product("ZO001", "PRODUTO DE TESTE ESPECIFICO 350ML", gtin("789400000001")); time.sleep(8) checks["HUB_OFFLINE_PENDING_NO_WRITE"] = row("ZO001") == (None, None, None) and store.get("ZO001")["state"] in ("RETRY", "PENDING") hub.online = True with store.db: store.db.execute("UPDATE enrich_products SET next_attempt_at=0 WHERE code='ZO001'") time.sleep(8) checks["HUB_BACK_RECOVERS"] = row("ZO001") == ("22021000", "0300700", 0.0) stopper.stop = True thread.join(timeout=10) # --- invariants on the real Firebird copy: only the 3 allowed columns of the new rows changed after_tables, _ = base.baseline() changed = base.diff_tables(before_tables, after_tables) checks["OTHER_TABLES_ONLY_ERP_TRIGGER_SIDE_EFFECTS_OF_TEST_ROWS"] = set(changed) <= {"C000025", "C000058", "C000100"} and side_hashes() == before_side conn = base.connect(False); cur = conn.cursor() cur.execute("SELECT COUNT(*) FROM C000025 WHERE CODIGO STARTING WITH 'Z' AND COD_ORIGEM_MERCADORIA IS NOT NULL AND TRIM(COD_ORIGEM_MERCADORIA) <> ''"); origin = cur.fetchone()[0]; conn.close() checks["ORIGIN_NEVER_WRITTEN"] = origin == 0 result = {"copy_only": True, "products_enabled": n, "detect_to_written_seconds": {"p50": statistics.median(latencies), "p95": sorted(latencies)[int(0.95 * (len(latencies) - 1))], "max": max(latencies), "all": latencies}, "hub_resolve_calls": hub.calls, "tables_changed": changed, "checks": checks, "ALL_PASS": all(checks.values()), "poll_seconds": worker.poll_seconds()} Path(ROOT / "docs" / "runs").mkdir(parents=True, exist_ok=True) (ROOT / "docs" / "runs" / "enrichment_copy_e2e.json").write_text(json.dumps(result, indent=1), encoding="utf-8") print(json.dumps(result, indent=1)) sys.exit(0 if result["ALL_PASS"] else 1) if __name__ == "__main__": main()