238 lines
13 KiB
Python
238 lines
13 KiB
Python
"""COPY-ONLY E2E of the new-product enrichment on a REAL Firebird (disposable copy restored from its own .fbk).
|
|
|
|
python scripts/enrichment_copy_e2e.py [--products 12]
|
|
|
|
The Agent code under test is the real one (gti_fiscal_agent.enrichment: real fdb connections, real typed UPDATE, real threads/timing). The Hub side is the real resolver run in-process on
|
|
SQLite with the official NCM dataset and fake providers (no internet). Proves on a real Firebird:
|
|
detection latency P50/P95 (product INSERTed by an "operator" -> fields written), only-empty fields written, other columns + every other table untouched, historical rows untouched,
|
|
second run = 0 writes, DRY_RUN = 0 writes, DISABLED / WAITING = 0 Firebird activity, human override incl. deleted value, Hub offline -> pending -> recovers, authorization revoked -> 0 writes.
|
|
"""
|
|
import json
|
|
import os
|
|
import statistics
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
|
|
from dotenv import load_dotenv # noqa: E402
|
|
|
|
load_dotenv(ROOT / ".env")
|
|
import rtc_copy_write_e2e as base # noqa: E402
|
|
from gti_fiscal_agent import enrichment as E # noqa: E402
|
|
from sqlalchemy import create_engine # noqa: E402
|
|
from sqlalchemy.orm import Session # noqa: E402
|
|
from sqlalchemy.pool import StaticPool # noqa: E402
|
|
|
|
from central.app.datasets.ncm import parse_ncm_payload # noqa: E402
|
|
from central.app.datasets.service import DatasetService # noqa: E402
|
|
from central.app.enrichment import kb # noqa: E402
|
|
from central.app.enrichment.providers import Provider, ProviderProduct # noqa: E402
|
|
from central.app.enrichment.quota import ProviderGateway # noqa: E402
|
|
from central.app.enrichment.resolver import resolve # noqa: E402
|
|
from central.app.models import Base, CestEntry, CestNcmEntry, Dataset, TipiEntry # noqa: E402
|
|
from datetime import datetime, timezone # noqa: E402
|
|
import httpx # noqa: E402
|
|
|
|
|
|
def gtin(body12):
|
|
total = sum(int(d) * (3 if i % 2 == 0 else 1) for i, d in enumerate(reversed(body12)))
|
|
return body12 + str((10 - total % 10) % 10)
|
|
|
|
|
|
class Fake(Provider):
|
|
def __init__(self, name, ncm):
|
|
super().__init__(client=httpx.Client())
|
|
self.name, self.ncm, self.provides_ncm = name, ncm, True
|
|
|
|
def lookup_gtin(self, g):
|
|
return ProviderProduct(self.name, g, True, title="PRODUTO DE TESTE ESPECIFICO 350ML", ncm=self.ncm)
|
|
|
|
|
|
class InProcessHub:
|
|
"""The real Hub resolver, in-process. `online=False` simulates the Hub being unreachable."""
|
|
|
|
def __init__(self):
|
|
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
|
Base.metadata.create_all(engine)
|
|
self.session = Session(engine)
|
|
DatasetService(self.session).import_payload(parse_ncm_payload((ROOT / "scratch_enrich" / "ncm_siscomex.json").read_bytes()), "official", datetime.now(timezone.utc))
|
|
tipi = Dataset(kind="TIPI_IPI", provider="RFB", sha256="b" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=1)
|
|
cest = Dataset(kind="CEST", provider="CONFAZ", sha256="c" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=1)
|
|
self.session.add_all([tipi, cest]); self.session.flush()
|
|
self.session.add(TipiEntry(dataset_id=tipi.id, ncm="22021000", ex=None, description="x", ipi_rate=0.0, ipi_rate_text="0", source_row=1))
|
|
e = CestEntry(dataset_id=cest.id, code="03.007.00", description="PRODUTO DE TESTE ESPECIFICO 350ML", annex="I")
|
|
self.session.add(e); self.session.flush(); self.session.add(CestNcmEntry(cest_entry_id=e.id, ncm_code="22021000")); self.session.commit()
|
|
self.providers = [Fake("cosmos", "22021000"), Fake("second", "22021000")]
|
|
self.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}
|
|
self.online, self.reports, self.overridden, self.calls = True, [], {}, 0
|
|
self.lock = threading.Lock()
|
|
|
|
def authorizations(self):
|
|
if not self.online:
|
|
raise ConnectionError("hub offline")
|
|
return {"features": dict(self.features)}
|
|
|
|
def resolve_product(self, payload):
|
|
if not self.online:
|
|
raise ConnectionError("hub offline")
|
|
with self.lock:
|
|
self.calls += 1
|
|
r = resolve(self.session, ProviderGateway(self.session, self.providers), gtin_raw=payload["gtin"], description=payload["description"], current=payload["current"])
|
|
for f in self.overridden.get(payload["product_code"], ()):
|
|
r["fields"][f] = {"status": "HUMAN_OVERRIDDEN", "value": None}
|
|
r["retry_at"] = None
|
|
return r
|
|
|
|
def report_enrichment(self, payload):
|
|
with self.lock:
|
|
self.reports.append(payload)
|
|
for it in payload["fields"]:
|
|
if it["outcome"] == "HUMAN_OVERRIDDEN":
|
|
self.overridden.setdefault(payload["product_code"], set()).add(it["field"])
|
|
return {"ok": True}
|
|
|
|
|
|
_orig_connect = base.connect
|
|
|
|
|
|
def _connect_retry(rw):
|
|
"""Same brief retry the Agent applies: local XNET connections can report 'connection lost' when several threads connect at the same instant."""
|
|
for attempt in range(6):
|
|
try:
|
|
return _orig_connect(rw)
|
|
except Exception: # noqa: BLE001
|
|
if attempt == 5:
|
|
raise
|
|
time.sleep(0.2 * (attempt + 1))
|
|
|
|
|
|
base.connect = _connect_retry
|
|
|
|
|
|
def insert_product(code, name, ean):
|
|
conn = base.connect(True); cur = conn.cursor()
|
|
cur.execute("INSERT INTO C000025 (CODIGO, PRODUTO, CODBARRA) VALUES (?, ?, ?)", (code, name, ean))
|
|
conn.commit(); conn.close()
|
|
|
|
|
|
def row(code):
|
|
conn = base.connect(False); cur = conn.cursor()
|
|
cur.execute("SELECT CLASSIFICACAO_FISCAL, CEST, IPI FROM C000025 WHERE CODIGO=?", (code,))
|
|
r = cur.fetchone(); conn.close()
|
|
return tuple(None if v is None else (str(v).strip() if not isinstance(v, float) else v) for v in r) if r else None
|
|
|
|
|
|
def side_hashes():
|
|
"""C000058 (ERP change journal) and C000100 (stock) are maintained by the SGV's OWN triggers on C000025 INSERT/UPDATE. Only rows of the test products (code 'Z...') may differ."""
|
|
import hashlib
|
|
conn = base.connect(False); cur = conn.cursor(); out = {}
|
|
for table, col in (("C000058", "CODREGISTRO"), ("C000100", "CODPRODUTO")):
|
|
cur.execute(f"SELECT * FROM {table} WHERE {col} NOT STARTING WITH 'Z'")
|
|
h = hashlib.sha256()
|
|
n = 0
|
|
for r in cur.fetchall():
|
|
h.update(repr(r).encode("utf-8", "replace")); n += 1
|
|
out[table] = f"{n}:{h.hexdigest()[:16]}"
|
|
conn.close()
|
|
return out
|
|
|
|
|
|
def main():
|
|
n = int(sys.argv[sys.argv.index("--products") + 1]) if "--products" in sys.argv else 12
|
|
checks = {}
|
|
base.restore_copy()
|
|
before_tables, _ = base.baseline()
|
|
before_side = side_hashes()
|
|
hub = InProcessHub()
|
|
for suffix in ("", "-wal", "-shm"):
|
|
try:
|
|
(ROOT / "scratch_enrich" / f"enrich_e2e.db{suffix}").unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
store = E.EnrichmentStore(ROOT / "scratch_enrich" / "enrich_e2e.db")
|
|
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: base.connect(False), write_conn=lambda: base.connect(True),
|
|
create_reprocess=lambda codes: None)
|
|
|
|
class Stop:
|
|
stop = False
|
|
stopper = Stop()
|
|
thread = threading.Thread(target=worker.run_forever, args=(stopper,), daemon=True)
|
|
|
|
# --- WAITING / DISABLED: zero Firebird activity, baseline NOT taken
|
|
hub.features = {"features_configured": False}
|
|
thread.start(); time.sleep(4)
|
|
checks["WAITING_NO_BASELINE_NO_CALLS"] = store.meta("baseline_done") is None and hub.calls == 0
|
|
hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "DISABLED"}; worker._features_at = -999; time.sleep(4)
|
|
checks["DISABLED_NO_BASELINE_NO_CALLS"] = store.meta("baseline_done") is None and hub.calls == 0
|
|
|
|
# --- DRY_RUN: baseline, detect, resolve, report, zero writes
|
|
hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "DRY_RUN"}; worker._features_at = -999; time.sleep(5)
|
|
checks["BASELINE_TAKEN_ON_ACTIVATION"] = store.meta("baseline_done") == "1"
|
|
hist = len(store.known())
|
|
insert_product("ZD001", "PRODUTO DE TESTE ESPECIFICO 350ML", gtin("789200000001")); time.sleep(8)
|
|
checks["DRY_RUN_ZERO_WRITES"] = row("ZD001") == (None, None, None) and hub.reports and hub.reports[-1]["fields"][0]["outcome"] == "DRY_RUN"
|
|
|
|
# --- ENABLED: N products inserted like an operator would, latency measured insert -> written
|
|
hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}; worker._features_at = -999; worker._auth_cache = (None, 0.0)
|
|
latencies = []
|
|
for i in range(n):
|
|
code, ean = f"ZE{i:03d}", gtin(f"78930000{i:04d}")
|
|
t0 = time.monotonic()
|
|
insert_product(code, "PRODUTO DE TESTE ESPECIFICO 350ML", ean)
|
|
while time.monotonic() - t0 < 20 and row(code) == (None, None, None):
|
|
time.sleep(0.1)
|
|
latencies.append(round(time.monotonic() - t0, 2))
|
|
time.sleep(0.5)
|
|
ok_rows = [row(f"ZE{i:03d}") for i in range(n)]
|
|
checks["FIELDS_WRITTEN"] = all(r and r[0] == "22021000" and r[1] == "0300700" and r[2] == 0.0 for r in ok_rows)
|
|
time.sleep(6)
|
|
writes_before = sum(1 for rep in hub.reports for f in rep["fields"] if f["outcome"] == "APPLIED")
|
|
calls_before = hub.calls
|
|
time.sleep(8) # a "second run": nothing pending, nothing re-written
|
|
checks["SECOND_RUN_ZERO_WRITES"] = writes_before == sum(1 for rep in hub.reports for f in rep["fields"] if f["outcome"] == "APPLIED") and hub.calls - calls_before <= 0
|
|
checks["HISTORICAL_KNOWN_COUNT"] = hist > 100
|
|
|
|
# --- human override incl. deleted value
|
|
conn = base.connect(True); cur = conn.cursor()
|
|
cur.execute("UPDATE C000025 SET CEST = NULL WHERE CODIGO = 'ZE000'"); cur.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL = '19059090' WHERE CODIGO = 'ZE001'"); conn.commit(); conn.close()
|
|
time.sleep(E.OVERRIDE_CHECK_SECONDS + 6)
|
|
checks["OVERRIDE_DELETED_VALUE"] = store.is_overridden("ZE000", "CEST") and store.is_overridden("ZE001", "CLASSIFICACAO_FISCAL")
|
|
checks["OVERRIDE_REPORTED_TO_HUB"] = "CEST" in hub.overridden.get("ZE000", set())
|
|
# ... and a later description edit does NOT bring the value back
|
|
conn = base.connect(True); cur = conn.cursor(); cur.execute("UPDATE C000025 SET PRODUTO = 'PRODUTO DE TESTE ESPECIFICO 350ML ZERO' WHERE CODIGO = 'ZE000'"); conn.commit(); conn.close()
|
|
time.sleep(10)
|
|
checks["OVERRIDE_NEVER_REWRITTEN"] = row("ZE000")[1] in (None, "") and row("ZE001")[0] == "19059090"
|
|
|
|
# --- Hub offline -> pending -> recovers ; authorization revoked -> 0 writes
|
|
hub.online = False
|
|
insert_product("ZO001", "PRODUTO DE TESTE ESPECIFICO 350ML", gtin("789400000001")); time.sleep(8)
|
|
checks["HUB_OFFLINE_PENDING_NO_WRITE"] = row("ZO001") == (None, None, None) and store.get("ZO001")["state"] in ("RETRY", "PENDING")
|
|
hub.online = True
|
|
with store.db:
|
|
store.db.execute("UPDATE enrich_products SET next_attempt_at=0 WHERE code='ZO001'")
|
|
time.sleep(8)
|
|
checks["HUB_BACK_RECOVERS"] = row("ZO001") == ("22021000", "0300700", 0.0)
|
|
stopper.stop = True
|
|
thread.join(timeout=10)
|
|
|
|
# --- invariants on the real Firebird copy: only the 3 allowed columns of the new rows changed
|
|
after_tables, _ = base.baseline()
|
|
changed = base.diff_tables(before_tables, after_tables)
|
|
checks["OTHER_TABLES_ONLY_ERP_TRIGGER_SIDE_EFFECTS_OF_TEST_ROWS"] = set(changed) <= {"C000025", "C000058", "C000100"} and side_hashes() == before_side
|
|
conn = base.connect(False); cur = conn.cursor()
|
|
cur.execute("SELECT COUNT(*) FROM C000025 WHERE CODIGO STARTING WITH 'Z' AND COD_ORIGEM_MERCADORIA IS NOT NULL AND TRIM(COD_ORIGEM_MERCADORIA) <> ''"); origin = cur.fetchone()[0]; conn.close()
|
|
checks["ORIGIN_NEVER_WRITTEN"] = origin == 0
|
|
result = {"copy_only": True, "products_enabled": n, "detect_to_written_seconds": {"p50": statistics.median(latencies), "p95": sorted(latencies)[int(0.95 * (len(latencies) - 1))], "max": max(latencies), "all": latencies},
|
|
"hub_resolve_calls": hub.calls, "tables_changed": changed, "checks": checks, "ALL_PASS": all(checks.values()), "poll_seconds": worker.poll_seconds()}
|
|
Path(ROOT / "docs" / "runs").mkdir(parents=True, exist_ok=True)
|
|
(ROOT / "docs" / "runs" / "enrichment_copy_e2e.json").write_text(json.dumps(result, indent=1), encoding="utf-8")
|
|
print(json.dumps(result, indent=1))
|
|
sys.exit(0 if result["ALL_PASS"] else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|