feat: product enrichment observability, lab KB study, benchmark, copy + local-Hub E2E, docs; 1.3.0
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
Claude Sonnet 5
@@ -0,0 +1,79 @@
|
||||
"""Conservative provider benchmark on REAL GTINs taken (read-only) from the lab ERP databases. Free quotas only, paced below documented limits.
|
||||
|
||||
python scripts/enrichment_benchmark.py --sample scratch_enrich/benchmark_gtins.json --out docs/PRODUCT_PROVIDER_BENCHMARK_V1.json [--limit 30]
|
||||
|
||||
Each provider gets at most --limit lookups (UPCitemdb: also <= 12 per 30 s burst window; Open Food Facts: <= 12/min). Cosmos is measured only when COSMOS_TOKEN is set.
|
||||
Reports sample_size / found / not_found / description / brand / NCM / CEST / latency / error_rate per provider. Secrets are never printed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
import httpx # noqa: E402
|
||||
|
||||
from central.app.enrichment.providers import Cosmos, OpenBeautyFacts, OpenFoodFacts, OpenProductsFacts, ProviderError, UpcItemDb, USER_AGENT # noqa: E402
|
||||
|
||||
PACE = {"upcitemdb": 2.6, "openfoodfacts": 5.2, "openproductsfacts": 5.2, "openbeautyfacts": 5.2, "cosmos": 1.0} # seconds between calls: comfortably under the documented burst / per-minute limits
|
||||
|
||||
|
||||
def bench(provider, items):
|
||||
out = {"provider": provider.name, "configured": provider.configured, "sample_size": 0, "found": 0, "not_found": 0, "with_description": 0, "with_brand": 0, "with_ncm": 0, "with_cest": 0, "errors": 0, "error_kinds": {}, "latencies_ms": []}
|
||||
if not provider.configured:
|
||||
out["status"] = "NOT_CONFIGURED"
|
||||
return out
|
||||
for item in items:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
product = provider.lookup_gtin(item["gtin"])
|
||||
except ProviderError as error:
|
||||
out["errors"] += 1
|
||||
out["error_kinds"][error.kind] = out["error_kinds"].get(error.kind, 0) + 1
|
||||
if error.kind in ("RATE_LIMITED", "AUTH_ERROR"):
|
||||
break # never insist against a limit
|
||||
time.sleep(PACE[provider.name])
|
||||
continue
|
||||
out["sample_size"] += 1
|
||||
out["latencies_ms"].append(int((time.monotonic() - started) * 1000))
|
||||
if product.found:
|
||||
out["found"] += 1
|
||||
out["with_description"] += bool(product.title)
|
||||
out["with_brand"] += bool(product.brand)
|
||||
out["with_ncm"] += bool(product.ncm)
|
||||
out["with_cest"] += bool(product.cest)
|
||||
else:
|
||||
out["not_found"] += 1
|
||||
time.sleep(PACE[provider.name])
|
||||
n = out["sample_size"] + out["errors"]
|
||||
lat = sorted(out.pop("latencies_ms"))
|
||||
out.update({"error_rate": round(out["errors"] / n, 3) if n else None, "latency_p50_ms": statistics.median(lat) if lat else None, "latency_p95_ms": lat[int(0.95 * (len(lat) - 1))] if lat else None,
|
||||
"found_rate": round(out["found"] / out["sample_size"], 3) if out["sample_size"] else None, "status": "MEASURED"})
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--sample", required=True)
|
||||
ap.add_argument("--out", required=True)
|
||||
ap.add_argument("--limit", type=int, default=30)
|
||||
ap.add_argument("--only", default="", help="comma list of provider names; results are MERGED into an existing --out")
|
||||
args = ap.parse_args()
|
||||
items = json.loads(Path(args.sample).read_text(encoding="utf-8"))[: args.limit]
|
||||
client = httpx.Client(timeout=8.0, headers={"User-Agent": USER_AGENT})
|
||||
providers = [Cosmos(client, os.environ.get("COSMOS_TOKEN")), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
|
||||
only = {n for n in args.only.split(",") if n}
|
||||
out = Path(args.out)
|
||||
result = json.loads(out.read_text(encoding="utf-8")) if only and out.exists() else {"measured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "sample_source": "lab ERP databases (read-only), valid GTINs only", "limit_per_provider": args.limit, "providers": []}
|
||||
measured = [bench(p, items) for p in providers if not only or p.name in only]
|
||||
result["providers"] = [p for p in result["providers"] if p["provider"] not in {m["provider"] for m in measured}] + measured
|
||||
Path(args.out).write_text(json.dumps(result, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
print(json.dumps(result, ensure_ascii=False, indent=1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,237 @@
|
||||
"""COPY-ONLY E2E of the new-product enrichment on a REAL Firebird (disposable copy restored from its own .fbk).
|
||||
|
||||
python scripts/enrichment_copy_e2e.py [--products 12]
|
||||
|
||||
The Agent code under test is the real one (gti_fiscal_agent.enrichment: real fdb connections, real typed UPDATE, real threads/timing). The Hub side is the real resolver run in-process on
|
||||
SQLite with the official NCM dataset and fake providers (no internet). Proves on a real Firebird:
|
||||
detection latency P50/P95 (product INSERTed by an "operator" -> fields written), only-empty fields written, other columns + every other table untouched, historical rows untouched,
|
||||
second run = 0 writes, DRY_RUN = 0 writes, DISABLED / WAITING = 0 Firebird activity, human override incl. deleted value, Hub offline -> pending -> recovers, authorization revoked -> 0 writes.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from dotenv import load_dotenv # noqa: E402
|
||||
|
||||
load_dotenv(ROOT / ".env")
|
||||
import rtc_copy_write_e2e as base # noqa: E402
|
||||
from gti_fiscal_agent import enrichment as E # noqa: E402
|
||||
from sqlalchemy import create_engine # noqa: E402
|
||||
from sqlalchemy.orm import Session # noqa: E402
|
||||
from sqlalchemy.pool import StaticPool # noqa: E402
|
||||
|
||||
from central.app.datasets.ncm import parse_ncm_payload # noqa: E402
|
||||
from central.app.datasets.service import DatasetService # noqa: E402
|
||||
from central.app.enrichment import kb # noqa: E402
|
||||
from central.app.enrichment.providers import Provider, ProviderProduct # noqa: E402
|
||||
from central.app.enrichment.quota import ProviderGateway # noqa: E402
|
||||
from central.app.enrichment.resolver import resolve # noqa: E402
|
||||
from central.app.models import Base, CestEntry, CestNcmEntry, Dataset, TipiEntry # noqa: E402
|
||||
from datetime import datetime, timezone # noqa: E402
|
||||
import httpx # noqa: E402
|
||||
|
||||
|
||||
def gtin(body12):
|
||||
total = sum(int(d) * (3 if i % 2 == 0 else 1) for i, d in enumerate(reversed(body12)))
|
||||
return body12 + str((10 - total % 10) % 10)
|
||||
|
||||
|
||||
class Fake(Provider):
|
||||
def __init__(self, name, ncm):
|
||||
super().__init__(client=httpx.Client())
|
||||
self.name, self.ncm, self.provides_ncm = name, ncm, True
|
||||
|
||||
def lookup_gtin(self, g):
|
||||
return ProviderProduct(self.name, g, True, title="PRODUTO DE TESTE ESPECIFICO 350ML", ncm=self.ncm)
|
||||
|
||||
|
||||
class InProcessHub:
|
||||
"""The real Hub resolver, in-process. `online=False` simulates the Hub being unreachable."""
|
||||
|
||||
def __init__(self):
|
||||
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(engine)
|
||||
self.session = Session(engine)
|
||||
DatasetService(self.session).import_payload(parse_ncm_payload((ROOT / "scratch_enrich" / "ncm_siscomex.json").read_bytes()), "official", datetime.now(timezone.utc))
|
||||
tipi = Dataset(kind="TIPI_IPI", provider="RFB", sha256="b" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=1)
|
||||
cest = Dataset(kind="CEST", provider="CONFAZ", sha256="c" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=1)
|
||||
self.session.add_all([tipi, cest]); self.session.flush()
|
||||
self.session.add(TipiEntry(dataset_id=tipi.id, ncm="22021000", ex=None, description="x", ipi_rate=0.0, ipi_rate_text="0", source_row=1))
|
||||
e = CestEntry(dataset_id=cest.id, code="03.007.00", description="PRODUTO DE TESTE ESPECIFICO 350ML", annex="I")
|
||||
self.session.add(e); self.session.flush(); self.session.add(CestNcmEntry(cest_entry_id=e.id, ncm_code="22021000")); self.session.commit()
|
||||
self.providers = [Fake("cosmos", "22021000"), Fake("second", "22021000")]
|
||||
self.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}
|
||||
self.online, self.reports, self.overridden, self.calls = True, [], {}, 0
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def authorizations(self):
|
||||
if not self.online:
|
||||
raise ConnectionError("hub offline")
|
||||
return {"features": dict(self.features)}
|
||||
|
||||
def resolve_product(self, payload):
|
||||
if not self.online:
|
||||
raise ConnectionError("hub offline")
|
||||
with self.lock:
|
||||
self.calls += 1
|
||||
r = resolve(self.session, ProviderGateway(self.session, self.providers), gtin_raw=payload["gtin"], description=payload["description"], current=payload["current"])
|
||||
for f in self.overridden.get(payload["product_code"], ()):
|
||||
r["fields"][f] = {"status": "HUMAN_OVERRIDDEN", "value": None}
|
||||
r["retry_at"] = None
|
||||
return r
|
||||
|
||||
def report_enrichment(self, payload):
|
||||
with self.lock:
|
||||
self.reports.append(payload)
|
||||
for it in payload["fields"]:
|
||||
if it["outcome"] == "HUMAN_OVERRIDDEN":
|
||||
self.overridden.setdefault(payload["product_code"], set()).add(it["field"])
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
_orig_connect = base.connect
|
||||
|
||||
|
||||
def _connect_retry(rw):
|
||||
"""Same brief retry the Agent applies: local XNET connections can report 'connection lost' when several threads connect at the same instant."""
|
||||
for attempt in range(6):
|
||||
try:
|
||||
return _orig_connect(rw)
|
||||
except Exception: # noqa: BLE001
|
||||
if attempt == 5:
|
||||
raise
|
||||
time.sleep(0.2 * (attempt + 1))
|
||||
|
||||
|
||||
base.connect = _connect_retry
|
||||
|
||||
|
||||
def insert_product(code, name, ean):
|
||||
conn = base.connect(True); cur = conn.cursor()
|
||||
cur.execute("INSERT INTO C000025 (CODIGO, PRODUTO, CODBARRA) VALUES (?, ?, ?)", (code, name, ean))
|
||||
conn.commit(); conn.close()
|
||||
|
||||
|
||||
def row(code):
|
||||
conn = base.connect(False); cur = conn.cursor()
|
||||
cur.execute("SELECT CLASSIFICACAO_FISCAL, CEST, IPI FROM C000025 WHERE CODIGO=?", (code,))
|
||||
r = cur.fetchone(); conn.close()
|
||||
return tuple(None if v is None else (str(v).strip() if not isinstance(v, float) else v) for v in r) if r else None
|
||||
|
||||
|
||||
def side_hashes():
|
||||
"""C000058 (ERP change journal) and C000100 (stock) are maintained by the SGV's OWN triggers on C000025 INSERT/UPDATE. Only rows of the test products (code 'Z...') may differ."""
|
||||
import hashlib
|
||||
conn = base.connect(False); cur = conn.cursor(); out = {}
|
||||
for table, col in (("C000058", "CODREGISTRO"), ("C000100", "CODPRODUTO")):
|
||||
cur.execute(f"SELECT * FROM {table} WHERE {col} NOT STARTING WITH 'Z'")
|
||||
h = hashlib.sha256()
|
||||
n = 0
|
||||
for r in cur.fetchall():
|
||||
h.update(repr(r).encode("utf-8", "replace")); n += 1
|
||||
out[table] = f"{n}:{h.hexdigest()[:16]}"
|
||||
conn.close()
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
n = int(sys.argv[sys.argv.index("--products") + 1]) if "--products" in sys.argv else 12
|
||||
checks = {}
|
||||
base.restore_copy()
|
||||
before_tables, _ = base.baseline()
|
||||
before_side = side_hashes()
|
||||
hub = InProcessHub()
|
||||
for suffix in ("", "-wal", "-shm"):
|
||||
try:
|
||||
(ROOT / "scratch_enrich" / f"enrich_e2e.db{suffix}").unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
store = E.EnrichmentStore(ROOT / "scratch_enrich" / "enrich_e2e.db")
|
||||
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: base.connect(False), write_conn=lambda: base.connect(True),
|
||||
create_reprocess=lambda codes: None)
|
||||
|
||||
class Stop:
|
||||
stop = False
|
||||
stopper = Stop()
|
||||
thread = threading.Thread(target=worker.run_forever, args=(stopper,), daemon=True)
|
||||
|
||||
# --- WAITING / DISABLED: zero Firebird activity, baseline NOT taken
|
||||
hub.features = {"features_configured": False}
|
||||
thread.start(); time.sleep(4)
|
||||
checks["WAITING_NO_BASELINE_NO_CALLS"] = store.meta("baseline_done") is None and hub.calls == 0
|
||||
hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "DISABLED"}; worker._features_at = -999; time.sleep(4)
|
||||
checks["DISABLED_NO_BASELINE_NO_CALLS"] = store.meta("baseline_done") is None and hub.calls == 0
|
||||
|
||||
# --- DRY_RUN: baseline, detect, resolve, report, zero writes
|
||||
hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "DRY_RUN"}; worker._features_at = -999; time.sleep(5)
|
||||
checks["BASELINE_TAKEN_ON_ACTIVATION"] = store.meta("baseline_done") == "1"
|
||||
hist = len(store.known())
|
||||
insert_product("ZD001", "PRODUTO DE TESTE ESPECIFICO 350ML", gtin("789200000001")); time.sleep(8)
|
||||
checks["DRY_RUN_ZERO_WRITES"] = row("ZD001") == (None, None, None) and hub.reports and hub.reports[-1]["fields"][0]["outcome"] == "DRY_RUN"
|
||||
|
||||
# --- ENABLED: N products inserted like an operator would, latency measured insert -> written
|
||||
hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}; worker._features_at = -999; worker._auth_cache = (None, 0.0)
|
||||
latencies = []
|
||||
for i in range(n):
|
||||
code, ean = f"ZE{i:03d}", gtin(f"78930000{i:04d}")
|
||||
t0 = time.monotonic()
|
||||
insert_product(code, "PRODUTO DE TESTE ESPECIFICO 350ML", ean)
|
||||
while time.monotonic() - t0 < 20 and row(code) == (None, None, None):
|
||||
time.sleep(0.1)
|
||||
latencies.append(round(time.monotonic() - t0, 2))
|
||||
time.sleep(0.5)
|
||||
ok_rows = [row(f"ZE{i:03d}") for i in range(n)]
|
||||
checks["FIELDS_WRITTEN"] = all(r and r[0] == "22021000" and r[1] == "0300700" and r[2] == 0.0 for r in ok_rows)
|
||||
time.sleep(6)
|
||||
writes_before = sum(1 for rep in hub.reports for f in rep["fields"] if f["outcome"] == "APPLIED")
|
||||
calls_before = hub.calls
|
||||
time.sleep(8) # a "second run": nothing pending, nothing re-written
|
||||
checks["SECOND_RUN_ZERO_WRITES"] = writes_before == sum(1 for rep in hub.reports for f in rep["fields"] if f["outcome"] == "APPLIED") and hub.calls - calls_before <= 0
|
||||
checks["HISTORICAL_KNOWN_COUNT"] = hist > 100
|
||||
|
||||
# --- human override incl. deleted value
|
||||
conn = base.connect(True); cur = conn.cursor()
|
||||
cur.execute("UPDATE C000025 SET CEST = NULL WHERE CODIGO = 'ZE000'"); cur.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL = '19059090' WHERE CODIGO = 'ZE001'"); conn.commit(); conn.close()
|
||||
time.sleep(E.OVERRIDE_CHECK_SECONDS + 6)
|
||||
checks["OVERRIDE_DELETED_VALUE"] = store.is_overridden("ZE000", "CEST") and store.is_overridden("ZE001", "CLASSIFICACAO_FISCAL")
|
||||
checks["OVERRIDE_REPORTED_TO_HUB"] = "CEST" in hub.overridden.get("ZE000", set())
|
||||
# ... and a later description edit does NOT bring the value back
|
||||
conn = base.connect(True); cur = conn.cursor(); cur.execute("UPDATE C000025 SET PRODUTO = 'PRODUTO DE TESTE ESPECIFICO 350ML ZERO' WHERE CODIGO = 'ZE000'"); conn.commit(); conn.close()
|
||||
time.sleep(10)
|
||||
checks["OVERRIDE_NEVER_REWRITTEN"] = row("ZE000")[1] in (None, "") and row("ZE001")[0] == "19059090"
|
||||
|
||||
# --- Hub offline -> pending -> recovers ; authorization revoked -> 0 writes
|
||||
hub.online = False
|
||||
insert_product("ZO001", "PRODUTO DE TESTE ESPECIFICO 350ML", gtin("789400000001")); time.sleep(8)
|
||||
checks["HUB_OFFLINE_PENDING_NO_WRITE"] = row("ZO001") == (None, None, None) and store.get("ZO001")["state"] in ("RETRY", "PENDING")
|
||||
hub.online = True
|
||||
with store.db:
|
||||
store.db.execute("UPDATE enrich_products SET next_attempt_at=0 WHERE code='ZO001'")
|
||||
time.sleep(8)
|
||||
checks["HUB_BACK_RECOVERS"] = row("ZO001") == ("22021000", "0300700", 0.0)
|
||||
stopper.stop = True
|
||||
thread.join(timeout=10)
|
||||
|
||||
# --- invariants on the real Firebird copy: only the 3 allowed columns of the new rows changed
|
||||
after_tables, _ = base.baseline()
|
||||
changed = base.diff_tables(before_tables, after_tables)
|
||||
checks["OTHER_TABLES_ONLY_ERP_TRIGGER_SIDE_EFFECTS_OF_TEST_ROWS"] = set(changed) <= {"C000025", "C000058", "C000100"} and side_hashes() == before_side
|
||||
conn = base.connect(False); cur = conn.cursor()
|
||||
cur.execute("SELECT COUNT(*) FROM C000025 WHERE CODIGO STARTING WITH 'Z' AND COD_ORIGEM_MERCADORIA IS NOT NULL AND TRIM(COD_ORIGEM_MERCADORIA) <> ''"); origin = cur.fetchone()[0]; conn.close()
|
||||
checks["ORIGIN_NEVER_WRITTEN"] = origin == 0
|
||||
result = {"copy_only": True, "products_enabled": n, "detect_to_written_seconds": {"p50": statistics.median(latencies), "p95": sorted(latencies)[int(0.95 * (len(latencies) - 1))], "max": max(latencies), "all": latencies},
|
||||
"hub_resolve_calls": hub.calls, "tables_changed": changed, "checks": checks, "ALL_PASS": all(checks.values()), "poll_seconds": worker.poll_seconds()}
|
||||
Path(ROOT / "docs" / "runs").mkdir(parents=True, exist_ok=True)
|
||||
(ROOT / "docs" / "runs" / "enrichment_copy_e2e.json").write_text(json.dumps(result, indent=1), encoding="utf-8")
|
||||
print(json.dumps(result, indent=1))
|
||||
sys.exit(0 if result["ALL_PASS"] else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,103 @@
|
||||
"""READ-ONLY study of the five lab ERP databases for the product enrichment: GTIN quality, NCM coverage, cross-database agreement, and KB seeding.
|
||||
|
||||
python scripts/enrichment_lab_kb.py [--seed-url sqlite:///kb.db] [--sample 40 --sample-out scratch/benchmark_gtins.json]
|
||||
|
||||
Firebird is opened READ-ONLY (no write path exists here). The KB seed goes ONLY to the database given by --seed-url (default: an in-memory SQLite, i.e. statistics only) and stores
|
||||
CLIENT_OBSERVATION evidence, never VERIFIED_KB by itself. Company-specific fiscal fields are not read at all.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from dotenv import load_dotenv # noqa: E402
|
||||
|
||||
load_dotenv(ROOT / ".env")
|
||||
import local_completion as lc # noqa: E402
|
||||
from sqlalchemy import create_engine, select # noqa: E402
|
||||
from sqlalchemy.orm import Session # noqa: E402
|
||||
|
||||
from central.app.enrichment import kb # noqa: E402
|
||||
from central.app.enrichment.gtin import normalize_gtin # noqa: E402
|
||||
from central.app.enrichment.observations import observe # noqa: E402
|
||||
from central.app.enrichment.resolver import ncm_is_official # noqa: E402
|
||||
from central.app.models import Base, Dataset, KbKnowledge, NcmEntry # noqa: E402
|
||||
|
||||
|
||||
def read_lab(name, path):
|
||||
conn = lc.connect(path)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT CODIGO, PRODUTO, GTIN, CLASSIFICACAO_FISCAL, CODBARRA FROM C000025")
|
||||
# the SGV operator types the EAN in CODBARRA; GTIN (fiscal) is mostly empty. The first VALID one wins (same rule as the Agent).
|
||||
return [{"CODIGO": r[0], "PRODUTO": (r[1] or "").strip(), "GTIN": next((x for x in (r[2], r[4]) if normalize_gtin(x)["status"] == "VALID_GTIN"), r[2] or r[4]), "NCM": r[3]} for r in cur.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def load_ncm_official(session, ncm_json: Path | None):
|
||||
"""Official NCM into the throw-away/seed session when it has none (parsed from the saved official JSON)."""
|
||||
if session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")):
|
||||
return
|
||||
from central.app.datasets.ncm import parse_ncm_payload
|
||||
from central.app.datasets.service import DatasetService
|
||||
from datetime import datetime, timezone
|
||||
payload = parse_ncm_payload(ncm_json.read_bytes())
|
||||
DatasetService(session).import_payload(payload, "official-json", datetime.now(timezone.utc))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--seed-url", default="sqlite://")
|
||||
ap.add_argument("--ncm-json", default=None, help="official Siscomex JSON (used only if the seed DB has no NCM dataset)")
|
||||
ap.add_argument("--sample", type=int, default=0)
|
||||
ap.add_argument("--sample-out", default=None)
|
||||
args = ap.parse_args()
|
||||
engine = create_engine(args.seed_url)
|
||||
Base.metadata.create_all(engine)
|
||||
dbs = dict(lc.allowlist())
|
||||
dbs["TESTESOFICIAIS"] = os.environ.get("FIREBIRD_DATABASE_PATH", str(ROOT / "TESTESOFICIAIS.FDB"))
|
||||
report, all_rows = {}, {}
|
||||
with Session(engine) as session:
|
||||
load_ncm_official(session, Path(args.ncm_json) if args.ncm_json else None)
|
||||
official = lambda v: ncm_is_official(session, v) # noqa: E731
|
||||
for name, path in dbs.items():
|
||||
if not Path(path).exists():
|
||||
report[name] = {"error": "missing"}
|
||||
continue
|
||||
rows = read_lab(name, path)
|
||||
all_rows[name] = rows
|
||||
status = Counter(normalize_gtin(r["GTIN"])["status"] for r in rows)
|
||||
stats = observe(session, rows, name, official)
|
||||
ncm_ok = sum(1 for r in rows if len("".join(c for c in str(r["NCM"] or "") if c.isdigit())) == 8 and official("".join(c for c in str(r["NCM"]) if c.isdigit())))
|
||||
report[name] = {"products": len(rows), "gtin_status": dict(status), "valid_gtin_rate": round(100 * status["VALID_GTIN"] / max(1, len(rows)), 1), "official_ncm_rate": round(100 * ncm_ok / max(1, len(rows)), 1), **stats}
|
||||
session.commit()
|
||||
by_gtin = defaultdict(set)
|
||||
for name, rows in all_rows.items():
|
||||
for r in rows:
|
||||
g = normalize_gtin(r["GTIN"])
|
||||
if g["status"] == "VALID_GTIN":
|
||||
by_gtin[g["key"]].add(name)
|
||||
shared = {k: v for k, v in by_gtin.items() if len(v) >= 2}
|
||||
conflicts = 0
|
||||
for kp in session.scalars(select(kb.KbProduct)):
|
||||
ncms = {r.value for r in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == kp.id, KbKnowledge.knowledge_key == "ncm"))}
|
||||
conflicts += len(ncms) > 1
|
||||
summary = {"databases": report, "distinct_valid_gtin": len(by_gtin), "gtin_in_2plus_databases": len(shared), "kb_products": session.scalar(select(kb.KbProduct.id).limit(1)) is not None and session.query(kb.KbProduct).count(),
|
||||
"kb_ncm_conflicts_between_clients": conflicts}
|
||||
if args.sample and args.sample_out:
|
||||
picks = sorted(by_gtin)[:: max(1, len(by_gtin) // args.sample)][: args.sample]
|
||||
names = {normalize_gtin(r["GTIN"])["key"]: r["PRODUTO"] for rows in all_rows.values() for r in rows if normalize_gtin(r["GTIN"])["status"] == "VALID_GTIN"}
|
||||
Path(args.sample_out).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.sample_out).write_text(json.dumps([{"gtin": k[1:] if k.startswith("0") else k, "erp_description": names[k]} for k in picks], ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
summary["sample_written"] = len(picks)
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Conservative LIVE smoke of the free providers (separate from the offline test suite): 1 lookup per provider on a famous, stable GTIN.
|
||||
|
||||
python scripts/enrichment_live_smoke.py
|
||||
|
||||
Goes through the real ProviderGateway (cache + quota + circuit breaker) on an in-memory database; tokens are never printed. Exit 0 when every CONFIGURED provider answered
|
||||
without an infrastructure error (NOT_FOUND is a valid answer; a rate limit is reported, not treated as a failure of our code).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from sqlalchemy import create_engine # noqa: E402
|
||||
from sqlalchemy.orm import Session # noqa: E402
|
||||
from sqlalchemy.pool import StaticPool # noqa: E402
|
||||
|
||||
from central.app.enrichment.providers import build_providers # noqa: E402
|
||||
from central.app.enrichment.quota import ProviderGateway # noqa: E402
|
||||
from central.app.models import Base # noqa: E402
|
||||
|
||||
GTIN = "7894900011517" # Coca-Cola 350 ml (BR): stable, widely catalogued
|
||||
|
||||
|
||||
def main():
|
||||
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(engine)
|
||||
settings = type("S", (), {"cosmos_token": os.environ.get("COSMOS_TOKEN", ""), "allow_paid_providers": False})()
|
||||
out, ok = [], True
|
||||
with Session(engine) as session:
|
||||
gateway = ProviderGateway(session, build_providers(settings))
|
||||
for provider in gateway.providers:
|
||||
result = gateway.lookup(provider, GTIN)
|
||||
row = {"provider": provider.name, "configured": provider.configured, "state": result.state, "reason": result.reason, "title": result.product.title if result.product else None}
|
||||
out.append(row)
|
||||
if provider.configured and result.state == "ERROR" and result.reason not in ("RATE_LIMITED",):
|
||||
ok = False
|
||||
print(json.dumps({"gtin": GTIN, "results": out, "OK": ok}, ensure_ascii=False, indent=1))
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,137 @@
|
||||
"""E2E of the enrichment across REAL HTTP: the real Agent worker (real Firebird COPY) <-> the real Hub container (dev PostgreSQL, real datasets, real providers).
|
||||
|
||||
docker compose up -d --build fiscal-hub # Hub at http://localhost:8000
|
||||
python scripts/enrichment_local_hub_e2e.py
|
||||
|
||||
Registers a throw-away dev Agent (AGENT_DEV_TOKEN), flips its per-installation features straight in the dev database (the panel route is covered by tests), seeds the KB of the dev Hub with a VERIFIED NCM for
|
||||
one test GTIN (two independent provider families), then walks: DISABLED -> DRY_RUN -> ENABLED -> DISABLED, an unknown GTIN (real providers, internet) and an invalid GTIN (no provider call).
|
||||
Writes go ONLY to scratch_rtc_e2e/TESTESOFICIAIS_RTC_COPY.FDB (guarded by rtc_copy_write_e2e.guard). The dev database receives test rows only.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from dotenv import load_dotenv # noqa: E402
|
||||
|
||||
load_dotenv(ROOT / ".env")
|
||||
import enrichment_copy_e2e as ce # noqa: E402 (also patches base.connect with the brief retry)
|
||||
from gti_fiscal_agent import enrichment as E # noqa: E402
|
||||
from gti_fiscal_agent.client import HubClient # noqa: E402
|
||||
|
||||
base = ce.base
|
||||
HUB = "http://127.0.0.1:8000" # NOT "localhost": on Windows it resolves ::1 first and adds ~500 ms per request
|
||||
TOKEN = os.environ["AGENT_DEV_TOKEN"]
|
||||
CONTAINER = "projeto-postgres-1"
|
||||
|
||||
|
||||
def sql(query):
|
||||
done = subprocess.run(["docker", "exec", CONTAINER, "psql", "-U", "gti", "-d", "gti_fiscal", "-At", "-c", query], capture_output=True, text=True, timeout=60)
|
||||
if done.returncode != 0:
|
||||
raise RuntimeError(done.stderr[-300:])
|
||||
return done.stdout.strip()
|
||||
|
||||
|
||||
def seed_kb(gtin14, description, ncm):
|
||||
code = f'''
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.enrichment import kb
|
||||
with SessionLocal() as s:
|
||||
p = kb.upsert_product(s, "{gtin14}", "{description}")
|
||||
kb.add_evidence(s, p, "ncm", "{ncm}", "provider:cosmos", family="PROVIDER:cosmos")
|
||||
kb.add_evidence(s, p, "ncm", "{ncm}", "provider:other", family="PROVIDER:other")
|
||||
s.commit()
|
||||
print("seeded")
|
||||
'''
|
||||
done = subprocess.run(["docker", "exec", "-i", "projeto-fiscal-hub-1", "python", "-"], input=code, capture_output=True, text=True, timeout=120, cwd=str(ROOT))
|
||||
assert "seeded" in done.stdout, done.stderr[-400:]
|
||||
|
||||
|
||||
def set_features(installation_id, mode, corrector=True):
|
||||
sql(f"UPDATE installations SET enrichment_mode='{mode}', corrector_enabled={'true' if corrector else 'false'}, features_configured=true WHERE installation_id='{installation_id}'")
|
||||
|
||||
|
||||
def main():
|
||||
checks = {}
|
||||
base.restore_copy()
|
||||
client = HubClient(HUB, TOKEN)
|
||||
try:
|
||||
installation_id = client.identity().get("installation_id") or str(uuid.uuid4()) # the dev Agent may already own an installation
|
||||
except Exception: # noqa: BLE001
|
||||
installation_id = str(uuid.uuid4())
|
||||
client.heartbeat_metadata({"installation_id": installation_id, "status": "IDLE", "agent_version": "1.3.0", "display_name": "E2E-ENRICH", "capabilities": {"PRODUCT_ENRICHMENT_WRITE_V1": True}})
|
||||
set_features(installation_id, "DISABLED") # start every run from the default
|
||||
checks["REGISTERED_DEFAULT_DISABLED"] = client.authorizations()["features"]["enrichment_mode"] == "DISABLED"
|
||||
gtin_ok = ce.gtin("789500000001")
|
||||
seed_kb("0" + gtin_ok, "REFRIGERANTE COLA LATA 350ML", "22021000")
|
||||
for suffix in ("", "-wal", "-shm"):
|
||||
try:
|
||||
(ROOT / "scratch_enrich" / f"enrich_hub_e2e.db{suffix}").unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
store = E.EnrichmentStore(ROOT / "scratch_enrich" / "enrich_hub_e2e.db")
|
||||
worker = E.EnrichmentWorker(hub=client, store=store, read_conn=lambda: base.connect(False), write_conn=lambda: base.connect(True), create_reprocess=lambda codes: client.create_job("REPROCESS_PRODUCTS", {"codigos": list(codes), "reason": "ENRICHMENT"}))
|
||||
|
||||
class Stop:
|
||||
stop = False
|
||||
stopper = Stop()
|
||||
threading.Thread(target=worker.run_forever, args=(stopper,), daemon=True).start()
|
||||
|
||||
def refresh():
|
||||
worker._features_at = -999
|
||||
worker._auth_cache = (None, 0.0)
|
||||
|
||||
time.sleep(4)
|
||||
checks["DISABLED_NO_ACTIVITY"] = store.meta("baseline_done") is None
|
||||
set_features(installation_id, "DRY_RUN"); refresh(); time.sleep(6)
|
||||
checks["BASELINE_ON_ACTIVATION"] = store.meta("baseline_done") == "1"
|
||||
ce.insert_product("HD001", "REFRIGERANTE COLA LATA 350ML", gtin_ok); time.sleep(8)
|
||||
checks["DRY_RUN_ZERO_WRITES"] = ce.row("HD001") == (None, None, None)
|
||||
audits = int(sql(f"SELECT count(*) FROM product_field_audits a JOIN installations i ON i.id=a.installation_id WHERE i.installation_id='{installation_id}' AND a.outcome='DRY_RUN' AND a.product_code='HD001'"))
|
||||
checks["DRY_RUN_AUDITED_IN_HUB"] = audits >= 1
|
||||
set_features(installation_id, "ENABLED"); refresh()
|
||||
ce.insert_product("HE001", "REFRIGERANTE COLA LATA 350ML", gtin_ok)
|
||||
t0 = time.monotonic()
|
||||
while time.monotonic() - t0 < 25 and ce.row("HE001")[0] is None:
|
||||
time.sleep(0.2)
|
||||
latency = round(time.monotonic() - t0, 2)
|
||||
r = ce.row("HE001")
|
||||
checks["ENABLED_NCM_WRITTEN"] = r[0] == "22021000"
|
||||
time.sleep(3) # the Agent reports the audit right after the write
|
||||
applied = sql(f"SELECT field||':'||coalesce(new_value,'') FROM product_field_audits a JOIN installations i ON i.id=a.installation_id WHERE i.installation_id='{installation_id}' AND a.outcome='APPLIED' AND a.product_code='HE001' ORDER BY a.id")
|
||||
checks["WRITTEN_FIELDS_MATCH_HUB_AUDIT"] = ("CLASSIFICACAO_FISCAL:22021000" in applied) and all(v in applied for v in ([f"CEST:{r[1]}"] if r[1] else []) + ([f"IPI:{r[2]}"] if r[2] not in (None,) else []))
|
||||
time.sleep(6)
|
||||
hd = ce.row("HD001")
|
||||
checks["EARLIER_DRY_RUN_PRODUCT_STILL_UNTOUCHED"] = hd == (None, None, None)
|
||||
# unknown GTIN goes to the REAL providers (internet); nothing may be written, a human blocker must appear
|
||||
unknown = ce.gtin("789600000009")
|
||||
ce.insert_product("HE002", "PRODUTO INEXISTENTE ESPECIAL XYZ 123", unknown); time.sleep(14)
|
||||
checks["UNKNOWN_GTIN_NO_WRITE"] = ce.row("HE002") == (None, None, None)
|
||||
blockers = sql("SELECT reason_code FROM blocked_products WHERE field='ENRICHMENT' AND product_code='HE002'")
|
||||
checks["UNKNOWN_GTIN_BLOCKER_PT"] = blockers in ("NOT_FOUND", "SINGLE_WEAK_SOURCE", "PENDING_PROVIDER_QUOTA", "PENDING_PROVIDERS_OFFLINE") or blockers.startswith("NOT_FOUND")
|
||||
bad = unknown[:-1] + str((int(unknown[-1]) + 1) % 10)
|
||||
before_req = sql("SELECT coalesce(sum(requests),0) FROM provider_usage_daily")
|
||||
ce.insert_product("HE003", "PRODUTO GTIN INVALIDO", bad); time.sleep(8)
|
||||
checks["INVALID_GTIN_NO_PROVIDER_CALL"] = sql("SELECT coalesce(sum(requests),0) FROM provider_usage_daily") == before_req and ce.row("HE003") == (None, None, None)
|
||||
# feature switched off centrally: the next new product is not touched
|
||||
set_features(installation_id, "DISABLED"); refresh(); time.sleep(4)
|
||||
ce.insert_product("HE004", "REFRIGERANTE COLA LATA 350ML", gtin_ok); time.sleep(8)
|
||||
checks["DISABLED_AGAIN_ZERO_WRITES"] = ce.row("HE004") == (None, None, None)
|
||||
# corrector off + enrichment on: independent switches
|
||||
set_features(installation_id, "ENABLED", corrector=False); refresh(); time.sleep(3)
|
||||
checks["CORRECTOR_OFF_NO_JOBS"] = client.next_job() is None
|
||||
stopper.stop = True
|
||||
result = {"copy_only": True, "hub": HUB, "detect_to_written_seconds_enabled": latency, "checks": checks, "ALL_PASS": all(checks.values())}
|
||||
(ROOT / "docs" / "runs" / "enrichment_local_hub_e2e.json").write_text(json.dumps(result, indent=1), encoding="utf-8")
|
||||
print(json.dumps(result, indent=1))
|
||||
sys.exit(0 if result["ALL_PASS"] else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user