86 lines
5.0 KiB
Python
86 lines
5.0 KiB
Python
"""Conservative provider benchmark on REAL GTINs taken (read-only) from the lab ERP databases. Free quotas only, paced below documented limits.
|
|
|
|
python scripts/enrichment_benchmark.py --sample scratch_enrich/benchmark_gtins.json --out docs/PRODUCT_PROVIDER_BENCHMARK_V1.json [--limit 30]
|
|
|
|
Each provider gets at most --limit lookups (UPCitemdb: also <= 12 per 30 s burst window; Open Food Facts: <= 12/min). Cosmos is measured only when COSMOS_TOKEN is set.
|
|
Reports sample_size / found / not_found / description / brand / NCM / CEST / latency / error_rate per provider. Secrets are never printed.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import statistics
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
import httpx # noqa: E402
|
|
|
|
from central.app.enrichment.providers import Cosmos, Oscbr, OpenBeautyFacts, OpenFoodFacts, OpenProductsFacts, ProviderError, UpcItemDb, USER_AGENT # noqa: E402
|
|
|
|
PACE = {"oscbr": 3.2, "upcitemdb": 2.6, "openfoodfacts": 5.2, "openproductsfacts": 5.2, "openbeautyfacts": 5.2, "cosmos": 1.0} # seconds between calls: comfortably under the documented burst / per-minute limits
|
|
|
|
|
|
def bench(provider, items):
|
|
out = {"provider": provider.name, "configured": provider.configured, "sample_size": 0, "found": 0, "not_found": 0, "with_description": 0, "with_brand": 0, "with_ncm": 0, "with_cest": 0, "errors": 0, "error_kinds": {}, "latencies_ms": [], "per_gtin": {}}
|
|
if not provider.configured:
|
|
out["status"] = "NOT_CONFIGURED"
|
|
return out
|
|
for item in items:
|
|
started = time.monotonic()
|
|
try:
|
|
product = provider.lookup_gtin(item["gtin"])
|
|
except ProviderError as error:
|
|
out["errors"] += 1
|
|
out["error_kinds"][error.kind] = out["error_kinds"].get(error.kind, 0) + 1
|
|
if error.kind in ("RATE_LIMITED", "AUTH_ERROR"):
|
|
break # never insist against a limit
|
|
time.sleep(PACE[provider.name])
|
|
continue
|
|
out["sample_size"] += 1
|
|
out["latencies_ms"].append(int((time.monotonic() - started) * 1000))
|
|
out["per_gtin"][item["gtin"]] = {"found": product.found, "description": bool(product.title), "ncm": bool(product.ncm), "cest": bool(product.cest)}
|
|
if product.found:
|
|
out["found"] += 1
|
|
out["with_description"] += bool(product.title)
|
|
out["with_brand"] += bool(product.brand)
|
|
out["with_ncm"] += bool(product.ncm)
|
|
out["with_cest"] += bool(product.cest)
|
|
else:
|
|
out["not_found"] += 1
|
|
time.sleep(PACE[provider.name])
|
|
n = out["sample_size"] + out["errors"]
|
|
lat = sorted(out.pop("latencies_ms"))
|
|
out.update({"error_rate": round(out["errors"] / n, 3) if n else None, "latency_p50_ms": statistics.median(lat) if lat else None, "latency_p95_ms": lat[int(0.95 * (len(lat) - 1))] if lat else None,
|
|
"found_rate": round(out["found"] / out["sample_size"], 3) if out["sample_size"] else None, "status": "MEASURED"})
|
|
return out
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--sample", required=True)
|
|
ap.add_argument("--out", required=True)
|
|
ap.add_argument("--limit", type=int, default=30)
|
|
ap.add_argument("--only", default="", help="comma list of provider names; results are MERGED into an existing --out")
|
|
args = ap.parse_args()
|
|
items = json.loads(Path(args.sample).read_text(encoding="utf-8"))[: args.limit]
|
|
client = httpx.Client(timeout=8.0, headers={"User-Agent": USER_AGENT})
|
|
providers = [Cosmos(client, os.environ.get("COSMOS_TOKEN")), Oscbr(client, None, os.environ.get("OSCBR_USER"), os.environ.get("OSCBR_PASSWORD")), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
|
|
only = {n for n in args.only.split(",") if n}
|
|
out = Path(args.out)
|
|
result = json.loads(out.read_text(encoding="utf-8")) if only and out.exists() else {"measured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "sample_source": "lab ERP databases (read-only), valid GTINs only", "limit_per_provider": args.limit, "providers": []}
|
|
measured = [bench(p, items) for p in providers if not only or p.name in only]
|
|
result["providers"] = [p for p in result["providers"] if p["provider"] not in {m["provider"] for m in measured}] + measured
|
|
per = [p.get("per_gtin", {}) for p in result["providers"]]
|
|
tested = sorted({g for d in per for g in d})
|
|
def rate(key):
|
|
return round(sum(1 for g in tested if any(d.get(g, {}).get(key) for d in per)) / len(tested), 3) if tested else None
|
|
result["chain"] = {"gtins_tested": len(tested), "ANY_PROVIDER_FOUND_RATE": rate("found"), "ANY_PROVIDER_NCM_RATE": rate("ncm"), "ANY_PROVIDER_CEST_RATE": rate("cest"), "ANY_PROVIDER_DESCRIPTION_RATE": rate("description")}
|
|
Path(args.out).write_text(json.dumps(result, ensure_ascii=False, indent=1), encoding="utf-8")
|
|
print(json.dumps(result, ensure_ascii=False, indent=1))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|