feat: product enrichment observability, lab KB study, benchmark, copy + local-Hub E2E, docs; 1.3.0

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

scripts/enrichment_benchmark.py

@@ -0,0 +1,79 @@
+"""Conservative provider benchmark on REAL GTINs taken (read-only) from the lab ERP databases. Free quotas only, paced below documented limits.
+
+    python scripts/enrichment_benchmark.py --sample scratch_enrich/benchmark_gtins.json --out docs/PRODUCT_PROVIDER_BENCHMARK_V1.json [--limit 30]
+
+Each provider gets at most --limit lookups (UPCitemdb: also <= 12 per 30 s burst window; Open Food Facts: <= 12/min). Cosmos is measured only when COSMOS_TOKEN is set.
+Reports sample_size / found / not_found / description / brand / NCM / CEST / latency / error_rate per provider. Secrets are never printed.
+"""
+import argparse
+import json
+import os
+import statistics
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+import httpx  # noqa: E402
+
+from central.app.enrichment.providers import Cosmos, OpenBeautyFacts, OpenFoodFacts, OpenProductsFacts, ProviderError, UpcItemDb, USER_AGENT  # noqa: E402
+
+PACE = {"upcitemdb": 2.6, "openfoodfacts": 5.2, "openproductsfacts": 5.2, "openbeautyfacts": 5.2, "cosmos": 1.0}      # seconds between calls: comfortably under the documented burst / per-minute limits
+
+
+def bench(provider, items):
+    out = {"provider": provider.name, "configured": provider.configured, "sample_size": 0, "found": 0, "not_found": 0, "with_description": 0, "with_brand": 0, "with_ncm": 0, "with_cest": 0, "errors": 0, "error_kinds": {}, "latencies_ms": []}
+    if not provider.configured:
+        out["status"] = "NOT_CONFIGURED"
+        return out
+    for item in items:
+        started = time.monotonic()
+        try:
+            product = provider.lookup_gtin(item["gtin"])
+        except ProviderError as error:
+            out["errors"] += 1
+            out["error_kinds"][error.kind] = out["error_kinds"].get(error.kind, 0) + 1
+            if error.kind in ("RATE_LIMITED", "AUTH_ERROR"):
+                break                                                            # never insist against a limit
+            time.sleep(PACE[provider.name])
+            continue
+        out["sample_size"] += 1
+        out["latencies_ms"].append(int((time.monotonic() - started) * 1000))
+        if product.found:
+            out["found"] += 1
+            out["with_description"] += bool(product.title)
+            out["with_brand"] += bool(product.brand)
+            out["with_ncm"] += bool(product.ncm)
+            out["with_cest"] += bool(product.cest)
+        else:
+            out["not_found"] += 1
+        time.sleep(PACE[provider.name])
+    n = out["sample_size"] + out["errors"]
+    lat = sorted(out.pop("latencies_ms"))
+    out.update({"error_rate": round(out["errors"] / n, 3) if n else None, "latency_p50_ms": statistics.median(lat) if lat else None, "latency_p95_ms": lat[int(0.95 * (len(lat) - 1))] if lat else None,
+                "found_rate": round(out["found"] / out["sample_size"], 3) if out["sample_size"] else None, "status": "MEASURED"})
+    return out
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--sample", required=True)
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--limit", type=int, default=30)
+    ap.add_argument("--only", default="", help="comma list of provider names; results are MERGED into an existing --out")
+    args = ap.parse_args()
+    items = json.loads(Path(args.sample).read_text(encoding="utf-8"))[: args.limit]
+    client = httpx.Client(timeout=8.0, headers={"User-Agent": USER_AGENT})
+    providers = [Cosmos(client, os.environ.get("COSMOS_TOKEN")), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
+    only = {n for n in args.only.split(",") if n}
+    out = Path(args.out)
+    result = json.loads(out.read_text(encoding="utf-8")) if only and out.exists() else {"measured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "sample_source": "lab ERP databases (read-only), valid GTINs only", "limit_per_provider": args.limit, "providers": []}
+    measured = [bench(p, items) for p in providers if not only or p.name in only]
+    result["providers"] = [p for p in result["providers"] if p["provider"] not in {m["provider"] for m in measured}] + measured
+    Path(args.out).write_text(json.dumps(result, ensure_ascii=False, indent=1), encoding="utf-8")
+    print(json.dumps(result, ensure_ascii=False, indent=1))
+
+
+if __name__ == "__main__":
+    main()