111 lines
6.7 KiB
Python
111 lines
6.7 KiB
Python
"""Gradual, quota-safe verification of locally CORROBORATED GTINs by an NCM-bearing free provider (OSCBR). Never spends the operational reserve.
|
|
|
|
Rules: only CORROBORATED single-NCM GTINs (never VERIFIED_KB, never CONFLICT, never an invalid GTIN); each GTIN is checked once per `RECHECK_DAYS` (next_checked_at kept in the KB evidence, plus the
|
|
provider cache); a provider answer is EVIDENCE only - the promotion/conflict decision is kb.evaluate's (local strong evidence + independent provider NCM + vigente + zero conflict).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import select
|
|
|
|
from ..models import KbKnowledge, KbProduct
|
|
from . import kb
|
|
from .gtin import normalize_gtin
|
|
|
|
RECHECK_DAYS = 30
|
|
|
|
|
|
def _now():
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def select_sample(session, is_official, limit: int, provider: str = "oscbr") -> list[dict]:
|
|
"""Priority: multi-client, more documents, more years; current-vigente NCM; zero conflict; not checked recently."""
|
|
ids = list(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.status == kb.CORROBORATED).distinct()))
|
|
by_product: dict[int, list] = {}
|
|
for start in range(0, len(ids), 500): # every NCM row of those products: a second (even weaker) value already disqualifies the GTIN
|
|
for r in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.kb_product_id.in_(ids[start:start + 500]))):
|
|
by_product.setdefault(r.kb_product_id, []).append(r)
|
|
now = _now().isoformat()
|
|
picks = []
|
|
for pid, items in by_product.items():
|
|
valid = [i for i in items if is_official(i.value)]
|
|
if len(valid) != 1 or valid[0].status != kb.CORROBORATED:
|
|
continue # conflicts / no vigente NCM / already verified are not asked
|
|
row = valid[0]
|
|
info = (row.evidence or {}).get("info") or {}
|
|
if info.get(f"{provider}_next_checked_at", "") > now:
|
|
continue
|
|
prod = session.get(KbProduct, pid)
|
|
if not prod or normalize_gtin(prod.gtin[1:] if prod.gtin and prod.gtin.startswith("0") else prod.gtin)["status"] != "VALID_GTIN":
|
|
continue
|
|
summary = kb._lineage_summary([row])
|
|
if any(f.startswith("PROVIDER:" + provider) for f in ((row.evidence or {}).get("families") or {})):
|
|
continue
|
|
picks.append({"gtin_key": prod.gtin, "ncm": row.value, "clients": len(summary["clients"]), "docs": summary["docs"], "years": len(summary["years"]), "product_id": pid})
|
|
picks.sort(key=lambda p: (-p["clients"], -p["docs"], -p["years"], p["gtin_key"]))
|
|
return picks[:limit]
|
|
|
|
|
|
def apply_outcome(session, gtin_key: str, provider: str, outcome, is_official) -> dict:
|
|
"""Turns one gateway Outcome into KB evidence. Returns the per-GTIN measurement record."""
|
|
product = kb.find_by_gtin(session, gtin_key)
|
|
rec = {"gtin": gtin_key[-13:] if gtin_key.startswith("0") else gtin_key, "state": outcome.state, "reason": outcome.reason, "found": False, "ncm": None, "cest": None, "description": None, "brand": None,
|
|
"local_ncm": None, "official_valid": None, "kb_status": None, "external_conflict": False}
|
|
local_rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm"))) if product else []
|
|
rec["local_ncm"] = local_rows[0].value if local_rows else None
|
|
stamp = (_now() + timedelta(days=RECHECK_DAYS)).isoformat()
|
|
if outcome.state in ("FOUND", "CACHE_HIT") and outcome.product:
|
|
p = outcome.product
|
|
rec.update({"found": True, "ncm": p.ncm, "cest": p.cest, "description": p.title, "brand": p.brand})
|
|
if p.ncm:
|
|
rec["official_valid"] = bool(is_official(p.ncm))
|
|
kb.add_evidence(session, product, "ncm", p.ncm, f"provider:{provider}"[:32], family=f"PROVIDER:{provider}", info={"raw_hash": p.raw_hash})
|
|
if p.cest:
|
|
kb.add_evidence(session, product, "cest", p.cest, f"provider:{provider}"[:32], family=f"PROVIDER:{provider}", info={"raw_hash": p.raw_hash})
|
|
if p.brand and not product.brand:
|
|
product.brand = p.brand
|
|
if p.title:
|
|
kb.add_alias(session, product, p.title.upper()[:200])
|
|
ev = kb.evaluate(session, product, "ncm", is_official)
|
|
rec["kb_status"] = ev["status"]
|
|
rec["external_conflict"] = bool(ev["status"] == kb.CONFLICT and rec["ncm"])
|
|
if outcome.state in ("FOUND", "CACHE_HIT", "NOT_FOUND"): # remember the check so it is not repeated (skipped/errors are retried later)
|
|
for r in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm")):
|
|
r.evidence = {**(r.evidence or {}), "info": {**((r.evidence or {}).get("info") or {}), f"{provider}_next_checked_at": stamp}}
|
|
return rec
|
|
|
|
|
|
def provider_query(key: str) -> str:
|
|
"""Canonical form sent to providers: GTIN-8 stays 8 digits, UPC-12/EAN-13 as 13 digits (same rule as the resolver), GTIN-14 with an indicator digit stays 14."""
|
|
if len(key) == 14 and key.startswith("000000"):
|
|
return key[-8:]
|
|
if len(key) == 14 and key.startswith("0"):
|
|
return key[1:]
|
|
return key
|
|
|
|
|
|
def run(session, gateway, provider, *, limit: int, reserve_percent: int, is_official, dry_run: bool = False, wait_rate_window: bool = True) -> dict:
|
|
"""One backfill pass. dry_run only lists the sample (no request)."""
|
|
sample = select_sample(session, is_official, limit, provider.name)
|
|
report = {"provider": provider.name, "sample_size": len(sample), "requests": 0, "records": [], "stopped": None, "dry_run": dry_run}
|
|
if dry_run:
|
|
report["records"] = [{k: v for k, v in s.items() if k != "product_id"} for s in sample]
|
|
return report
|
|
for item in sample:
|
|
query = provider_query(item["gtin_key"])
|
|
outcome = gateway.lookup(provider, query, "BACKFILL", reserve_percent)
|
|
while outcome.state == "SKIPPED" and outcome.reason == "RATE_WINDOW" and wait_rate_window: # per-minute governor: wait for the window, never exceed it
|
|
time.sleep(max(1, min(65, int((outcome.retry_at - _now()).total_seconds()) + 1)) if outcome.retry_at else 12)
|
|
outcome = gateway.lookup(provider, query, "BACKFILL", reserve_percent)
|
|
if outcome.state == "SKIPPED":
|
|
report["stopped"] = outcome.reason # quota / reserve / per-minute window: stop, never insist
|
|
break
|
|
if not outcome.from_cache:
|
|
report["requests"] += 1
|
|
report["records"].append(apply_outcome(session, item["gtin_key"], provider.name, outcome, is_official))
|
|
session.commit()
|
|
return report
|