"""Central quota / circuit breaker / cache / single-flight for external product providers. Quota belongs to the HUB: 70 Agents never spend it uncoordinated.""" from __future__ import annotations import threading import time from collections import defaultdict, deque from dataclasses import dataclass from datetime import datetime, timedelta, timezone from sqlalchemy import select from ..models import GtinLookupCache, ProviderState, ProviderUsageDaily from .providers import Provider, ProviderError, ProviderProduct POSITIVE_TTL = timedelta(days=30) NEGATIVE_TTL = timedelta(hours=24) CIRCUIT_BASE = timedelta(seconds=60) CIRCUIT_MAX = timedelta(minutes=15) FAILURES_TO_OPEN = 3 _windows: dict[str, deque] = defaultdict(deque) # per-process sliding windows (burst / per-minute governors) _flights: dict[tuple, threading.Lock] = {} _flights_guard = threading.Lock() def utcnow() -> datetime: return datetime.now(timezone.utc) def _aware(value): if value is not None and value.tzinfo is None: return value.replace(tzinfo=timezone.utc) return value def next_utc_midnight(now: datetime) -> datetime: return (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) @dataclass class Outcome: provider: str state: str # CACHE_HIT | FOUND | NOT_FOUND | SKIPPED | ERROR product: ProviderProduct | None = None reason: str | None = None # NOT_CONFIGURED | CIRCUIT_OPEN | RATE_LIMITED | RATE_WINDOW | TIMEOUT | AUTH_ERROR | NETWORK | SERVER_ERROR | ... retry_at: datetime | None = None from_cache: bool = False class QuotaManager: def __init__(self, session, now=utcnow): self.session, self.now = session, now # ---- state ------------------------------------------------------------------------------------------------------------------------------------- def state(self, provider: Provider) -> ProviderState: row = self.session.get(ProviderState, provider.name) if row is None: row = ProviderState(provider=provider.name, configured=provider.configured, status="ONLINE" if provider.configured else "NOT_CONFIGURED", daily_limit=provider.daily_limit, rate_per_minute=provider.rate_per_minute, quota_kind="LOCAL_ESTIMATE" if provider.daily_limit else "DOCUMENTED_RATE") self.session.add(row) self.session.flush() row.configured = provider.configured if not provider.configured: row.status = "NOT_CONFIGURED" elif row.status == "NOT_CONFIGURED": row.status = "ONLINE" row.daily_limit = provider.daily_limit row.rate_per_minute = provider.rate_per_minute return row def usage(self, provider_name: str, day: str | None = None) -> ProviderUsageDaily: day = day or self.now().strftime("%Y-%m-%d") row = self.session.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == provider_name, ProviderUsageDaily.day == day)) if row is None: row = ProviderUsageDaily(provider=provider_name, day=day) self.session.add(row) self.session.flush() return row # ---- decision ---------------------------------------------------------------------------------------------------------------------------------- def allow(self, provider: Provider) -> tuple[bool, str | None, datetime | None]: state, now = self.state(provider), self.now() if not provider.configured: return False, "NOT_CONFIGURED", None if state.circuit_open_until and _aware(state.circuit_open_until) > now: return False, "CIRCUIT_OPEN", _aware(state.circuit_open_until) usage = self.usage(provider.name) if provider.daily_limit and state.quota_kind == "REMOTE_QUOTA" and state.remaining is not None and state.remaining <= 0 and state.reset_at and _aware(state.reset_at) > now: return False, "RATE_LIMITED", _aware(state.reset_at) if provider.daily_limit and usage.requests >= provider.daily_limit and state.quota_kind != "REMOTE_QUOTA": state.status = "RATE_LIMITED" return False, "RATE_LIMITED", state.reset_at and _aware(state.reset_at) or next_utc_midnight(now) window = (provider.burst[1] if provider.burst else 60) limit = (provider.burst[0] if provider.burst else provider.rate_per_minute) if limit: q, cutoff = _windows[provider.name], time.monotonic() - window while q and q[0] < cutoff: q.popleft() if len(q) >= limit: return False, "RATE_WINDOW", now + timedelta(seconds=max(1, int(window - (time.monotonic() - q[0])))) return True, None, None # ---- accounting -------------------------------------------------------------------------------------------------------------------------------- def _remote_quota(self, state: ProviderState, headers: dict): remaining, limit, reset = headers.get("x-ratelimit-remaining"), headers.get("x-ratelimit-limit"), headers.get("x-ratelimit-reset") if remaining is not None and str(remaining).isdigit(): state.remaining = int(remaining) state.quota_kind = "REMOTE_QUOTA" if limit and str(limit).isdigit(): state.daily_limit = int(limit) if reset and str(reset).isdigit(): state.reset_at = datetime.fromtimestamp(int(reset), tz=timezone.utc) def _sent(self, provider: Provider): q = _windows[provider.name] q.append(time.monotonic()) def record_ok(self, provider: Provider, result: ProviderProduct): state, usage, now = self.state(provider), self.usage(provider.name), self.now() self._sent(provider) usage.requests += 1 usage.success += 1 if not result.found: usage.not_found += 1 state.consecutive_failures = 0 state.circuit_open_until = None state.status = "ONLINE" state.last_request_at = state.last_success_at = now state.last_latency_ms = result.latency_ms or None self._remote_quota(state, {k.lower(): v for k, v in (result.headers or {}).items()}) if provider.daily_limit and state.quota_kind != "REMOTE_QUOTA": state.remaining = max(0, provider.daily_limit - usage.requests) state.reset_at = next_utc_midnight(now) self.session.commit() def record_error(self, provider: Provider, error: ProviderError): state, usage, now = self.state(provider), self.usage(provider.name), self.now() self._sent(provider) usage.requests += 1 state.last_request_at, state.last_error_at = now, now state.last_error = f"{error.kind}: {error.detail}"[:160] state.consecutive_failures += 1 headers = {k.lower(): v for k, v in (error.headers or {}).items()} self._remote_quota(state, headers) if error.kind == "RATE_LIMITED": usage.rate_limited += 1 state.status = "RATE_LIMITED" wait = timedelta(seconds=error.retry_after) if error.retry_after else None state.circuit_open_until = now + (wait or (state.reset_at and max(timedelta(minutes=5), _aware(state.reset_at) - now) or timedelta(minutes=15))) if state.reset_at is None: state.reset_at = state.circuit_open_until elif error.kind == "AUTH_ERROR": usage.errors += 1 state.status = "AUTH_ERROR" state.circuit_open_until = now + timedelta(hours=1) else: if error.kind == "TIMEOUT": usage.timeouts += 1 usage.errors += 1 state.status = "DEGRADED" if state.consecutive_failures >= FAILURES_TO_OPEN: delay = min(CIRCUIT_MAX, CIRCUIT_BASE * (2 ** (state.consecutive_failures - FAILURES_TO_OPEN))) state.circuit_open_until = now + delay state.status = "OFFLINE" self.session.commit() def record_cache(self, provider_name: str, hit: bool): usage = self.usage(provider_name) if hit: usage.cache_hits += 1 else: usage.cache_misses += 1 class ProviderGateway: """lookup(provider, gtin): cache -> single-flight -> quota/circuit -> HTTP -> account -> cache.""" def __init__(self, session, providers: list[Provider], now=utcnow): self.session, self.providers, self.now = session, providers, now self.quota = QuotaManager(session, now) def _cached(self, provider: Provider, gtin: str) -> ProviderProduct | None | bool: row = self.session.scalar(select(GtinLookupCache).where(GtinLookupCache.gtin == gtin, GtinLookupCache.provider == provider.name)) if row is None or _aware(row.expires_at) <= self.now(): return False if not row.found: return None d = row.data or {} return ProviderProduct(provider.name, gtin, True, title=d.get("title"), brand=d.get("brand"), category=d.get("category"), quantity=d.get("quantity"), ncm=d.get("ncm"), cest=d.get("cest"), raw_hash=d.get("raw_hash", "")) def _store(self, provider: Provider, gtin: str, product: ProviderProduct): now = self.now() ttl = POSITIVE_TTL if product.found else NEGATIVE_TTL row = self.session.scalar(select(GtinLookupCache).where(GtinLookupCache.gtin == gtin, GtinLookupCache.provider == provider.name)) data = {**product.as_data(), "raw_hash": product.raw_hash} if row is None: self.session.add(GtinLookupCache(gtin=gtin, provider=provider.name, found=product.found, data=data, fetched_at=now, expires_at=now + ttl)) else: row.found, row.data, row.fetched_at, row.expires_at = product.found, data, now, now + ttl self.session.commit() def lookup(self, provider: Provider, gtin: str) -> Outcome: hit = self._cached(provider, gtin) if hit is not False: self.quota.record_cache(provider.name, True) self.session.commit() return Outcome(provider.name, "CACHE_HIT", hit or None, from_cache=True) if hit else Outcome(provider.name, "NOT_FOUND", None, from_cache=True) with _flights_guard: lock = _flights.setdefault((provider.name, gtin), threading.Lock()) with lock: # 10 installations, same EAN -> 1 external call, the rest wait for the same answer self.session.expire_all() hit = self._cached(provider, gtin) if hit is not False: self.quota.record_cache(provider.name, True) self.session.commit() return Outcome(provider.name, "CACHE_HIT", hit or None, from_cache=True) if hit else Outcome(provider.name, "NOT_FOUND", None, from_cache=True) self.quota.record_cache(provider.name, False) ok, reason, retry_at = self.quota.allow(provider) if not ok: self.session.commit() return Outcome(provider.name, "SKIPPED", None, reason, retry_at) try: product = provider.lookup_gtin(gtin) except ProviderError as error: self.quota.record_error(provider, error) return Outcome(provider.name, "ERROR", None, error.kind, self.quota.state(provider).circuit_open_until) self.quota.record_ok(provider, product) self._store(provider, gtin, product) return Outcome(provider.name, "FOUND" if product.found else "NOT_FOUND", product if product.found else None)