231 lines
12 KiB
Python
231 lines
12 KiB
Python
"""Central quota / circuit breaker / cache / single-flight for external product providers. Quota belongs to the HUB: 70 Agents never spend it uncoordinated."""
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
import time
|
|
from collections import defaultdict, deque
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import select
|
|
|
|
from ..models import GtinLookupCache, ProviderState, ProviderUsageDaily
|
|
from .providers import Provider, ProviderError, ProviderProduct
|
|
|
|
POSITIVE_TTL = timedelta(days=30)
|
|
NEGATIVE_TTL = timedelta(hours=24)
|
|
CIRCUIT_BASE = timedelta(seconds=60)
|
|
CIRCUIT_MAX = timedelta(minutes=15)
|
|
FAILURES_TO_OPEN = 3
|
|
|
|
_windows: dict[str, deque] = defaultdict(deque) # per-process sliding windows (burst / per-minute governors)
|
|
_flights: dict[tuple, threading.Lock] = {}
|
|
_flights_guard = threading.Lock()
|
|
|
|
|
|
def utcnow() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def _aware(value):
|
|
if value is not None and value.tzinfo is None:
|
|
return value.replace(tzinfo=timezone.utc)
|
|
return value
|
|
|
|
|
|
def next_utc_midnight(now: datetime) -> datetime:
|
|
return (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
|
|
|
|
|
|
@dataclass
|
|
class Outcome:
|
|
provider: str
|
|
state: str # CACHE_HIT | FOUND | NOT_FOUND | SKIPPED | ERROR
|
|
product: ProviderProduct | None = None
|
|
reason: str | None = None # NOT_CONFIGURED | CIRCUIT_OPEN | RATE_LIMITED | RATE_WINDOW | TIMEOUT | AUTH_ERROR | NETWORK | SERVER_ERROR | ...
|
|
retry_at: datetime | None = None
|
|
from_cache: bool = False
|
|
|
|
|
|
class QuotaManager:
|
|
def __init__(self, session, now=utcnow):
|
|
self.session, self.now = session, now
|
|
|
|
# ---- state -------------------------------------------------------------------------------------------------------------------------------------
|
|
def state(self, provider: Provider) -> ProviderState:
|
|
row = self.session.get(ProviderState, provider.name)
|
|
if row is None:
|
|
row = ProviderState(provider=provider.name, configured=provider.configured, status="ONLINE" if provider.configured else "NOT_CONFIGURED",
|
|
daily_limit=provider.daily_limit, rate_per_minute=provider.rate_per_minute,
|
|
quota_kind="LOCAL_ESTIMATE" if provider.daily_limit else "DOCUMENTED_RATE")
|
|
self.session.add(row)
|
|
self.session.flush()
|
|
row.configured = provider.configured
|
|
if not provider.configured:
|
|
row.status = "NOT_CONFIGURED"
|
|
elif row.status == "NOT_CONFIGURED":
|
|
row.status = "ONLINE"
|
|
row.daily_limit = provider.daily_limit
|
|
row.rate_per_minute = provider.rate_per_minute
|
|
return row
|
|
|
|
def usage(self, provider_name: str, day: str | None = None) -> ProviderUsageDaily:
|
|
day = day or self.now().strftime("%Y-%m-%d")
|
|
row = self.session.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == provider_name, ProviderUsageDaily.day == day))
|
|
if row is None:
|
|
row = ProviderUsageDaily(provider=provider_name, day=day)
|
|
self.session.add(row)
|
|
self.session.flush()
|
|
return row
|
|
|
|
# ---- decision ----------------------------------------------------------------------------------------------------------------------------------
|
|
def allow(self, provider: Provider) -> tuple[bool, str | None, datetime | None]:
|
|
state, now = self.state(provider), self.now()
|
|
if not provider.configured:
|
|
return False, "NOT_CONFIGURED", None
|
|
if state.circuit_open_until and _aware(state.circuit_open_until) > now:
|
|
return False, "CIRCUIT_OPEN", _aware(state.circuit_open_until)
|
|
usage = self.usage(provider.name)
|
|
if provider.daily_limit and state.quota_kind == "REMOTE_QUOTA" and state.remaining is not None and state.remaining <= 0 and state.reset_at and _aware(state.reset_at) > now:
|
|
return False, "RATE_LIMITED", _aware(state.reset_at)
|
|
if provider.daily_limit and usage.requests >= provider.daily_limit and state.quota_kind != "REMOTE_QUOTA":
|
|
state.status = "RATE_LIMITED"
|
|
return False, "RATE_LIMITED", state.reset_at and _aware(state.reset_at) or next_utc_midnight(now)
|
|
window = (provider.burst[1] if provider.burst else 60)
|
|
limit = (provider.burst[0] if provider.burst else provider.rate_per_minute)
|
|
if limit:
|
|
q, cutoff = _windows[provider.name], time.monotonic() - window
|
|
while q and q[0] < cutoff:
|
|
q.popleft()
|
|
if len(q) >= limit:
|
|
return False, "RATE_WINDOW", now + timedelta(seconds=max(1, int(window - (time.monotonic() - q[0]))))
|
|
return True, None, None
|
|
|
|
# ---- accounting --------------------------------------------------------------------------------------------------------------------------------
|
|
def _remote_quota(self, state: ProviderState, headers: dict):
|
|
remaining, limit, reset = headers.get("x-ratelimit-remaining"), headers.get("x-ratelimit-limit"), headers.get("x-ratelimit-reset")
|
|
if remaining is not None and str(remaining).isdigit():
|
|
state.remaining = int(remaining)
|
|
state.quota_kind = "REMOTE_QUOTA"
|
|
if limit and str(limit).isdigit():
|
|
state.daily_limit = int(limit)
|
|
if reset and str(reset).isdigit():
|
|
state.reset_at = datetime.fromtimestamp(int(reset), tz=timezone.utc)
|
|
|
|
def _sent(self, provider: Provider):
|
|
q = _windows[provider.name]
|
|
q.append(time.monotonic())
|
|
|
|
def record_ok(self, provider: Provider, result: ProviderProduct):
|
|
state, usage, now = self.state(provider), self.usage(provider.name), self.now()
|
|
self._sent(provider)
|
|
usage.requests += 1
|
|
usage.success += 1
|
|
if not result.found:
|
|
usage.not_found += 1
|
|
state.consecutive_failures = 0
|
|
state.circuit_open_until = None
|
|
state.status = "ONLINE"
|
|
state.last_request_at = state.last_success_at = now
|
|
state.last_latency_ms = result.latency_ms or None
|
|
self._remote_quota(state, {k.lower(): v for k, v in (result.headers or {}).items()})
|
|
if provider.daily_limit and state.quota_kind != "REMOTE_QUOTA":
|
|
state.remaining = max(0, provider.daily_limit - usage.requests)
|
|
state.reset_at = next_utc_midnight(now)
|
|
self.session.commit()
|
|
|
|
def record_error(self, provider: Provider, error: ProviderError):
|
|
state, usage, now = self.state(provider), self.usage(provider.name), self.now()
|
|
self._sent(provider)
|
|
usage.requests += 1
|
|
state.last_request_at, state.last_error_at = now, now
|
|
state.last_error = f"{error.kind}: {error.detail}"[:160]
|
|
state.consecutive_failures += 1
|
|
headers = {k.lower(): v for k, v in (error.headers or {}).items()}
|
|
self._remote_quota(state, headers)
|
|
if error.kind == "RATE_LIMITED":
|
|
usage.rate_limited += 1
|
|
state.status = "RATE_LIMITED"
|
|
wait = timedelta(seconds=error.retry_after) if error.retry_after else None
|
|
state.circuit_open_until = now + (wait or (state.reset_at and max(timedelta(minutes=5), _aware(state.reset_at) - now) or timedelta(minutes=15)))
|
|
if state.reset_at is None:
|
|
state.reset_at = state.circuit_open_until
|
|
elif error.kind == "AUTH_ERROR":
|
|
usage.errors += 1
|
|
state.status = "AUTH_ERROR"
|
|
state.circuit_open_until = now + timedelta(hours=1)
|
|
else:
|
|
if error.kind == "TIMEOUT":
|
|
usage.timeouts += 1
|
|
usage.errors += 1
|
|
state.status = "DEGRADED"
|
|
if state.consecutive_failures >= FAILURES_TO_OPEN:
|
|
delay = min(CIRCUIT_MAX, CIRCUIT_BASE * (2 ** (state.consecutive_failures - FAILURES_TO_OPEN)))
|
|
state.circuit_open_until = now + delay
|
|
state.status = "OFFLINE"
|
|
self.session.commit()
|
|
|
|
def record_cache(self, provider_name: str, hit: bool):
|
|
usage = self.usage(provider_name)
|
|
if hit:
|
|
usage.cache_hits += 1
|
|
else:
|
|
usage.cache_misses += 1
|
|
|
|
|
|
class ProviderGateway:
|
|
"""lookup(provider, gtin): cache -> single-flight -> quota/circuit -> HTTP -> account -> cache."""
|
|
|
|
def __init__(self, session, providers: list[Provider], now=utcnow):
|
|
self.session, self.providers, self.now = session, providers, now
|
|
self.quota = QuotaManager(session, now)
|
|
|
|
def _cached(self, provider: Provider, gtin: str) -> ProviderProduct | None | bool:
|
|
row = self.session.scalar(select(GtinLookupCache).where(GtinLookupCache.gtin == gtin, GtinLookupCache.provider == provider.name))
|
|
if row is None or _aware(row.expires_at) <= self.now():
|
|
return False
|
|
if not row.found:
|
|
return None
|
|
d = row.data or {}
|
|
return ProviderProduct(provider.name, gtin, True, title=d.get("title"), brand=d.get("brand"), category=d.get("category"), quantity=d.get("quantity"), ncm=d.get("ncm"), cest=d.get("cest"), raw_hash=d.get("raw_hash", ""))
|
|
|
|
def _store(self, provider: Provider, gtin: str, product: ProviderProduct):
|
|
now = self.now()
|
|
ttl = POSITIVE_TTL if product.found else NEGATIVE_TTL
|
|
row = self.session.scalar(select(GtinLookupCache).where(GtinLookupCache.gtin == gtin, GtinLookupCache.provider == provider.name))
|
|
data = {**product.as_data(), "raw_hash": product.raw_hash}
|
|
if row is None:
|
|
self.session.add(GtinLookupCache(gtin=gtin, provider=provider.name, found=product.found, data=data, fetched_at=now, expires_at=now + ttl))
|
|
else:
|
|
row.found, row.data, row.fetched_at, row.expires_at = product.found, data, now, now + ttl
|
|
self.session.commit()
|
|
|
|
def lookup(self, provider: Provider, gtin: str) -> Outcome:
|
|
hit = self._cached(provider, gtin)
|
|
if hit is not False:
|
|
self.quota.record_cache(provider.name, True)
|
|
self.session.commit()
|
|
return Outcome(provider.name, "CACHE_HIT", hit or None, from_cache=True) if hit else Outcome(provider.name, "NOT_FOUND", None, from_cache=True)
|
|
with _flights_guard:
|
|
lock = _flights.setdefault((provider.name, gtin), threading.Lock())
|
|
with lock: # 10 installations, same EAN -> 1 external call, the rest wait for the same answer
|
|
self.session.expire_all()
|
|
hit = self._cached(provider, gtin)
|
|
if hit is not False:
|
|
self.quota.record_cache(provider.name, True)
|
|
self.session.commit()
|
|
return Outcome(provider.name, "CACHE_HIT", hit or None, from_cache=True) if hit else Outcome(provider.name, "NOT_FOUND", None, from_cache=True)
|
|
self.quota.record_cache(provider.name, False)
|
|
ok, reason, retry_at = self.quota.allow(provider)
|
|
if not ok:
|
|
self.session.commit()
|
|
return Outcome(provider.name, "SKIPPED", None, reason, retry_at)
|
|
try:
|
|
product = provider.lookup_gtin(gtin)
|
|
except ProviderError as error:
|
|
self.quota.record_error(provider, error)
|
|
return Outcome(provider.name, "ERROR", None, error.kind, self.quota.state(provider).circuit_open_until)
|
|
self.quota.record_ok(provider, product)
|
|
self._store(provider, gtin, product)
|
|
return Outcome(provider.name, "FOUND" if product.found else "NOT_FOUND", product if product.found else None)
|