"""Read-only views for the panel: product sources status, official dataset freshness and enrichment coverage metrics. Never exposes tokens.""" from __future__ import annotations from datetime import datetime, timezone from sqlalchemy import func, select from ..config import settings from ..datasets.updater import last_runs from ..models import Dataset, GtinLookupCache, ProductEnrichment, ProviderState, ProviderUsageDaily from .providers import build_providers STATUS_LABELS = {"ONLINE": "Online", "DEGRADED": "Instável", "RATE_LIMITED": "Limite atingido", "AUTH_ERROR": "Credencial recusada", "OFFLINE": "Indisponível", "NOT_CONFIGURED": "Não configurado"} OFFICIAL_KINDS = (("NCM", "NCM (Siscomex)"), ("TIPI_IPI", "TIPI / IPI"), ("CEST", "CEST (CONFAZ)"), ("BA_ST_MVA", "ST/MVA Bahia")) def _aware(dt): return dt.replace(tzinfo=timezone.utc) if dt is not None and dt.tzinfo is None else dt def provider_sources(session, providers=None) -> list[dict]: providers = providers or build_providers(settings) today = datetime.now(timezone.utc).strftime("%Y-%m-%d") rows = [] for provider in providers: state = session.get(ProviderState, provider.name) usage = session.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == provider.name, ProviderUsageDaily.day == today)) requests = usage.requests if usage else 0 lookups = (usage.cache_hits + usage.cache_misses) if usage else 0 status = ("NOT_CONFIGURED" if not provider.configured else (state.status if state else "ONLINE")) rows.append({ "provider": provider.name, "configured": bool(provider.configured), "status": status, "status_label": STATUS_LABELS.get(status, status), "used": requests, "limit": provider.daily_limit or (state.daily_limit if state else None), "rate_per_minute": provider.rate_per_minute, "remaining": state.remaining if state and state.remaining is not None else ((provider.daily_limit - requests) if provider.daily_limit else None), "quota_kind": state.quota_kind if state else ("LOCAL_ESTIMATE" if provider.daily_limit else "DOCUMENTED_RATE"), "reset_at": _aware(state.reset_at) if state and state.reset_at else None, "last_success_at": _aware(state.last_success_at) if state and state.last_success_at else None, "last_error": state.last_error if state else None, "last_error_at": _aware(state.last_error_at) if state and state.last_error_at else None, "latency_ms": state.last_latency_ms if state else None, "cache_hit_rate": round(100 * usage.cache_hits / lookups) if usage and lookups else None, "errors": usage.errors if usage else 0, "rate_limited": usage.rate_limited if usage else 0, }) return rows def official_sources(session) -> list[dict]: out = [] for kind, label in OFFICIAL_KINDS: ds = session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").order_by(Dataset.id.desc()).limit(1)) item = {"kind": kind, "label": label, "active": ds is not None, "records": ds.record_count if ds else None, "version": ds.version_label if ds else None, "sha256": (ds.sha256[:12] if ds else None), "last_update": _aware(ds.downloaded_at) if ds else None, "status": "ATUALIZADA" if ds else "SEM DADOS", "error": None, "auto_refresh": kind == "NCM"} if kind == "NCM": recent = last_runs(session, "NCM", 1) if recent: last = recent[0] item["last_check"], item["last_run_status"] = _aware(last.finished_at or last.started_at), last.status if last.status == "FAILED": item["status"], item["error"] = "FALHA NA ÚLTIMA ATUALIZAÇÃO (dados anteriores mantidos)", last.error out.append(item) return out def coverage_metrics(session) -> dict: total = session.scalar(select(func.count()).select_from(ProductEnrichment)) or 0 by = dict(session.execute(select(ProductEnrichment.status, func.count()).group_by(ProductEnrichment.status)).all()) with_gtin = session.scalar(select(func.count()).select_from(ProductEnrichment).where(ProductEnrichment.gtin.is_not(None))) or 0 rows = [r.decisions or {} for r in session.scalars(select(ProductEnrichment).limit(5000))] states = {} for d in rows: states[d.get("state")] = states.get(d.get("state"), 0) + 1 def rate(n): return round(100 * n / total, 1) if total else None auto_ncm = sum(1 for d in rows if ((d.get("fields") or {}).get("CLASSIFICACAO_FISCAL") or {}).get("status") == "AUTO_FILL_SAFE") usage = session.execute(select(func.sum(ProviderUsageDaily.requests), func.sum(ProviderUsageDaily.cache_hits), func.sum(ProviderUsageDaily.cache_misses), func.sum(ProviderUsageDaily.rate_limited))).one() requests, hits, misses, limited = (int(x or 0) for x in usage) return {"tracked": total, "GTIN_VALID_RATE": rate(with_gtin), "KB_HIT_RATE": rate(states.get("EXACT_GTIN_VERIFIED", 0) + states.get("EXACT_KB_NAME_VERIFIED", 0)), "EXTERNAL_LOOKUP_RATE": round(100 * requests / (requests + hits), 1) if (requests + hits) else None, "PRODUCT_FOUND_RATE": rate(total - states.get("NOT_FOUND", 0) - states.get("INVALID_GTIN", 0) - states.get("AMBIGUOUS", 0)), "NCM_AUTO_FILL_SAFE_RATE": rate(auto_ncm), "NAME_ONLY_RESOLUTION_RATE": rate(states.get("EXACT_KB_NAME_VERIFIED", 0)), "AMBIGUOUS_RATE": rate(states.get("AMBIGUOUS", 0)), "CONFLICT_RATE": rate(states.get("CONFLICT", 0)), "PROVIDER_QUOTA_BLOCK_RATE": round(100 * limited / requests, 1) if requests else None, "states": states, "by_status": by, "provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}