feat: product enrichment observability, lab KB study, benchmark, copy + local-Hub E2E, docs; 1.3.0

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-25 19:26:13 -03:00
parent 5deebea14e
commit 4fc83a33d5
38 changed files with 1282 additions and 60 deletions
@@ -0,0 +1,82 @@
+"""Read-only views for the panel: product sources status, official dataset freshness and enrichment coverage metrics. Never exposes tokens."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from sqlalchemy import func, select
+
+from ..config import settings
+from ..datasets.updater import last_runs
+from ..models import Dataset, GtinLookupCache, ProductEnrichment, ProviderState, ProviderUsageDaily
+from .providers import build_providers
+
+STATUS_LABELS = {"ONLINE": "Online", "DEGRADED": "Instável", "RATE_LIMITED": "Limite atingido", "AUTH_ERROR": "Credencial recusada", "OFFLINE": "Indisponível", "NOT_CONFIGURED": "Não configurado"}
+OFFICIAL_KINDS = (("NCM", "NCM (Siscomex)"), ("TIPI_IPI", "TIPI / IPI"), ("CEST", "CEST (CONFAZ)"), ("BA_ST_MVA", "ST/MVA Bahia"))
+
+
+def _aware(dt):
+    return dt.replace(tzinfo=timezone.utc) if dt is not None and dt.tzinfo is None else dt
+
+
+def provider_sources(session, providers=None) -> list[dict]:
+    providers = providers or build_providers(settings)
+    today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    rows = []
+    for provider in providers:
+        state = session.get(ProviderState, provider.name)
+        usage = session.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == provider.name, ProviderUsageDaily.day == today))
+        requests = usage.requests if usage else 0
+        lookups = (usage.cache_hits + usage.cache_misses) if usage else 0
+        status = ("NOT_CONFIGURED" if not provider.configured else (state.status if state else "ONLINE"))
+        rows.append({
+            "provider": provider.name, "configured": bool(provider.configured), "status": status, "status_label": STATUS_LABELS.get(status, status),
+            "used": requests, "limit": provider.daily_limit or (state.daily_limit if state else None), "rate_per_minute": provider.rate_per_minute,
+            "remaining": state.remaining if state and state.remaining is not None else ((provider.daily_limit - requests) if provider.daily_limit else None),
+            "quota_kind": state.quota_kind if state else ("LOCAL_ESTIMATE" if provider.daily_limit else "DOCUMENTED_RATE"),
+            "reset_at": _aware(state.reset_at) if state and state.reset_at else None,
+            "last_success_at": _aware(state.last_success_at) if state and state.last_success_at else None,
+            "last_error": state.last_error if state else None, "last_error_at": _aware(state.last_error_at) if state and state.last_error_at else None,
+            "latency_ms": state.last_latency_ms if state else None,
+            "cache_hit_rate": round(100 * usage.cache_hits / lookups) if usage and lookups else None,
+            "errors": usage.errors if usage else 0, "rate_limited": usage.rate_limited if usage else 0,
+        })
+    return rows
+
+
+def official_sources(session) -> list[dict]:
+    out = []
+    for kind, label in OFFICIAL_KINDS:
+        ds = session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").order_by(Dataset.id.desc()).limit(1))
+        item = {"kind": kind, "label": label, "active": ds is not None, "records": ds.record_count if ds else None, "version": ds.version_label if ds else None,
+                "sha256": (ds.sha256[:12] if ds else None), "last_update": _aware(ds.downloaded_at) if ds else None, "status": "ATUALIZADA" if ds else "SEM DADOS", "error": None, "auto_refresh": kind == "NCM"}
+        if kind == "NCM":
+            recent = last_runs(session, "NCM", 1)
+            if recent:
+                last = recent[0]
+                item["last_check"], item["last_run_status"] = _aware(last.finished_at or last.started_at), last.status
+                if last.status == "FAILED":
+                    item["status"], item["error"] = "FALHA NA ÚLTIMA ATUALIZAÇÃO (dados anteriores mantidos)", last.error
+        out.append(item)
+    return out
+
+
+def coverage_metrics(session) -> dict:
+    total = session.scalar(select(func.count()).select_from(ProductEnrichment)) or 0
+    by = dict(session.execute(select(ProductEnrichment.status, func.count()).group_by(ProductEnrichment.status)).all())
+    with_gtin = session.scalar(select(func.count()).select_from(ProductEnrichment).where(ProductEnrichment.gtin.is_not(None))) or 0
+    rows = [r.decisions or {} for r in session.scalars(select(ProductEnrichment).limit(5000))]
+    states = {}
+    for d in rows:
+        states[d.get("state")] = states.get(d.get("state"), 0) + 1
+
+    def rate(n):
+        return round(100 * n / total, 1) if total else None
+    auto_ncm = sum(1 for d in rows if ((d.get("fields") or {}).get("CLASSIFICACAO_FISCAL") or {}).get("status") == "AUTO_FILL_SAFE")
+    usage = session.execute(select(func.sum(ProviderUsageDaily.requests), func.sum(ProviderUsageDaily.cache_hits), func.sum(ProviderUsageDaily.cache_misses), func.sum(ProviderUsageDaily.rate_limited))).one()
+    requests, hits, misses, limited = (int(x or 0) for x in usage)
+    return {"tracked": total, "GTIN_VALID_RATE": rate(with_gtin), "KB_HIT_RATE": rate(states.get("EXACT_GTIN_VERIFIED", 0) + states.get("EXACT_KB_NAME_VERIFIED", 0)),
+            "EXTERNAL_LOOKUP_RATE": round(100 * requests / (requests + hits), 1) if (requests + hits) else None,
+            "PRODUCT_FOUND_RATE": rate(total - states.get("NOT_FOUND", 0) - states.get("INVALID_GTIN", 0) - states.get("AMBIGUOUS", 0)),
+            "NCM_AUTO_FILL_SAFE_RATE": rate(auto_ncm), "NAME_ONLY_RESOLUTION_RATE": rate(states.get("EXACT_KB_NAME_VERIFIED", 0)), "AMBIGUOUS_RATE": rate(states.get("AMBIGUOUS", 0)),
+            "CONFLICT_RATE": rate(states.get("CONFLICT", 0)), "PROVIDER_QUOTA_BLOCK_RATE": round(100 * limited / requests, 1) if requests else None, "states": states, "by_status": by,
+            "provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}