138 lines
10 KiB
Python
138 lines
10 KiB
Python
"""Read-only views for the panel: product sources status, official dataset freshness and enrichment coverage metrics. Never exposes tokens."""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import func, select
|
|
|
|
from ..config import settings
|
|
from ..datasets.updater import last_runs
|
|
from ..models import Dataset, GtinLookupCache, ProductEnrichment, ProviderState, ProviderUsageDaily
|
|
from .providers import build_providers
|
|
|
|
STATUS_LABELS = {"ONLINE": "Online", "DEGRADED": "Instável", "RATE_LIMITED": "Limite atingido", "AUTH_ERROR": "Credencial recusada", "OFFLINE": "Indisponível", "NOT_CONFIGURED": "Não configurado"}
|
|
OFFICIAL_KINDS = (("NCM", "NCM (Siscomex)"), ("TIPI_IPI", "TIPI / IPI"), ("CEST", "CEST (CONFAZ)"), ("BA_ST_MVA", "ST/MVA Bahia"))
|
|
|
|
|
|
def _aware(dt):
|
|
return dt.replace(tzinfo=timezone.utc) if dt is not None and dt.tzinfo is None else dt
|
|
|
|
|
|
def provider_sources(session, providers=None) -> list[dict]:
|
|
providers = providers or build_providers(settings)
|
|
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
|
rows = []
|
|
for provider in providers:
|
|
state = session.get(ProviderState, provider.name)
|
|
usage = session.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == provider.name, ProviderUsageDaily.day == today))
|
|
requests = usage.requests if usage else 0
|
|
lookups = (usage.cache_hits + usage.cache_misses) if usage else 0
|
|
status = ("NOT_CONFIGURED" if not provider.configured else (state.status if state else "ONLINE"))
|
|
rows.append({
|
|
"provider": provider.name, "configured": bool(provider.configured), "status": status, "status_label": STATUS_LABELS.get(status, status),
|
|
"used": requests, "limit": provider.daily_limit or (state.daily_limit if state else None), "rate_per_minute": provider.rate_per_minute,
|
|
"remaining": state.remaining if state and state.remaining is not None else ((provider.daily_limit - requests) if provider.daily_limit else None),
|
|
"quota_kind": state.quota_kind if state else ("LOCAL_ESTIMATE" if provider.daily_limit else "DOCUMENTED_RATE"),
|
|
"reset_at": _aware(state.reset_at) if state and state.reset_at else None,
|
|
"last_success_at": _aware(state.last_success_at) if state and state.last_success_at else None,
|
|
"last_error": state.last_error if state else None, "last_error_at": _aware(state.last_error_at) if state and state.last_error_at else None,
|
|
"latency_ms": state.last_latency_ms if state else None,
|
|
"cache_hit_rate": round(100 * usage.cache_hits / lookups) if usage and lookups else None,
|
|
"errors": usage.errors if usage else 0, "rate_limited": usage.rate_limited if usage else 0,
|
|
"found_today": usage.found if usage else 0, "ncm_found_today": usage.ncm_found if usage else 0, "cache_hits": usage.cache_hits if usage else 0,
|
|
"next_reset": _aware(state.reset_at) if state and state.reset_at else None, "identity_only": not getattr(provider, "provides_ncm", False),
|
|
})
|
|
return rows
|
|
|
|
|
|
def official_sources(session) -> list[dict]:
|
|
out = []
|
|
for kind, label in OFFICIAL_KINDS:
|
|
ds = session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").order_by(Dataset.id.desc()).limit(1))
|
|
item = {"kind": kind, "label": label, "active": ds is not None, "records": ds.record_count if ds else None, "version": ds.version_label if ds else None,
|
|
"sha256": (ds.sha256[:12] if ds else None), "last_update": _aware(ds.downloaded_at) if ds else None, "status": "ATUALIZADA" if ds else "SEM DADOS", "error": None, "auto_refresh": kind == "NCM"}
|
|
if kind == "NCM":
|
|
recent = last_runs(session, "NCM", 1)
|
|
if recent:
|
|
last = recent[0]
|
|
item["last_check"], item["last_run_status"] = _aware(last.finished_at or last.started_at), last.status
|
|
if last.status == "FAILED":
|
|
item["status"], item["error"] = "FALHA NA ÚLTIMA ATUALIZAÇÃO (dados anteriores mantidos)", last.error
|
|
out.append(item)
|
|
return out
|
|
|
|
|
|
def coverage_metrics(session) -> dict:
|
|
total = session.scalar(select(func.count()).select_from(ProductEnrichment)) or 0
|
|
by = dict(session.execute(select(ProductEnrichment.status, func.count()).group_by(ProductEnrichment.status)).all())
|
|
with_gtin = session.scalar(select(func.count()).select_from(ProductEnrichment).where(ProductEnrichment.gtin.is_not(None))) or 0
|
|
rows = [r.decisions or {} for r in session.scalars(select(ProductEnrichment).limit(5000))]
|
|
states = {}
|
|
for d in rows:
|
|
states[d.get("state")] = states.get(d.get("state"), 0) + 1
|
|
|
|
def rate(n):
|
|
return round(100 * n / total, 1) if total else None
|
|
auto_ncm = sum(1 for d in rows if ((d.get("fields") or {}).get("CLASSIFICACAO_FISCAL") or {}).get("status") == "AUTO_FILL_SAFE")
|
|
usage = session.execute(select(func.sum(ProviderUsageDaily.requests), func.sum(ProviderUsageDaily.cache_hits), func.sum(ProviderUsageDaily.cache_misses), func.sum(ProviderUsageDaily.rate_limited))).one()
|
|
requests, hits, misses, limited = (int(x or 0) for x in usage)
|
|
return {"tracked": total, "GTIN_VALID_RATE": rate(with_gtin), "KB_HIT_RATE": rate(states.get("EXACT_GTIN_VERIFIED", 0) + states.get("EXACT_KB_NAME_VERIFIED", 0)),
|
|
"EXTERNAL_LOOKUP_RATE": round(100 * requests / (requests + hits), 1) if (requests + hits) else None,
|
|
"PRODUCT_FOUND_RATE": rate(total - states.get("NOT_FOUND", 0) - states.get("INVALID_GTIN", 0) - states.get("AMBIGUOUS", 0)),
|
|
"NCM_AUTO_FILL_SAFE_RATE": rate(auto_ncm), "NAME_ONLY_RESOLUTION_RATE": rate(states.get("EXACT_KB_NAME_VERIFIED", 0)), "AMBIGUOUS_RATE": rate(states.get("AMBIGUOUS", 0)),
|
|
"CONFLICT_RATE": rate(states.get("CONFLICT", 0)), "PROVIDER_QUOTA_BLOCK_RATE": round(100 * limited / requests, 1) if requests else None, "states": states, "by_status": by,
|
|
"provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}
|
|
|
|
|
|
def _candidate_cards(session, products) -> list[dict]:
|
|
from ..models import Dataset, KbKnowledge, NcmEntry
|
|
out = []
|
|
for prod in products:
|
|
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == prod.id, KbKnowledge.knowledge_key == "ncm")))
|
|
cands = []
|
|
for r in rows:
|
|
ev = r.evidence or {}
|
|
text = session.scalar(select(NcmEntry.description).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == r.value).limit(1))
|
|
cands.append({"ncm": r.value, "official_text": text, "status": r.status, "clients": sorted(ev.get("clients", [])), "documents": sum(int(l.get("docs", 0)) for l in (ev.get("lineages") or {}).values()),
|
|
"providers": sorted(f for f in (ev.get("families") or {}) if f.startswith("PROVIDER:")), "attested": (ev.get("info") or {}).get("attested_by")})
|
|
status = "VERIFIED_KB" if any(c["status"] == "VERIFIED_KB" for c in cands) else ("CONFLICT" if any(c["status"] == "CONFLICT" for c in cands) else (cands[0]["status"] if cands else "OBSERVED"))
|
|
out.append({"gtin": (prod.gtin or "")[-13:], "gtin_key": prod.gtin, "description": (prod.normalized_description or "")[:70], "status": status, "candidates": [c for c in cands if c["status"] != "STALE" or c["attested"] is None][:4]})
|
|
return out
|
|
|
|
|
|
def kb_search(session, query: str, limit: int = 6) -> list[dict]:
|
|
from ..models import KbProduct
|
|
from . import kb as kbm
|
|
q = (query or "").strip()
|
|
if not q:
|
|
return []
|
|
digits = "".join(ch for ch in q if ch.isdigit())
|
|
products = []
|
|
if len(digits) >= 8 and len(digits) == len(q.replace(" ", "")):
|
|
p = kbm.find_by_gtin(session, digits.zfill(14))
|
|
products = [p] if p else []
|
|
else:
|
|
products = kbm.name_candidates(session, q)[:limit]
|
|
return _candidate_cards(session, products)
|
|
|
|
|
|
def kb_summary(session, query: str | None = None) -> dict:
|
|
"""Compact Knowledge Base panel: known products, statuses of the NCM knowledge, hits today, and the latest conflicts with their evidence (no dashboard of its own)."""
|
|
from ..models import KbKnowledge, KbProduct
|
|
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
|
products = session.scalar(select(func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
|
|
by = dict(session.execute(select(KbKnowledge.status, func.count(func.distinct(KbKnowledge.kb_product_id))).where(KbKnowledge.knowledge_key == "ncm").group_by(KbKnowledge.status)).all())
|
|
hits = session.scalar(select(func.coalesce(func.sum(ProviderUsageDaily.kb_hits), 0)).where(ProviderUsageDaily.day == today)) or 0
|
|
conflicts = []
|
|
rows = session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.status == "CONFLICT").order_by(KbKnowledge.id.desc()).limit(60)).all()
|
|
grouped = {}
|
|
for r in rows:
|
|
grouped.setdefault(r.kb_product_id, []).append(r)
|
|
for pid, items in list(grouped.items())[:5]:
|
|
prod = session.get(KbProduct, pid)
|
|
conflicts.append({"gtin": (prod.gtin or "")[-13:], "description": (prod.normalized_description or "")[:60],
|
|
"candidates": [{"ncm": i.value, "clients": sorted({c for c in (i.evidence or {}).get("clients", [])}), "documents": sum(int(l.get("docs", 0)) for l in ((i.evidence or {}).get("lineages") or {}).values()),
|
|
"providers": sorted(f for f in ((i.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:"))} for i in items]})
|
|
return {"products": products, "verified": by.get("VERIFIED_KB", 0), "corroborated": by.get("CORROBORATED", 0) + by.get("EXTERNAL_CORROBORATED", 0), "observed": by.get("OBSERVED", 0),
|
|
"conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts, "search": kb_search(session, query) if query else []}
|