166 lines
12 KiB
Python
166 lines
12 KiB
Python
"""Read-only views for the panel: product sources status, official dataset freshness and enrichment coverage metrics. Never exposes tokens."""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import func, select
|
|
|
|
from ..config import settings
|
|
from ..datasets.updater import last_runs
|
|
from ..models import Dataset, GtinLookupCache, ProductEnrichment, ProviderState, ProviderUsageDaily
|
|
from .providers import build_providers
|
|
|
|
STATUS_LABELS = {"ONLINE": "Online", "DEGRADED": "Instável", "RATE_LIMITED": "Limite atingido", "AUTH_ERROR": "Credencial recusada", "OFFLINE": "Indisponível", "NOT_CONFIGURED": "Não configurado"}
|
|
OFFICIAL_KINDS = (("NCM", "NCM (Siscomex)"), ("TIPI_IPI", "TIPI / IPI"), ("CEST", "CEST (CONFAZ)"), ("BA_ST_MVA", "ST/MVA Bahia"))
|
|
|
|
|
|
def _aware(dt):
|
|
return dt.replace(tzinfo=timezone.utc) if dt is not None and dt.tzinfo is None else dt
|
|
|
|
|
|
def provider_sources(session, providers=None) -> list[dict]:
|
|
providers = providers or build_providers(settings)
|
|
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
|
rows = []
|
|
for provider in providers:
|
|
state = session.get(ProviderState, provider.name)
|
|
usage = session.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == provider.name, ProviderUsageDaily.day == today))
|
|
requests = usage.requests if usage else 0
|
|
lookups = (usage.cache_hits + usage.cache_misses) if usage else 0
|
|
status = ("NOT_CONFIGURED" if not provider.configured else (state.status if state else "ONLINE"))
|
|
rows.append({
|
|
"provider": provider.name, "configured": bool(provider.configured), "status": status, "status_label": STATUS_LABELS.get(status, status),
|
|
"used": requests, "limit": provider.daily_limit or (state.daily_limit if state else None), "rate_per_minute": provider.rate_per_minute,
|
|
"remaining": state.remaining if state and state.remaining is not None else ((provider.daily_limit - requests) if provider.daily_limit else None),
|
|
"quota_kind": state.quota_kind if state else ("LOCAL_ESTIMATE" if provider.daily_limit else "DOCUMENTED_RATE"),
|
|
"reset_at": _aware(state.reset_at) if state and state.reset_at else None,
|
|
"last_success_at": _aware(state.last_success_at) if state and state.last_success_at else None,
|
|
"last_error": state.last_error if state else None, "last_error_at": _aware(state.last_error_at) if state and state.last_error_at else None,
|
|
"latency_ms": state.last_latency_ms if state else None,
|
|
"cache_hit_rate": round(100 * usage.cache_hits / lookups) if usage and lookups else None,
|
|
"errors": usage.errors if usage else 0, "rate_limited": usage.rate_limited if usage else 0,
|
|
"found_today": usage.found if usage else 0, "ncm_found_today": usage.ncm_found if usage else 0, "cache_hits": usage.cache_hits if usage else 0,
|
|
"next_reset": _aware(state.reset_at) if state and state.reset_at else None, "identity_only": not getattr(provider, "provides_ncm", False),
|
|
})
|
|
return rows
|
|
|
|
|
|
def official_sources(session) -> list[dict]:
|
|
out = []
|
|
for kind, label in OFFICIAL_KINDS:
|
|
ds = session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").order_by(Dataset.id.desc()).limit(1))
|
|
item = {"kind": kind, "label": label, "active": ds is not None, "records": ds.record_count if ds else None, "version": ds.version_label if ds else None,
|
|
"sha256": (ds.sha256[:12] if ds else None), "last_update": _aware(ds.downloaded_at) if ds else None, "status": "ATUALIZADA" if ds else "SEM DADOS", "error": None, "auto_refresh": kind == "NCM"}
|
|
if kind == "NCM":
|
|
recent = last_runs(session, "NCM", 1)
|
|
if recent:
|
|
last = recent[0]
|
|
item["last_check"], item["last_run_status"] = _aware(last.finished_at or last.started_at), last.status
|
|
if last.status == "FAILED":
|
|
item["status"], item["error"] = "FALHA NA ÚLTIMA ATUALIZAÇÃO (dados anteriores mantidos)", last.error
|
|
out.append(item)
|
|
return out
|
|
|
|
|
|
def coverage_metrics(session) -> dict:
|
|
total = session.scalar(select(func.count()).select_from(ProductEnrichment)) or 0
|
|
by = dict(session.execute(select(ProductEnrichment.status, func.count()).group_by(ProductEnrichment.status)).all())
|
|
with_gtin = session.scalar(select(func.count()).select_from(ProductEnrichment).where(ProductEnrichment.gtin.is_not(None))) or 0
|
|
rows = [r.decisions or {} for r in session.scalars(select(ProductEnrichment).limit(5000))]
|
|
states = {}
|
|
for d in rows:
|
|
states[d.get("state")] = states.get(d.get("state"), 0) + 1
|
|
|
|
def rate(n):
|
|
return round(100 * n / total, 1) if total else None
|
|
auto_ncm = sum(1 for d in rows if ((d.get("fields") or {}).get("CLASSIFICACAO_FISCAL") or {}).get("status") == "AUTO_FILL_SAFE")
|
|
usage = session.execute(select(func.sum(ProviderUsageDaily.requests), func.sum(ProviderUsageDaily.cache_hits), func.sum(ProviderUsageDaily.cache_misses), func.sum(ProviderUsageDaily.rate_limited))).one()
|
|
requests, hits, misses, limited = (int(x or 0) for x in usage)
|
|
return {"tracked": total, "GTIN_VALID_RATE": rate(with_gtin), "KB_HIT_RATE": rate(states.get("EXACT_GTIN_VERIFIED", 0) + states.get("EXACT_KB_NAME_VERIFIED", 0)),
|
|
"EXTERNAL_LOOKUP_RATE": round(100 * requests / (requests + hits), 1) if (requests + hits) else None,
|
|
"PRODUCT_FOUND_RATE": rate(total - states.get("NOT_FOUND", 0) - states.get("INVALID_GTIN", 0) - states.get("AMBIGUOUS", 0)),
|
|
"NCM_AUTO_FILL_SAFE_RATE": rate(auto_ncm), "NAME_ONLY_RESOLUTION_RATE": rate(states.get("EXACT_KB_NAME_VERIFIED", 0)), "AMBIGUOUS_RATE": rate(states.get("AMBIGUOUS", 0)),
|
|
"CONFLICT_RATE": rate(states.get("CONFLICT", 0)), "PROVIDER_QUOTA_BLOCK_RATE": round(100 * limited / requests, 1) if requests else None, "states": states, "by_status": by,
|
|
"provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}
|
|
|
|
|
|
def _candidate_cards(session, products) -> list[dict]:
|
|
from ..models import Dataset, KbKnowledge, NcmEntry
|
|
out = []
|
|
for prod in products:
|
|
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == prod.id, KbKnowledge.knowledge_key == "ncm")))
|
|
cands = []
|
|
for r in rows:
|
|
ev = r.evidence or {}
|
|
text = session.scalar(select(NcmEntry.description).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == r.value).limit(1))
|
|
cands.append({"ncm": r.value, "official_text": text, "status": r.status, "clients": sorted(ev.get("clients", [])), "documents": sum(int(l.get("docs", 0)) for l in (ev.get("lineages") or {}).values()),
|
|
"providers": sorted(f for f in (ev.get("families") or {}) if f.startswith("PROVIDER:")), "attested": (ev.get("info") or {}).get("attested_by")})
|
|
status = "VERIFIED_KB" if any(c["status"] == "VERIFIED_KB" for c in cands) else ("CONFLICT" if any(c["status"] == "CONFLICT" for c in cands) else (cands[0]["status"] if cands else "OBSERVED"))
|
|
out.append({"gtin": (prod.gtin or "")[-13:], "gtin_key": prod.gtin, "description": (prod.normalized_description or "")[:70], "status": status, "candidates": [c for c in cands if c["status"] != "STALE" or c["attested"] is None][:4]})
|
|
return out
|
|
|
|
|
|
def kb_search_ids(session, query: str) -> list[int]:
|
|
"""GLOBAL, server-side match over the whole KB (never limited to a rendered page): GTIN exact, or every normalized query word present somewhere among the
|
|
product's known names (any alias row OR its own normalized_description - words can come from different aliases of the same product). Deliberately lenient
|
|
(a human exploring the KB, e.g. "COCA COLA LATA 350" without the unit, must still find "COCA-COLA ORIGINAL LATA 350 ML"); this is DIFFERENT from
|
|
`kb.alias_match`, the strict rule that authorizes an automatic fiscal write, which stays untouched."""
|
|
import re
|
|
|
|
from ..models import KbKnowledge, KbProduct
|
|
from . import kb as kbm, textnorm
|
|
q = (query or "").strip()
|
|
if not q:
|
|
return []
|
|
digits = "".join(ch for ch in q if ch.isdigit())
|
|
if len(digits) >= 8 and re.fullmatch(r"[\d\s.\-]+", q):
|
|
p = kbm.find_by_gtin(session, digits.zfill(14))
|
|
return [p.id] if p else []
|
|
tokens = [t for t in textnorm.tokens(q) if t not in textnorm.MARKETING]
|
|
if not tokens:
|
|
return []
|
|
ids: set[int] | None = None
|
|
for token in tokens: # SQL ILIKE narrows the candidates (portable: Postgres in production, SQLite in tests); the exact whole-word
|
|
like = f"%{token}%" # check happens in Python so "ML" never matches inside "SIMPLES", etc.
|
|
matched = {pid for pid, val in session.execute(select(KbKnowledge.kb_product_id, KbKnowledge.value).where(KbKnowledge.knowledge_key == "alias", KbKnowledge.value.ilike(like))) if token in (val or "").split()}
|
|
matched |= {pid for pid, val in session.execute(select(KbProduct.id, KbProduct.normalized_description).where(KbProduct.normalized_description.ilike(like))) if token in (val or "").split()}
|
|
ids = matched if ids is None else (ids & matched)
|
|
if not ids:
|
|
return []
|
|
return sorted(ids)
|
|
|
|
|
|
def kb_search(session, query: str, *, page: int = 1, page_size: int = 10) -> dict:
|
|
"""query/filter GLOBAL over the whole KB -> count -> order -> pagination (never the other way round)."""
|
|
from ..models import KbProduct
|
|
ids = kb_search_ids(session, query)
|
|
total = len(ids)
|
|
page = max(1, page)
|
|
pages = max(1, -(-total // page_size))
|
|
page = min(page, pages)
|
|
page_ids = ids[(page - 1) * page_size: page * page_size]
|
|
products = [session.get(KbProduct, i) for i in page_ids]
|
|
return {"results": _candidate_cards(session, products), "total": total, "page": page, "pages": pages, "page_size": page_size}
|
|
|
|
|
|
def kb_summary(session, query: str | None = None, page: int = 1) -> dict:
|
|
"""Compact Knowledge Base panel: known products, statuses of the NCM knowledge, hits today, and the latest conflicts with their evidence (no dashboard of its own)."""
|
|
from ..models import KbKnowledge, KbProduct
|
|
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
|
products = session.scalar(select(func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
|
|
by = dict(session.execute(select(KbKnowledge.status, func.count(func.distinct(KbKnowledge.kb_product_id))).where(KbKnowledge.knowledge_key == "ncm").group_by(KbKnowledge.status)).all())
|
|
hits = session.scalar(select(func.coalesce(func.sum(ProviderUsageDaily.kb_hits), 0)).where(ProviderUsageDaily.day == today)) or 0
|
|
conflicts = []
|
|
rows = session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.status == "CONFLICT").order_by(KbKnowledge.id.desc()).limit(60)).all()
|
|
grouped = {}
|
|
for r in rows:
|
|
grouped.setdefault(r.kb_product_id, []).append(r)
|
|
for pid, items in list(grouped.items())[:5]:
|
|
prod = session.get(KbProduct, pid)
|
|
conflicts.append({"gtin": (prod.gtin or "")[-13:], "description": (prod.normalized_description or "")[:60],
|
|
"candidates": [{"ncm": i.value, "clients": sorted({c for c in (i.evidence or {}).get("clients", [])}), "documents": sum(int(l.get("docs", 0)) for l in ((i.evidence or {}).get("lineages") or {}).values()),
|
|
"providers": sorted(f for f in ((i.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:"))} for i in items]})
|
|
return {"products": products, "verified": by.get("VERIFIED_KB", 0), "corroborated": by.get("CORROBORATED", 0) + by.get("EXTERNAL_CORROBORATED", 0), "observed": by.get("OBSERVED", 0),
|
|
"conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts,
|
|
"search": kb_search(session, query, page=page) if query else {"results": [], "total": 0, "page": 1, "pages": 1, "page_size": 10}}
|