feat: product enrichment observability, lab KB study, benchmark, copy + local-Hub E2E, docs; 1.3.0

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

central/app/enrichment/observability.py

@@ -0,0 +1,82 @@
+"""Read-only views for the panel: product sources status, official dataset freshness and enrichment coverage metrics. Never exposes tokens."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from sqlalchemy import func, select
+
+from ..config import settings
+from ..datasets.updater import last_runs
+from ..models import Dataset, GtinLookupCache, ProductEnrichment, ProviderState, ProviderUsageDaily
+from .providers import build_providers
+
+STATUS_LABELS = {"ONLINE": "Online", "DEGRADED": "Instável", "RATE_LIMITED": "Limite atingido", "AUTH_ERROR": "Credencial recusada", "OFFLINE": "Indisponível", "NOT_CONFIGURED": "Não configurado"}
+OFFICIAL_KINDS = (("NCM", "NCM (Siscomex)"), ("TIPI_IPI", "TIPI / IPI"), ("CEST", "CEST (CONFAZ)"), ("BA_ST_MVA", "ST/MVA Bahia"))
+
+
+def _aware(dt):
+    return dt.replace(tzinfo=timezone.utc) if dt is not None and dt.tzinfo is None else dt
+
+
+def provider_sources(session, providers=None) -> list[dict]:
+    providers = providers or build_providers(settings)
+    today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    rows = []
+    for provider in providers:
+        state = session.get(ProviderState, provider.name)
+        usage = session.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == provider.name, ProviderUsageDaily.day == today))
+        requests = usage.requests if usage else 0
+        lookups = (usage.cache_hits + usage.cache_misses) if usage else 0
+        status = ("NOT_CONFIGURED" if not provider.configured else (state.status if state else "ONLINE"))
+        rows.append({
+            "provider": provider.name, "configured": bool(provider.configured), "status": status, "status_label": STATUS_LABELS.get(status, status),
+            "used": requests, "limit": provider.daily_limit or (state.daily_limit if state else None), "rate_per_minute": provider.rate_per_minute,
+            "remaining": state.remaining if state and state.remaining is not None else ((provider.daily_limit - requests) if provider.daily_limit else None),
+            "quota_kind": state.quota_kind if state else ("LOCAL_ESTIMATE" if provider.daily_limit else "DOCUMENTED_RATE"),
+            "reset_at": _aware(state.reset_at) if state and state.reset_at else None,
+            "last_success_at": _aware(state.last_success_at) if state and state.last_success_at else None,
+            "last_error": state.last_error if state else None, "last_error_at": _aware(state.last_error_at) if state and state.last_error_at else None,
+            "latency_ms": state.last_latency_ms if state else None,
+            "cache_hit_rate": round(100 * usage.cache_hits / lookups) if usage and lookups else None,
+            "errors": usage.errors if usage else 0, "rate_limited": usage.rate_limited if usage else 0,
+        })
+    return rows
+
+
+def official_sources(session) -> list[dict]:
+    out = []
+    for kind, label in OFFICIAL_KINDS:
+        ds = session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").order_by(Dataset.id.desc()).limit(1))
+        item = {"kind": kind, "label": label, "active": ds is not None, "records": ds.record_count if ds else None, "version": ds.version_label if ds else None,
+                "sha256": (ds.sha256[:12] if ds else None), "last_update": _aware(ds.downloaded_at) if ds else None, "status": "ATUALIZADA" if ds else "SEM DADOS", "error": None, "auto_refresh": kind == "NCM"}
+        if kind == "NCM":
+            recent = last_runs(session, "NCM", 1)
+            if recent:
+                last = recent[0]
+                item["last_check"], item["last_run_status"] = _aware(last.finished_at or last.started_at), last.status
+                if last.status == "FAILED":
+                    item["status"], item["error"] = "FALHA NA ÚLTIMA ATUALIZAÇÃO (dados anteriores mantidos)", last.error
+        out.append(item)
+    return out
+
+
+def coverage_metrics(session) -> dict:
+    total = session.scalar(select(func.count()).select_from(ProductEnrichment)) or 0
+    by = dict(session.execute(select(ProductEnrichment.status, func.count()).group_by(ProductEnrichment.status)).all())
+    with_gtin = session.scalar(select(func.count()).select_from(ProductEnrichment).where(ProductEnrichment.gtin.is_not(None))) or 0
+    rows = [r.decisions or {} for r in session.scalars(select(ProductEnrichment).limit(5000))]
+    states = {}
+    for d in rows:
+        states[d.get("state")] = states.get(d.get("state"), 0) + 1
+
+    def rate(n):
+        return round(100 * n / total, 1) if total else None
+    auto_ncm = sum(1 for d in rows if ((d.get("fields") or {}).get("CLASSIFICACAO_FISCAL") or {}).get("status") == "AUTO_FILL_SAFE")
+    usage = session.execute(select(func.sum(ProviderUsageDaily.requests), func.sum(ProviderUsageDaily.cache_hits), func.sum(ProviderUsageDaily.cache_misses), func.sum(ProviderUsageDaily.rate_limited))).one()
+    requests, hits, misses, limited = (int(x or 0) for x in usage)
+    return {"tracked": total, "GTIN_VALID_RATE": rate(with_gtin), "KB_HIT_RATE": rate(states.get("EXACT_GTIN_VERIFIED", 0) + states.get("EXACT_KB_NAME_VERIFIED", 0)),
+            "EXTERNAL_LOOKUP_RATE": round(100 * requests / (requests + hits), 1) if (requests + hits) else None,
+            "PRODUCT_FOUND_RATE": rate(total - states.get("NOT_FOUND", 0) - states.get("INVALID_GTIN", 0) - states.get("AMBIGUOUS", 0)),
+            "NCM_AUTO_FILL_SAFE_RATE": rate(auto_ncm), "NAME_ONLY_RESOLUTION_RATE": rate(states.get("EXACT_KB_NAME_VERIFIED", 0)), "AMBIGUOUS_RATE": rate(states.get("AMBIGUOUS", 0)),
+            "CONFLICT_RATE": rate(states.get("CONFLICT", 0)), "PROVIDER_QUOTA_BLOCK_RATE": round(100 * limited / requests, 1) if requests else None, "states": states, "by_status": by,
+            "provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}

central/app/enrichment/observations.py

@@ -0,0 +1,27 @@
+"""ERP observations -> KB evidence. A client observation is EVIDENCE of family CLIENT_OBSERVATION; it never becomes VERIFIED_KB by itself
+(needs the official dataset AND a second independent family, or >= 2 distinct clients + another family - see kb.evaluate)."""
+from __future__ import annotations
+
+from ..classification import normalize_code
+from . import kb
+from .gtin import normalize_gtin
+
+
+def observe(session, rows, client_label: str, is_official) -> dict:
+    """rows: iterable of {"GTIN","PRODUTO","NCM"}. Only VALID_GTIN rows with an OFFICIAL 8-digit NCM are recorded. Returns counters."""
+    stats = {"rows": 0, "valid_gtin": 0, "with_official_ncm": 0, "recorded": 0}
+    for row in rows:
+        stats["rows"] += 1
+        g = normalize_gtin(row.get("GTIN"))
+        if g["status"] != "VALID_GTIN":
+            continue
+        stats["valid_gtin"] += 1
+        ncm = normalize_code(row.get("NCM"))
+        if len(ncm) != 8 or not is_official(ncm):
+            continue
+        stats["with_official_ncm"] += 1
+        product = kb.upsert_product(session, g["key"], row.get("PRODUTO"))
+        kb.add_evidence(session, product, "ncm", ncm, "client:observation", family=kb.CLIENT_FAMILY, client_label=client_label)
+        stats["recorded"] += 1
+    session.flush()
+    return stats

central/app/enrichment/providers.py

@@ -138,9 +138,10 @@ class OpenFoodFacts(Provider):
     name = "openfoodfacts"
     supports_search = True
     rate_per_minute = 15
+    host = "world.openfoodfacts.org"
 
     def lookup_gtin(self, gtin: str) -> ProviderProduct:
-        response = self._get(f"https://world.openfoodfacts.org/api/v2/product/{gtin}", params={"fields": "code,product_name,brands,quantity,categories,generic_name"})
+        response = self._get(f"https://{self.host}/api/v2/product/{gtin}", params={"fields": "code,product_name,brands,quantity,categories,generic_name"})
         headers = {k.lower(): v for k, v in response.headers.items()}
         if response.status_code == 404:
             return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
@@ -166,6 +167,19 @@ class OpenFoodFacts(Provider):
                 for p in (response.json().get("products") or [])[:5] if p.get("product_name")]
 
 
+class OpenProductsFacts(OpenFoodFacts):
+    """Same open (ODbL) platform and API as Open Food Facts, for non-food goods. Identity only (no NCM)."""
+    name = "openproductsfacts"
+    host = "world.openproductsfacts.org"
+    supports_search = False
+
+
+class OpenBeautyFacts(OpenFoodFacts):
+    name = "openbeautyfacts"
+    host = "world.openbeautyfacts.org"
+    supports_search = False
+
+
 class Cosmos(Provider):
     name = "cosmos"
     supports_search = True
@@ -212,8 +226,8 @@ class Cosmos(Provider):
 
 
 def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]:
-    """Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos (fiscal-capable) first when configured, then UPCitemdb, then Open Food Facts. Paid providers do not exist here."""
-    providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), UpcItemDb(client), OpenFoodFacts(client)]
+    """Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here."""
+    providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
     if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False):
         raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)")
     return providers

central/app/enrichment/resolver.py

@@ -45,6 +45,17 @@ def ncm_is_official(session, code: str, today: date | None = None) -> bool:
     return False
 
 
+class _ScopedDatasets(DatasetService):
+    """The Classifier only needs the ONE exact NCM row here; loading the whole 15k-row catalogue on every product would cost ~300 ms of the 5 s budget."""
+
+    def __init__(self, session, code: str):
+        super().__init__(session)
+        self.code = code
+
+    def active_entries(self):
+        return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == self.code)).all()
+
+
 def dataset_versions(session) -> dict:
     return {d.kind: {"sha256": (d.sha256 or "")[:12], "version": d.version_label} for d in session.scalars(select(Dataset).where(Dataset.status == "ACTIVE"))}
 
@@ -75,11 +86,13 @@ def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current:
         for name in ("CEST", "IPI"):
             out[name] = _field(ALREADY if not _empty(current, name) else INSUFFICIENT, reason="depende de um NCM oficial válido")
         return out
-    decision = Classifier(DatasetService(session)).classify({"current_ncm": ncm_for_engine, "description": description})
+    # CEST / IPI are deterministic functions of an OFFICIAL NCM: the NCM we are about to write (AUTO) or one already in the product (validated above against the official dataset)
+    ncm_solid = ncm["status"] == AUTO or (ncm["status"] == ALREADY and ncm_is_official(session, ncm_for_engine))
+    decision = Classifier(_ScopedDatasets(session, ncm_for_engine)).classify({"current_ncm": ncm_for_engine, "description": description})
     cest = (decision["classification"].get("cest") or [])
     if not _empty(current, "CEST"):
         out["CEST"] = _field(ALREADY, reason="campo já preenchido")
-    elif decision["cest_status"] == "CEST_CONFIRMED" and len(cest) == 1 and ncm["status"] == AUTO:
+    elif decision["cest_status"] == "CEST_CONFIRMED" and len(cest) == 1 and ncm_solid:
         out["CEST"] = _field(AUTO, normalize_code(cest[0]), "CEST_OFFICIAL_UNIQUE_MAPPING", ncm["kb_status"], [f"NCM {ncm_for_engine}"])
     elif decision["cest_status"] == "NO_CEST_MAPPING_FOUND":
         out["CEST"] = _field(NA, reason="NCM sem mapeamento de CEST no dataset oficial")
@@ -88,7 +101,7 @@ def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current:
     ipi = decision["classification"].get("ipi") or {}
     if not _empty(current, "IPI"):
         out["IPI"] = _field(ALREADY, reason="campo já preenchido")
-    elif decision["ipi_status"] == "IPI_RESOLVED" and ipi.get("rate_kind") == "NUMERIC" and ncm["status"] == AUTO:
+    elif decision["ipi_status"] == "IPI_RESOLVED" and ipi.get("rate_kind") == "NUMERIC" and ncm_solid:
         out["IPI"] = _field(AUTO, str(ipi["rate_numeric"]), "IPI_TIPI_NUMERIC", ncm["kb_status"], [f"TIPI NCM {ncm_for_engine}"])
     elif decision["ipi_status"] in {"IPI_RESOLVED"}:
         out["IPI"] = _field(NA, reason="TIPI sem alíquota numérica (NT/branca)")

central/app/version.py

@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
 once published — never a second, independent version number. Fiscal semantics
 are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
 DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
-GTI_FISCAL_VERSION = "1.2.5"
+GTI_FISCAL_VERSION = "1.3.0"
 API_COMPAT_MAJOR_MINOR = "1.0"  # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x

central/app/web.py

@@ -22,6 +22,7 @@ from .panel_presentation import format_ui_datetime
 from .models import AdminUser, Agent, AgentEnrollment, AgentRelease, AgentSnapshot, BlockedProduct, FiscalContextEntry, Installation, Job, ProductEnrichment, ProductFieldAudit, RtcCandidate, RtcWriteAudit, Tenant
 from .jobs import create_job
 from .audit import record_audit
+from .enrichment.observability import coverage_metrics, official_sources, provider_sources
 
 
 def router(session_factory):
@@ -160,7 +161,7 @@ def router(session_factory):
     def dashboard(request: Request):
         session, auth_data, redirect = page_auth(request)
         if redirect: return redirect
-        try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data})
+        try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session)})
         finally: session.close()
 
     @routes.get("/agents", response_class=HTMLResponse)