feat: product enrichment observability, lab KB study, benchmark, copy + local-Hub E2E, docs; 1.3.0

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
2026-09-25 19:26:13 -03:00
co-authored by Claude Sonnet 5
parent 5deebea14e
commit 4fc83a33d5
38 changed files with 1282 additions and 60 deletions
+82
View File
@@ -0,0 +1,82 @@
"""Read-only views for the panel: product sources status, official dataset freshness and enrichment coverage metrics. Never exposes tokens."""
from __future__ import annotations
from datetime import datetime, timezone
from sqlalchemy import func, select
from ..config import settings
from ..datasets.updater import last_runs
from ..models import Dataset, GtinLookupCache, ProductEnrichment, ProviderState, ProviderUsageDaily
from .providers import build_providers
STATUS_LABELS = {"ONLINE": "Online", "DEGRADED": "Instável", "RATE_LIMITED": "Limite atingido", "AUTH_ERROR": "Credencial recusada", "OFFLINE": "Indisponível", "NOT_CONFIGURED": "Não configurado"}
OFFICIAL_KINDS = (("NCM", "NCM (Siscomex)"), ("TIPI_IPI", "TIPI / IPI"), ("CEST", "CEST (CONFAZ)"), ("BA_ST_MVA", "ST/MVA Bahia"))
def _aware(dt):
return dt.replace(tzinfo=timezone.utc) if dt is not None and dt.tzinfo is None else dt
def provider_sources(session, providers=None) -> list[dict]:
providers = providers or build_providers(settings)
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
rows = []
for provider in providers:
state = session.get(ProviderState, provider.name)
usage = session.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == provider.name, ProviderUsageDaily.day == today))
requests = usage.requests if usage else 0
lookups = (usage.cache_hits + usage.cache_misses) if usage else 0
status = ("NOT_CONFIGURED" if not provider.configured else (state.status if state else "ONLINE"))
rows.append({
"provider": provider.name, "configured": bool(provider.configured), "status": status, "status_label": STATUS_LABELS.get(status, status),
"used": requests, "limit": provider.daily_limit or (state.daily_limit if state else None), "rate_per_minute": provider.rate_per_minute,
"remaining": state.remaining if state and state.remaining is not None else ((provider.daily_limit - requests) if provider.daily_limit else None),
"quota_kind": state.quota_kind if state else ("LOCAL_ESTIMATE" if provider.daily_limit else "DOCUMENTED_RATE"),
"reset_at": _aware(state.reset_at) if state and state.reset_at else None,
"last_success_at": _aware(state.last_success_at) if state and state.last_success_at else None,
"last_error": state.last_error if state else None, "last_error_at": _aware(state.last_error_at) if state and state.last_error_at else None,
"latency_ms": state.last_latency_ms if state else None,
"cache_hit_rate": round(100 * usage.cache_hits / lookups) if usage and lookups else None,
"errors": usage.errors if usage else 0, "rate_limited": usage.rate_limited if usage else 0,
})
return rows
def official_sources(session) -> list[dict]:
out = []
for kind, label in OFFICIAL_KINDS:
ds = session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").order_by(Dataset.id.desc()).limit(1))
item = {"kind": kind, "label": label, "active": ds is not None, "records": ds.record_count if ds else None, "version": ds.version_label if ds else None,
"sha256": (ds.sha256[:12] if ds else None), "last_update": _aware(ds.downloaded_at) if ds else None, "status": "ATUALIZADA" if ds else "SEM DADOS", "error": None, "auto_refresh": kind == "NCM"}
if kind == "NCM":
recent = last_runs(session, "NCM", 1)
if recent:
last = recent[0]
item["last_check"], item["last_run_status"] = _aware(last.finished_at or last.started_at), last.status
if last.status == "FAILED":
item["status"], item["error"] = "FALHA NA ÚLTIMA ATUALIZAÇÃO (dados anteriores mantidos)", last.error
out.append(item)
return out
def coverage_metrics(session) -> dict:
total = session.scalar(select(func.count()).select_from(ProductEnrichment)) or 0
by = dict(session.execute(select(ProductEnrichment.status, func.count()).group_by(ProductEnrichment.status)).all())
with_gtin = session.scalar(select(func.count()).select_from(ProductEnrichment).where(ProductEnrichment.gtin.is_not(None))) or 0
rows = [r.decisions or {} for r in session.scalars(select(ProductEnrichment).limit(5000))]
states = {}
for d in rows:
states[d.get("state")] = states.get(d.get("state"), 0) + 1
def rate(n):
return round(100 * n / total, 1) if total else None
auto_ncm = sum(1 for d in rows if ((d.get("fields") or {}).get("CLASSIFICACAO_FISCAL") or {}).get("status") == "AUTO_FILL_SAFE")
usage = session.execute(select(func.sum(ProviderUsageDaily.requests), func.sum(ProviderUsageDaily.cache_hits), func.sum(ProviderUsageDaily.cache_misses), func.sum(ProviderUsageDaily.rate_limited))).one()
requests, hits, misses, limited = (int(x or 0) for x in usage)
return {"tracked": total, "GTIN_VALID_RATE": rate(with_gtin), "KB_HIT_RATE": rate(states.get("EXACT_GTIN_VERIFIED", 0) + states.get("EXACT_KB_NAME_VERIFIED", 0)),
"EXTERNAL_LOOKUP_RATE": round(100 * requests / (requests + hits), 1) if (requests + hits) else None,
"PRODUCT_FOUND_RATE": rate(total - states.get("NOT_FOUND", 0) - states.get("INVALID_GTIN", 0) - states.get("AMBIGUOUS", 0)),
"NCM_AUTO_FILL_SAFE_RATE": rate(auto_ncm), "NAME_ONLY_RESOLUTION_RATE": rate(states.get("EXACT_KB_NAME_VERIFIED", 0)), "AMBIGUOUS_RATE": rate(states.get("AMBIGUOUS", 0)),
"CONFLICT_RATE": rate(states.get("CONFLICT", 0)), "PROVIDER_QUOTA_BLOCK_RATE": round(100 * limited / requests, 1) if requests else None, "states": states, "by_status": by,
"provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}
+27
View File
@@ -0,0 +1,27 @@
"""ERP observations -> KB evidence. A client observation is EVIDENCE of family CLIENT_OBSERVATION; it never becomes VERIFIED_KB by itself
(needs the official dataset AND a second independent family, or >= 2 distinct clients + another family - see kb.evaluate)."""
from __future__ import annotations
from ..classification import normalize_code
from . import kb
from .gtin import normalize_gtin
def observe(session, rows, client_label: str, is_official) -> dict:
"""rows: iterable of {"GTIN","PRODUTO","NCM"}. Only VALID_GTIN rows with an OFFICIAL 8-digit NCM are recorded. Returns counters."""
stats = {"rows": 0, "valid_gtin": 0, "with_official_ncm": 0, "recorded": 0}
for row in rows:
stats["rows"] += 1
g = normalize_gtin(row.get("GTIN"))
if g["status"] != "VALID_GTIN":
continue
stats["valid_gtin"] += 1
ncm = normalize_code(row.get("NCM"))
if len(ncm) != 8 or not is_official(ncm):
continue
stats["with_official_ncm"] += 1
product = kb.upsert_product(session, g["key"], row.get("PRODUTO"))
kb.add_evidence(session, product, "ncm", ncm, "client:observation", family=kb.CLIENT_FAMILY, client_label=client_label)
stats["recorded"] += 1
session.flush()
return stats
+17 -3
View File
@@ -138,9 +138,10 @@ class OpenFoodFacts(Provider):
name = "openfoodfacts"
supports_search = True
rate_per_minute = 15
host = "world.openfoodfacts.org"
def lookup_gtin(self, gtin: str) -> ProviderProduct:
response = self._get(f"https://world.openfoodfacts.org/api/v2/product/{gtin}", params={"fields": "code,product_name,brands,quantity,categories,generic_name"})
response = self._get(f"https://{self.host}/api/v2/product/{gtin}", params={"fields": "code,product_name,brands,quantity,categories,generic_name"})
headers = {k.lower(): v for k, v in response.headers.items()}
if response.status_code == 404:
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
@@ -166,6 +167,19 @@ class OpenFoodFacts(Provider):
for p in (response.json().get("products") or [])[:5] if p.get("product_name")]
class OpenProductsFacts(OpenFoodFacts):
"""Same open (ODbL) platform and API as Open Food Facts, for non-food goods. Identity only (no NCM)."""
name = "openproductsfacts"
host = "world.openproductsfacts.org"
supports_search = False
class OpenBeautyFacts(OpenFoodFacts):
name = "openbeautyfacts"
host = "world.openbeautyfacts.org"
supports_search = False
class Cosmos(Provider):
name = "cosmos"
supports_search = True
@@ -212,8 +226,8 @@ class Cosmos(Provider):
def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]:
"""Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos (fiscal-capable) first when configured, then UPCitemdb, then Open Food Facts. Paid providers do not exist here."""
providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), UpcItemDb(client), OpenFoodFacts(client)]
"""Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here."""
providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False):
raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)")
return providers
+16 -3
View File
@@ -45,6 +45,17 @@ def ncm_is_official(session, code: str, today: date | None = None) -> bool:
return False
class _ScopedDatasets(DatasetService):
"""The Classifier only needs the ONE exact NCM row here; loading the whole 15k-row catalogue on every product would cost ~300 ms of the 5 s budget."""
def __init__(self, session, code: str):
super().__init__(session)
self.code = code
def active_entries(self):
return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == self.code)).all()
def dataset_versions(session) -> dict:
return {d.kind: {"sha256": (d.sha256 or "")[:12], "version": d.version_label} for d in session.scalars(select(Dataset).where(Dataset.status == "ACTIVE"))}
@@ -75,11 +86,13 @@ def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current:
for name in ("CEST", "IPI"):
out[name] = _field(ALREADY if not _empty(current, name) else INSUFFICIENT, reason="depende de um NCM oficial válido")
return out
decision = Classifier(DatasetService(session)).classify({"current_ncm": ncm_for_engine, "description": description})
# CEST / IPI are deterministic functions of an OFFICIAL NCM: the NCM we are about to write (AUTO) or one already in the product (validated above against the official dataset)
ncm_solid = ncm["status"] == AUTO or (ncm["status"] == ALREADY and ncm_is_official(session, ncm_for_engine))
decision = Classifier(_ScopedDatasets(session, ncm_for_engine)).classify({"current_ncm": ncm_for_engine, "description": description})
cest = (decision["classification"].get("cest") or [])
if not _empty(current, "CEST"):
out["CEST"] = _field(ALREADY, reason="campo já preenchido")
elif decision["cest_status"] == "CEST_CONFIRMED" and len(cest) == 1 and ncm["status"] == AUTO:
elif decision["cest_status"] == "CEST_CONFIRMED" and len(cest) == 1 and ncm_solid:
out["CEST"] = _field(AUTO, normalize_code(cest[0]), "CEST_OFFICIAL_UNIQUE_MAPPING", ncm["kb_status"], [f"NCM {ncm_for_engine}"])
elif decision["cest_status"] == "NO_CEST_MAPPING_FOUND":
out["CEST"] = _field(NA, reason="NCM sem mapeamento de CEST no dataset oficial")
@@ -88,7 +101,7 @@ def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current:
ipi = decision["classification"].get("ipi") or {}
if not _empty(current, "IPI"):
out["IPI"] = _field(ALREADY, reason="campo já preenchido")
elif decision["ipi_status"] == "IPI_RESOLVED" and ipi.get("rate_kind") == "NUMERIC" and ncm["status"] == AUTO:
elif decision["ipi_status"] == "IPI_RESOLVED" and ipi.get("rate_kind") == "NUMERIC" and ncm_solid:
out["IPI"] = _field(AUTO, str(ipi["rate_numeric"]), "IPI_TIPI_NUMERIC", ncm["kb_status"], [f"TIPI NCM {ncm_for_engine}"])
elif decision["ipi_status"] in {"IPI_RESOLVED"}:
out["IPI"] = _field(NA, reason="TIPI sem alíquota numérica (NT/branca)")
+1 -1
View File
@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
once published — never a second, independent version number. Fiscal semantics
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
GTI_FISCAL_VERSION = "1.2.5"
GTI_FISCAL_VERSION = "1.3.0"
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
+2 -1
View File
@@ -22,6 +22,7 @@ from .panel_presentation import format_ui_datetime
from .models import AdminUser, Agent, AgentEnrollment, AgentRelease, AgentSnapshot, BlockedProduct, FiscalContextEntry, Installation, Job, ProductEnrichment, ProductFieldAudit, RtcCandidate, RtcWriteAudit, Tenant
from .jobs import create_job
from .audit import record_audit
from .enrichment.observability import coverage_metrics, official_sources, provider_sources
def router(session_factory):
@@ -160,7 +161,7 @@ def router(session_factory):
def dashboard(request: Request):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data})
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session)})
finally: session.close()
@routes.get("/agents", response_class=HTMLResponse)
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "gti-fiscal-hub"
version = "1.2.5"
version = "1.3.0"
requires-python = ">=3.10"
dependencies = [
"fastapi>=0.115,<1",
+17
View File
@@ -12,6 +12,23 @@
</a>
{% endfor %}
</section>
<section class="split overview-split" id="fontes">
<article class="panel">
<div class="panel-title"><div><h2>Fontes de produtos</h2><p class="muted">Consultas feitas somente pelo Hub, dentro do limite gratuito.</p></div></div>
{% for src in sources %}<div class="list-row"><span><b>{{ src.provider }}</b> <em class="badge {{ '' if src.status == 'ONLINE' else 'critical' }}">{{ src.status_label }}</em>
<small>{% if src.limit %}{{ src.used }} / {{ src.limit }} consultas hoje · restam {{ src.remaining if src.remaining is not none else "—" }}{% if src.reset_at %} · renova {{ src.reset_at.strftime("%d/%m %H:%M") }} UTC{% endif %}{% elif src.rate_per_minute %}{{ src.rate_per_minute }} consultas/min (limite documentado) · {{ src.used }} hoje{% else %}{{ src.used }} consultas hoje{% endif %}
{% if src.latency_ms %} · {{ src.latency_ms }} ms{% endif %}{% if src.cache_hit_rate is not none %} · cache {{ src.cache_hit_rate }}%{% endif %}
{% if src.last_success_at %} · último sucesso {{ src.last_success_at.strftime("%d/%m %H:%M") }}{% endif %}{% if src.last_error %} · último erro: {{ src.last_error }}{% endif %}
· configurado: {{ "sim" if src.configured else "não" }}</small></span></div>{% endfor %}
</article>
<article class="panel">
<div class="panel-title"><div><h2>Fontes oficiais</h2><p class="muted">Bases fiscais usadas para validar qualquer preenchimento.</p></div></div>
{% for o in official %}<div class="list-row"><span><b>{{ o.label }}</b> <em class="badge {{ '' if o.active and not o.error else 'critical' }}">{{ o.status }}</em>
<small>{% if o.active %}{{ o.records }} registros · {{ o.version }} · {{ o.sha256 }}{% if o.last_update %} · baixada {{ o.last_update.strftime("%d/%m/%Y") }}{% endif %}{% endif %}{% if o.auto_refresh %} · atualização automática diária{% endif %}{% if o.error %} · {{ o.error }}{% endif %}</small></span></div>{% endfor %}
</article>
</section>
<section class="panel" id="enriquecimento-resumo"><div class="panel-title"><div><h2>Enriquecimento de produtos novos</h2><p class="muted">Cobertura medida sobre os produtos novos acompanhados (o catálogo histórico não entra).</p></div></div>
{% if enrich_metrics.tracked %}<div class="coverage">{% for key,label in [("GTIN_VALID_RATE","Com GTIN válido"),("KB_HIT_RATE","Resolvidos pela base"),("PRODUCT_FOUND_RATE","Produto identificado"),("NCM_AUTO_FILL_SAFE_RATE","NCM preenchível com segurança"),("AMBIGUOUS_RATE","Ambíguos"),("CONFLICT_RATE","Em conflito")] %}<div class="coverage-item"><strong>{{ enrich_metrics[key] if enrich_metrics[key] is not none else "—" }}{% if enrich_metrics[key] is not none %}%{% endif %}</strong><span>{{ label }}</span></div>{% endfor %}</div>{% else %}<p class="empty-state">Nenhum produto novo acompanhado ainda.</p>{% endif %}</section>
<section class="split overview-split">
<article class="panel">
<div class="panel-title"><h2>Cobertura dos produtos</h2><a href="/agents?scope=active">Ver servidores</a></div>
+1 -1
View File
@@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down():
def test_version_endpoint_reports_release_and_fiscal_versions():
body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
assert body["version"] == "1.2.5" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
assert body["version"] == "1.3.0" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
def test_production_refuses_the_dev_token(monkeypatch):
@@ -94,3 +94,67 @@ def test_features_route_is_admin_only_csrf_and_audited():
assert "INSTALLATION_FEATURES" in [e.action for e in s.scalars(select(AuditEvent))]
page = client.get(f"/agents/{agent_id}")
assert "Funcionalidades" in page.text and "Enriquecimento" in page.text
def test_dashboard_shows_sources_without_tokens_and_official_freshness(monkeypatch):
from central.app.enrichment import observability, providers
monkeypatch.setattr(observability, "build_providers", lambda _s: providers.build_providers(type("S", (), {"cosmos_token": "SEGREDO-NAO-VAZAR", "allow_paid_providers": False})()))
client, agent_id, _ = _panel_client()
page = client.get("/").text
assert "Fontes de produtos" in page and "cosmos" in page and "configurado: sim" in page and "upcitemdb" in page and "openfoodfacts" in page
assert "SEGREDO-NAO-VAZAR" not in page and "Fontes oficiais" in page and "NCM (Siscomex)" in page
def _second_agent(engine, mode="ENABLED", corrector=True):
from central.app.auth import token_hash
from central.app.models import Agent, Tenant
with Session(engine) as s:
first = s.scalars(select(Installation)).first()
agent = Agent(tenant_id=first.tenant_id, name="second", token_hash=token_hash("second-token"), enabled=True)
s.add(agent)
s.flush()
s.add(Installation(agent_id=agent.id, tenant_id=first.tenant_id, installation_id="00000000-0000-4000-8000-0000000000bb", display_name="B", features_configured=True, corrector_enabled=corrector, enrichment_mode=mode))
s.commit()
return {"Authorization": "Bearer second-token"}
def test_same_product_in_another_installation_costs_zero_extra_provider_requests(monkeypatch):
client, engine = setup(monkeypatch)
h2 = _second_agent(engine)
providers = enrich_api.PROVIDERS_OVERRIDE
first = resolve(client)
calls_after_first = [p.calls for p in providers]
second = client.post("/v1/products/resolve", json={"product_code": "X9", "gtin": G1, "description": "REFRIGERANTE COLA LATA 350ML", "current": {}}, headers=h2).json()
assert [p.calls for p in providers] == calls_after_first # other installation: served by KB, zero external calls
assert second["state"] == "EXACT_GTIN_VERIFIED" and second["fields"]["CLASSIFICACAO_FISCAL"]["value"] == first["fields"]["CLASSIFICACAO_FISCAL"]["value"]
with Session(engine) as s:
assert {r.installation_id for r in s.scalars(select(ProductEnrichment))} == {1, 2}
def test_installations_are_isolated_by_their_own_feature_flags(monkeypatch):
client, engine = setup(monkeypatch, "DISABLED")
h2 = _second_agent(engine, "ENABLED")
assert resolve(client)["state"] == "FEATURE_DISABLED"
body = client.post("/v1/products/resolve", json={"product_code": "N1", "gtin": G1, "description": "REFRIGERANTE COLA LATA 350ML", "current": {}}, headers=h2).json()
assert body["state"] != "FEATURE_DISABLED" and body["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "AUTO_FILL_SAFE"
def test_quota_exhausted_gives_pending_state_and_retry_time_through_the_api(monkeypatch):
client, engine = setup(monkeypatch)
monkeypatch.setattr(enrich_api, "PROVIDERS_OVERRIDE", [Fake("cosmos", COSMOS(), True, daily_limit=1)])
resolve(client, "A1", G1)
body = resolve(client, "A2", G4, "OUTRO PRODUTO")
assert body["state"] == "PENDING_PROVIDER_QUOTA" and body["retry_at"]
with Session(engine) as s:
row = s.scalars(select(ProductEnrichment).where(ProductEnrichment.product_code == "A2")).one()
assert row.status == "PENDING" and row.next_attempt_at is not None
assert s.scalars(select(BlockedProduct).where(BlockedProduct.reason_code == "PENDING_PROVIDER_QUOTA")).one().field == "ENRICHMENT"
def test_corrector_switch_off_stops_jobs_but_not_enrichment(monkeypatch):
client, engine = setup(monkeypatch)
with Session(engine) as s:
s.scalars(select(Installation)).one().corrector_enabled = False
s.commit()
assert client.get("/v1/jobs/next", headers=H).json() is None
assert resolve(client)["state"] != "FEATURE_DISABLED"
@@ -223,7 +223,7 @@ def test_cosmos_without_token_is_not_configured_and_costs_nothing(env):
def test_paid_provider_guard():
class S: allow_paid_providers = False; cosmos_token = None
assert [p.name for p in build_providers(S())] == ["cosmos", "upcitemdb", "openfoodfacts"]
assert [p.name for p in build_providers(S())] == ["cosmos", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"]
# ---- provider contract tests against recorded fixtures (httpx.MockTransport, no network) ----------------------------------------------------------