feat: visible per-product enrichment status, flagged re-evaluation (panel/Reprocessar/Sincronizar/KB decision), operator NCM decision, deterministic name-only match, stale-write guard; 1.3.5

central/app/enrichment/api.py

@@ -10,6 +10,7 @@ from sqlalchemy import select
 from ..blocked_products import upsert_blocker
 from ..config import settings
 from ..models import Installation, ProductEnrichment, ProductFieldAudit
+from . import status as enrichment_status, textnorm
 from .placeholders import is_placeholder
 from .providers import build_providers
 from .quota import ProviderGateway
@@ -80,6 +81,8 @@ def router(principal, providers_factory=None):
                     decision["technical_placeholder"] = str(current.get(name))
             row.gtin, row.description, row.latency_ms = result.get("gtin"), description, latency
             row.attempts = (row.attempts or 0) + 1
+            row.ui_status, row.reason_code = enrichment_status.classify(result, has_gtin=bool(result.get("gtin")), specific_name=textnorm.is_specific(description))
+            row.last_attempt_at, row.reevaluate = datetime.now(timezone.utc), False
             row.status = "PENDING" if pending else "RESOLVED"
             row.decisions = {"state": result["state"], "fields": {k: {kk: vv for kk, vv in v.items() if kk != "evidence"} for k, v in result["fields"].items()}}
             row.next_attempt_at = _aware(result["retry_at"]) if pending else None
@@ -124,6 +127,10 @@ def router(principal, providers_factory=None):
                 elif outcome == "HUMAN_OVERRIDDEN":
                     overridden[item["field"]] = True
             row.fields_applied, row.human_overridden = applied_fields, overridden
+            ui, why = enrichment_status.after_write([i for i in (payload.get("fields") or []) if isinstance(i, dict)], installation.enrichment_mode)
+            if ui:
+                row.ui_status, row.reason_code = ui, why
+                row.reevaluate = False
             row.status = "APPLIED" if applied_fields and not overridden else ("PARTIAL" if applied_fields else row.status if row.status != "PENDING" else "RESOLVED")
             row.updated_at = datetime.now(timezone.utc)
             session.commit()
@@ -131,6 +138,19 @@ def router(principal, providers_factory=None):
         finally:
             session.close()
 
+    @routes.get("/enrichment/pending")
+    def pending_reevaluation(identity=Depends(principal)):
+        """Tracked products flagged for a new evaluation (panel button, REPROCESS/SYNC, KB decision). The Agent polls this cheaply and re-runs resolve for each code."""
+        session, agent = identity
+        try:
+            installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
+            if installation is None or not installation.features_configured or installation.enrichment_mode == "DISABLED":
+                return {"codes": []}
+            codes = list(session.scalars(select(ProductEnrichment.product_code).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.reevaluate.is_(True)).order_by(ProductEnrichment.id).limit(25)))
+            return {"codes": codes}
+        finally:
+            session.close()
+
     @routes.get("/enrichment/authorization")
     def authorization(identity=Depends(principal)):
         """Fresh central authorization read by the Agent right before EVERY field write."""

central/app/enrichment/kb.py

@@ -36,13 +36,39 @@ def find_by_gtin(session, key: str | None) -> KbProduct | None:
     return session.scalar(select(KbProduct).where(KbProduct.gtin == key)) if key else None
 
 
+NEUTRAL_EXTRAS = frozenset({"REFRIGERANTE", "REFRIG", "REFR", "BEBIDA", "PRODUTO", "PROD"})       # category nouns only: ZERO / DIET / a flavour / another size are NEVER neutral
+
+
+def _tokset(text) -> frozenset:
+    return frozenset(t for t in textnorm.tokens(text) if t not in textnorm.MARKETING)
+
+
+def alias_match(query: frozenset, alias: frozenset) -> bool:
+    """Deterministic, conservative: same token set, or one set contains the other and the difference is only neutral category nouns (so "COCA COLA LATA 350 ML" = "REFRIGERANTE COCA COLA LATA 350 ML", but not "... ZERO ...")."""
+    if not query or not alias:
+        return False
+    if query == alias:
+        return True
+    if query < alias:
+        return (alias - query) <= NEUTRAL_EXTRAS
+    if alias < query:
+        return (query - alias) <= NEUTRAL_EXTRAS
+    return False
+
+
 def name_candidates(session, description) -> list[KbProduct]:
-    """Exact-fingerprint matches (0, 1 or several)."""
+    """Every KB product with an alias that matches the description under `alias_match` (exact fingerprint included). 0, 1 or several - the caller decides (unique VERIFIED only)."""
     fp = textnorm.fingerprint(description)
     if not fp:
         return []
-    ids = list(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "name_fingerprint", KbKnowledge.value == fp).distinct()))
-    return [p for p in (session.get(KbProduct, i) for i in ids) if p]
+    ids = set(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "name_fingerprint", KbKnowledge.value == fp).distinct()))
+    query = _tokset(description)
+    anchor = max((t for t in query if not t.replace(".", "").isdigit() and len(t) > 2), key=len, default=None)      # most selective informative token, then verify in Python
+    if anchor:
+        for row in session.execute(select(KbKnowledge.kb_product_id, KbKnowledge.value).where(KbKnowledge.knowledge_key == "alias", KbKnowledge.value.contains(anchor))):
+            if alias_match(query, _tokset(row[1])):
+                ids.add(row[0])
+    return [p for p in (session.get(KbProduct, i) for i in sorted(ids)) if p]
 
 
 def upsert_product(session, key: str | None, description, *, brand=None, quantity=None) -> KbProduct:
@@ -140,6 +166,28 @@ def _mark(row, state):
     row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "historical_state": state}}
 
 
+OPERATOR_FAMILY = "OPERATOR_ATTESTATION"
+
+
+def attest(session, gtin_key: str, ncm: str, operator: str, is_official, note: str = "") -> dict:
+    """A human fiscal decision by an ADMIN (audited by the caller). Only a value that is ALREADY a candidate in the KB (some local/provider evidence) and vigente in the official dataset can be attested;
+    the dissenting candidates become STALE (REJECTED_BY_OPERATOR) and the product is promoted with the promotion audit naming the operator."""
+    product = find_by_gtin(session, gtin_key)
+    if product is None:
+        raise ValueError("GTIN_UNKNOWN_IN_KB")
+    row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm", KbKnowledge.value == ncm))
+    if row is None:
+        raise ValueError("NCM_IS_NOT_A_CANDIDATE_OF_THIS_GTIN")
+    if not is_official(ncm):
+        raise ValueError("NCM_NOT_VIGENTE_IN_OFFICIAL_DATASET")
+    add_evidence(session, product, "ncm", ncm, f"operator:{operator}"[:32], family=OPERATOR_FAMILY, info={"attested_by": operator, "attested_at": _now().isoformat(), "note": note[:200]})
+    for other in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm", KbKnowledge.value != ncm)):
+        if OPERATOR_FAMILY not in ((other.evidence or {}).get("families") or {}):
+            other.status = STALE
+            _mark(other, "REJECTED_BY_OPERATOR")
+    return evaluate(session, product, "ncm", is_official)
+
+
 def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
     """Recomputes and PERSISTS the status of one field. Never 'majority wins': two current-valid values => CONFLICT."""
     rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key)))
@@ -152,6 +200,14 @@ def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
             state = _official_period(session, r.value) if key == "ncm" else "HISTORICAL_INVALID"
             r.status = STALE if state == "HISTORICAL_EXPIRED" else OBSERVED
             _mark(r, state)
+    attested = {r.value for r in valid if OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {})}
+    if len(attested) == 1:                                                     # a human decision resolves the candidates: the others are recorded as rejected, not "outvoted"
+        winner = next(iter(attested))
+        for r in valid:
+            if r.value != winner:
+                r.status = STALE
+                _mark(r, "REJECTED_BY_OPERATOR")
+        valid = [r for r in valid if r.value == winner]
     values = sorted({r.value for r in valid})
     if len(values) > 1:
         for r in valid:
@@ -169,6 +225,9 @@ def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
     if superseded and CLIENT_FAMILY in names:
         names.remove(CLIENT_FAMILY)
         count -= 1
+    if any(OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen):
+        names = sorted(set(names) | {OPERATOR_FAMILY} | ({CLIENT_FAMILY} if any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen) else set()))
+        count = len(names)
     providers = [n for n in names if n.startswith("PROVIDER:")]
     already = any(r.status == VERIFIED_KB and r.validated_at for r in chosen)
     has_client_data = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen)
@@ -188,7 +247,8 @@ def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
     if status == VERIFIED_KB and old != VERIFIED_KB and key == "ncm":
         from .resolver import dataset_versions
         session.add(KbPromotionAudit(kb_product_id=product.id, gtin=product.gtin, field=key, value=values[0], old_status=old, new_status=status, families=names,
-                                     sources={"clients": summary["clients"], "documents": summary["docs"], "years": summary["years"], "providers": providers},
+                                     sources={"clients": summary["clients"], "documents": summary["docs"], "years": summary["years"], "providers": providers,
+                                                 "operator": next((((r.evidence or {}).get("info") or {}).get("attested_by") for r in chosen if OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {})), None)},
                                      official_dataset=dataset_versions(session).get("NCM", {}), rule_version=RULE_VERSION))
     return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected, "detail": summary}
 

central/app/enrichment/observability.py

@@ -84,7 +84,39 @@ def coverage_metrics(session) -> dict:
             "provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}
 
 
-def kb_summary(session) -> dict:
+def _candidate_cards(session, products) -> list[dict]:
+    from ..models import Dataset, KbKnowledge, NcmEntry
+    out = []
+    for prod in products:
+        rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == prod.id, KbKnowledge.knowledge_key == "ncm")))
+        cands = []
+        for r in rows:
+            ev = r.evidence or {}
+            text = session.scalar(select(NcmEntry.description).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == r.value).limit(1))
+            cands.append({"ncm": r.value, "official_text": text, "status": r.status, "clients": sorted(ev.get("clients", [])), "documents": sum(int(l.get("docs", 0)) for l in (ev.get("lineages") or {}).values()),
+                          "providers": sorted(f for f in (ev.get("families") or {}) if f.startswith("PROVIDER:")), "attested": (ev.get("info") or {}).get("attested_by")})
+        status = "VERIFIED_KB" if any(c["status"] == "VERIFIED_KB" for c in cands) else ("CONFLICT" if any(c["status"] == "CONFLICT" for c in cands) else (cands[0]["status"] if cands else "OBSERVED"))
+        out.append({"gtin": (prod.gtin or "")[-13:], "gtin_key": prod.gtin, "description": (prod.normalized_description or "")[:70], "status": status, "candidates": [c for c in cands if c["status"] != "STALE" or c["attested"] is None][:4]})
+    return out
+
+
+def kb_search(session, query: str, limit: int = 6) -> list[dict]:
+    from ..models import KbProduct
+    from . import kb as kbm
+    q = (query or "").strip()
+    if not q:
+        return []
+    digits = "".join(ch for ch in q if ch.isdigit())
+    products = []
+    if len(digits) >= 8 and len(digits) == len(q.replace(" ", "")):
+        p = kbm.find_by_gtin(session, digits.zfill(14))
+        products = [p] if p else []
+    else:
+        products = kbm.name_candidates(session, q)[:limit]
+    return _candidate_cards(session, products)
+
+
+def kb_summary(session, query: str | None = None) -> dict:
     """Compact Knowledge Base panel: known products, statuses of the NCM knowledge, hits today, and the latest conflicts with their evidence (no dashboard of its own)."""
     from ..models import KbKnowledge, KbProduct
     today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
@@ -102,4 +134,4 @@ def kb_summary(session) -> dict:
                           "candidates": [{"ncm": i.value, "clients": sorted({c for c in (i.evidence or {}).get("clients", [])}), "documents": sum(int(l.get("docs", 0)) for l in ((i.evidence or {}).get("lineages") or {}).values()),
                                           "providers": sorted(f for f in ((i.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:"))} for i in items]})
     return {"products": products, "verified": by.get("VERIFIED_KB", 0), "corroborated": by.get("CORROBORATED", 0) + by.get("EXTERNAL_CORROBORATED", 0), "observed": by.get("OBSERVED", 0),
-            "conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts}
+            "conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts, "search": kb_search(session, query) if query else []}

central/app/enrichment/status.py

@@ -0,0 +1,86 @@
+"""Human-facing status of every tracked product: nothing may silently 'do nothing'. Computed from the resolver result and refined when the Agent reports the write outcome."""
+from __future__ import annotations
+
+from sqlalchemy import select, update
+
+LABELS = {
+    "ENRIQUECIDO": "Enriquecido: campos seguros preenchidos.",
+    "PRONTO_PARA_PREENCHER": "Evidência suficiente: aguardando a escrita segura pelo Agent.",
+    "SIMULADO": "Simulação (DRY_RUN): há candidato, nada foi gravado.",
+    "AGUARDANDO_EVIDENCIA": "Aguardando evidência: nenhuma fonte confiável ainda.",
+    "EAN_AUSENTE": "Sem EAN e sem produto único verificado na base: informe o código de barras ou peça a decisão de um administrador.",
+    "EAN_INVALIDO": "Código de barras inválido: nenhuma fonte externa foi consultada.",
+    "NOME_AMBIGUO": "Nome genérico ou ambíguo demais para identificar o produto com segurança.",
+    "SEM_MATCH_DE_PRODUTO": "Nenhum produto conhecido corresponde a este nome/EAN.",
+    "NCM_NAO_VERIFICADO": "Produto identificado, mas o NCM ainda não foi verificado (falta segunda fonte independente ou decisão de administrador).",
+    "CONFLITO_DE_NCM": "As fontes divergem sobre o NCM: nada é preenchido até um administrador decidir.",
+    "PROVIDER_SEM_QUOTA": "Limite diário das fontes externas atingido; será repetido após o reset.",
+    "PROVIDER_NAO_CONFIGURADO": "Nenhuma fonte externa com NCM está configurada.",
+    "AGUARDANDO_RETRY": "Fontes indisponíveis agora; nova tentativa automática agendada.",
+    "HUMAN_OVERRIDE": "Alterado por uma pessoa: nunca mais preenchido automaticamente.",
+    "TECHNICAL_PLACEHOLDER": "NCM provisório (00000000) reconhecido como 'ainda não classificado'.",
+    "WRITE_FAILED": "A escrita segura foi recusada ou falhou (veja o motivo).",
+    "VERIFICACAO_POS_WRITE_FALHOU": "A escrita foi feita mas a releitura não confirmou o valor.",
+    "NADA_A_PREENCHER": "Todos os campos elegíveis já estão preenchidos.",
+    "FUNCAO_DESATIVADA": "Enriquecimento desativado para esta instalação.",
+}
+
+
+def classify(result: dict, *, has_gtin: bool, specific_name: bool) -> tuple[str, str]:
+    """(ui_status, reason_code) from a resolve() result."""
+    fields = result.get("fields") or {}
+    ncm = fields.get("CLASSIFICACAO_FISCAL") or {}
+    state = result.get("state")
+    if state == "FEATURE_DISABLED":
+        return "FUNCAO_DESATIVADA", state
+    if state == "INVALID_GTIN":
+        return "EAN_INVALIDO", state
+    if state == "PENDING_PROVIDER_QUOTA":
+        return "PROVIDER_SEM_QUOTA", state
+    if state in ("PENDING_PROVIDERS_OFFLINE", "PENDING_TIME_BUDGET"):
+        return "AGUARDANDO_RETRY", state
+    if state == "CONFLICT" or ncm.get("status") == "CONFLICT":
+        return "CONFLITO_DE_NCM", "CONFLICT"
+    if ncm.get("status") == "HUMAN_OVERRIDDEN":
+        return "HUMAN_OVERRIDE", "HUMAN_OVERRIDDEN"
+    if any(f.get("status") == "AUTO_FILL_SAFE" for f in fields.values()):
+        return "PRONTO_PARA_PREENCHER", state or "RESOLVED"
+    if ncm.get("status") == "ALREADY_SET" and not any(f.get("status") in ("AUTO_FILL_SAFE", "SUGGESTION_ONLY") for f in fields.values()):
+        return "NADA_A_PREENCHER", "ALL_FIELDS_SET"
+    if not has_gtin:
+        if state == "AMBIGUOUS" or not specific_name:
+            return "NOME_AMBIGUO", "GENERIC_OR_AMBIGUOUS_NAME"
+        return "EAN_AUSENTE", "NAME_ONLY_NO_VERIFIED_MATCH"
+    if ncm.get("status") == "SUGGESTION_ONLY":
+        return "NCM_NAO_VERIFICADO", ncm.get("rule") or "NCM_NOT_VERIFIED"
+    if state == "NOT_FOUND":
+        return "SEM_MATCH_DE_PRODUTO", "GTIN_UNKNOWN"
+    return "AGUARDANDO_EVIDENCIA", state or "INSUFFICIENT_EVIDENCE"
+
+
+def after_write(outcomes: list[dict], mode: str) -> tuple[str | None, str | None]:
+    """Refines the status with what the Agent reported (APPLIED / DRY_RUN / ABORTED / FAILED / HUMAN_OVERRIDDEN)."""
+    kinds = {o.get("outcome") for o in outcomes}
+    reasons = [o.get("reason") for o in outcomes if o.get("reason")]
+    if "HUMAN_OVERRIDDEN" in kinds and "APPLIED" not in kinds:
+        return "HUMAN_OVERRIDE", "HUMAN_OVERRIDDEN"
+    if "APPLIED" in kinds:
+        return "ENRIQUECIDO", None
+    if "FAILED" in kinds:
+        return ("VERIFICACAO_POS_WRITE_FALHOU" if "POST_WRITE_VERIFICATION_FAILED" in reasons else "WRITE_FAILED"), (reasons[0] if reasons else "FAILED")
+    if "ABORTED" in kinds:
+        return "WRITE_FAILED", (reasons[0] if reasons else "ABORTED")
+    if "DRY_RUN" in kinds:
+        return "SIMULADO", "DRY_RUN_NO_WRITE"
+    return None, None
+
+
+def mark_for_reevaluation(session, installation_id: int, *, codes: list[str] | None = None, gtin: str | None = None) -> int:
+    """Flags tracked, NOT-yet-enriched products for a new evaluation (KB/cache first; providers only when really needed). Idempotent; never touches enriched / human-overridden ones."""
+    from ..models import ProductEnrichment
+    q = update(ProductEnrichment).where(ProductEnrichment.installation_id == installation_id, ProductEnrichment.ui_status.notin_(["ENRIQUECIDO", "HUMAN_OVERRIDE"]) | ProductEnrichment.ui_status.is_(None))
+    if codes:
+        q = q.where(ProductEnrichment.product_code.in_(codes))
+    if gtin:
+        q = q.where(ProductEnrichment.gtin == gtin)
+    return session.execute(q.values(reevaluate=True)).rowcount or 0