feat: visible per-product enrichment status, flagged re-evaluation (panel/Reprocessar/Sincronizar/KB decision), operator NCM decision, deterministic name-only match, stale-write guard; 1.3.5
This commit is contained in:
@@ -10,6 +10,7 @@ from sqlalchemy import select
|
||||
from ..blocked_products import upsert_blocker
|
||||
from ..config import settings
|
||||
from ..models import Installation, ProductEnrichment, ProductFieldAudit
|
||||
from . import status as enrichment_status, textnorm
|
||||
from .placeholders import is_placeholder
|
||||
from .providers import build_providers
|
||||
from .quota import ProviderGateway
|
||||
@@ -80,6 +81,8 @@ def router(principal, providers_factory=None):
|
||||
decision["technical_placeholder"] = str(current.get(name))
|
||||
row.gtin, row.description, row.latency_ms = result.get("gtin"), description, latency
|
||||
row.attempts = (row.attempts or 0) + 1
|
||||
row.ui_status, row.reason_code = enrichment_status.classify(result, has_gtin=bool(result.get("gtin")), specific_name=textnorm.is_specific(description))
|
||||
row.last_attempt_at, row.reevaluate = datetime.now(timezone.utc), False
|
||||
row.status = "PENDING" if pending else "RESOLVED"
|
||||
row.decisions = {"state": result["state"], "fields": {k: {kk: vv for kk, vv in v.items() if kk != "evidence"} for k, v in result["fields"].items()}}
|
||||
row.next_attempt_at = _aware(result["retry_at"]) if pending else None
|
||||
@@ -124,6 +127,10 @@ def router(principal, providers_factory=None):
|
||||
elif outcome == "HUMAN_OVERRIDDEN":
|
||||
overridden[item["field"]] = True
|
||||
row.fields_applied, row.human_overridden = applied_fields, overridden
|
||||
ui, why = enrichment_status.after_write([i for i in (payload.get("fields") or []) if isinstance(i, dict)], installation.enrichment_mode)
|
||||
if ui:
|
||||
row.ui_status, row.reason_code = ui, why
|
||||
row.reevaluate = False
|
||||
row.status = "APPLIED" if applied_fields and not overridden else ("PARTIAL" if applied_fields else row.status if row.status != "PENDING" else "RESOLVED")
|
||||
row.updated_at = datetime.now(timezone.utc)
|
||||
session.commit()
|
||||
@@ -131,6 +138,19 @@ def router(principal, providers_factory=None):
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.get("/enrichment/pending")
|
||||
def pending_reevaluation(identity=Depends(principal)):
|
||||
"""Tracked products flagged for a new evaluation (panel button, REPROCESS/SYNC, KB decision). The Agent polls this cheaply and re-runs resolve for each code."""
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is None or not installation.features_configured or installation.enrichment_mode == "DISABLED":
|
||||
return {"codes": []}
|
||||
codes = list(session.scalars(select(ProductEnrichment.product_code).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.reevaluate.is_(True)).order_by(ProductEnrichment.id).limit(25)))
|
||||
return {"codes": codes}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.get("/enrichment/authorization")
|
||||
def authorization(identity=Depends(principal)):
|
||||
"""Fresh central authorization read by the Agent right before EVERY field write."""
|
||||
|
||||
@@ -36,13 +36,39 @@ def find_by_gtin(session, key: str | None) -> KbProduct | None:
|
||||
return session.scalar(select(KbProduct).where(KbProduct.gtin == key)) if key else None
|
||||
|
||||
|
||||
NEUTRAL_EXTRAS = frozenset({"REFRIGERANTE", "REFRIG", "REFR", "BEBIDA", "PRODUTO", "PROD"}) # category nouns only: ZERO / DIET / a flavour / another size are NEVER neutral
|
||||
|
||||
|
||||
def _tokset(text) -> frozenset:
|
||||
return frozenset(t for t in textnorm.tokens(text) if t not in textnorm.MARKETING)
|
||||
|
||||
|
||||
def alias_match(query: frozenset, alias: frozenset) -> bool:
|
||||
"""Deterministic, conservative: same token set, or one set contains the other and the difference is only neutral category nouns (so "COCA COLA LATA 350 ML" = "REFRIGERANTE COCA COLA LATA 350 ML", but not "... ZERO ...")."""
|
||||
if not query or not alias:
|
||||
return False
|
||||
if query == alias:
|
||||
return True
|
||||
if query < alias:
|
||||
return (alias - query) <= NEUTRAL_EXTRAS
|
||||
if alias < query:
|
||||
return (query - alias) <= NEUTRAL_EXTRAS
|
||||
return False
|
||||
|
||||
|
||||
def name_candidates(session, description) -> list[KbProduct]:
|
||||
"""Exact-fingerprint matches (0, 1 or several)."""
|
||||
"""Every KB product with an alias that matches the description under `alias_match` (exact fingerprint included). 0, 1 or several - the caller decides (unique VERIFIED only)."""
|
||||
fp = textnorm.fingerprint(description)
|
||||
if not fp:
|
||||
return []
|
||||
ids = list(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "name_fingerprint", KbKnowledge.value == fp).distinct()))
|
||||
return [p for p in (session.get(KbProduct, i) for i in ids) if p]
|
||||
ids = set(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "name_fingerprint", KbKnowledge.value == fp).distinct()))
|
||||
query = _tokset(description)
|
||||
anchor = max((t for t in query if not t.replace(".", "").isdigit() and len(t) > 2), key=len, default=None) # most selective informative token, then verify in Python
|
||||
if anchor:
|
||||
for row in session.execute(select(KbKnowledge.kb_product_id, KbKnowledge.value).where(KbKnowledge.knowledge_key == "alias", KbKnowledge.value.contains(anchor))):
|
||||
if alias_match(query, _tokset(row[1])):
|
||||
ids.add(row[0])
|
||||
return [p for p in (session.get(KbProduct, i) for i in sorted(ids)) if p]
|
||||
|
||||
|
||||
def upsert_product(session, key: str | None, description, *, brand=None, quantity=None) -> KbProduct:
|
||||
@@ -140,6 +166,28 @@ def _mark(row, state):
|
||||
row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "historical_state": state}}
|
||||
|
||||
|
||||
OPERATOR_FAMILY = "OPERATOR_ATTESTATION"
|
||||
|
||||
|
||||
def attest(session, gtin_key: str, ncm: str, operator: str, is_official, note: str = "") -> dict:
|
||||
"""A human fiscal decision by an ADMIN (audited by the caller). Only a value that is ALREADY a candidate in the KB (some local/provider evidence) and vigente in the official dataset can be attested;
|
||||
the dissenting candidates become STALE (REJECTED_BY_OPERATOR) and the product is promoted with the promotion audit naming the operator."""
|
||||
product = find_by_gtin(session, gtin_key)
|
||||
if product is None:
|
||||
raise ValueError("GTIN_UNKNOWN_IN_KB")
|
||||
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm", KbKnowledge.value == ncm))
|
||||
if row is None:
|
||||
raise ValueError("NCM_IS_NOT_A_CANDIDATE_OF_THIS_GTIN")
|
||||
if not is_official(ncm):
|
||||
raise ValueError("NCM_NOT_VIGENTE_IN_OFFICIAL_DATASET")
|
||||
add_evidence(session, product, "ncm", ncm, f"operator:{operator}"[:32], family=OPERATOR_FAMILY, info={"attested_by": operator, "attested_at": _now().isoformat(), "note": note[:200]})
|
||||
for other in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm", KbKnowledge.value != ncm)):
|
||||
if OPERATOR_FAMILY not in ((other.evidence or {}).get("families") or {}):
|
||||
other.status = STALE
|
||||
_mark(other, "REJECTED_BY_OPERATOR")
|
||||
return evaluate(session, product, "ncm", is_official)
|
||||
|
||||
|
||||
def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
|
||||
"""Recomputes and PERSISTS the status of one field. Never 'majority wins': two current-valid values => CONFLICT."""
|
||||
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key)))
|
||||
@@ -152,6 +200,14 @@ def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
|
||||
state = _official_period(session, r.value) if key == "ncm" else "HISTORICAL_INVALID"
|
||||
r.status = STALE if state == "HISTORICAL_EXPIRED" else OBSERVED
|
||||
_mark(r, state)
|
||||
attested = {r.value for r in valid if OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {})}
|
||||
if len(attested) == 1: # a human decision resolves the candidates: the others are recorded as rejected, not "outvoted"
|
||||
winner = next(iter(attested))
|
||||
for r in valid:
|
||||
if r.value != winner:
|
||||
r.status = STALE
|
||||
_mark(r, "REJECTED_BY_OPERATOR")
|
||||
valid = [r for r in valid if r.value == winner]
|
||||
values = sorted({r.value for r in valid})
|
||||
if len(values) > 1:
|
||||
for r in valid:
|
||||
@@ -169,6 +225,9 @@ def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
|
||||
if superseded and CLIENT_FAMILY in names:
|
||||
names.remove(CLIENT_FAMILY)
|
||||
count -= 1
|
||||
if any(OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen):
|
||||
names = sorted(set(names) | {OPERATOR_FAMILY} | ({CLIENT_FAMILY} if any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen) else set()))
|
||||
count = len(names)
|
||||
providers = [n for n in names if n.startswith("PROVIDER:")]
|
||||
already = any(r.status == VERIFIED_KB and r.validated_at for r in chosen)
|
||||
has_client_data = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen)
|
||||
@@ -188,7 +247,8 @@ def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
|
||||
if status == VERIFIED_KB and old != VERIFIED_KB and key == "ncm":
|
||||
from .resolver import dataset_versions
|
||||
session.add(KbPromotionAudit(kb_product_id=product.id, gtin=product.gtin, field=key, value=values[0], old_status=old, new_status=status, families=names,
|
||||
sources={"clients": summary["clients"], "documents": summary["docs"], "years": summary["years"], "providers": providers},
|
||||
sources={"clients": summary["clients"], "documents": summary["docs"], "years": summary["years"], "providers": providers,
|
||||
"operator": next((((r.evidence or {}).get("info") or {}).get("attested_by") for r in chosen if OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {})), None)},
|
||||
official_dataset=dataset_versions(session).get("NCM", {}), rule_version=RULE_VERSION))
|
||||
return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected, "detail": summary}
|
||||
|
||||
|
||||
@@ -84,7 +84,39 @@ def coverage_metrics(session) -> dict:
|
||||
"provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}
|
||||
|
||||
|
||||
def kb_summary(session) -> dict:
|
||||
def _candidate_cards(session, products) -> list[dict]:
|
||||
from ..models import Dataset, KbKnowledge, NcmEntry
|
||||
out = []
|
||||
for prod in products:
|
||||
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == prod.id, KbKnowledge.knowledge_key == "ncm")))
|
||||
cands = []
|
||||
for r in rows:
|
||||
ev = r.evidence or {}
|
||||
text = session.scalar(select(NcmEntry.description).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == r.value).limit(1))
|
||||
cands.append({"ncm": r.value, "official_text": text, "status": r.status, "clients": sorted(ev.get("clients", [])), "documents": sum(int(l.get("docs", 0)) for l in (ev.get("lineages") or {}).values()),
|
||||
"providers": sorted(f for f in (ev.get("families") or {}) if f.startswith("PROVIDER:")), "attested": (ev.get("info") or {}).get("attested_by")})
|
||||
status = "VERIFIED_KB" if any(c["status"] == "VERIFIED_KB" for c in cands) else ("CONFLICT" if any(c["status"] == "CONFLICT" for c in cands) else (cands[0]["status"] if cands else "OBSERVED"))
|
||||
out.append({"gtin": (prod.gtin or "")[-13:], "gtin_key": prod.gtin, "description": (prod.normalized_description or "")[:70], "status": status, "candidates": [c for c in cands if c["status"] != "STALE" or c["attested"] is None][:4]})
|
||||
return out
|
||||
|
||||
|
||||
def kb_search(session, query: str, limit: int = 6) -> list[dict]:
|
||||
from ..models import KbProduct
|
||||
from . import kb as kbm
|
||||
q = (query or "").strip()
|
||||
if not q:
|
||||
return []
|
||||
digits = "".join(ch for ch in q if ch.isdigit())
|
||||
products = []
|
||||
if len(digits) >= 8 and len(digits) == len(q.replace(" ", "")):
|
||||
p = kbm.find_by_gtin(session, digits.zfill(14))
|
||||
products = [p] if p else []
|
||||
else:
|
||||
products = kbm.name_candidates(session, q)[:limit]
|
||||
return _candidate_cards(session, products)
|
||||
|
||||
|
||||
def kb_summary(session, query: str | None = None) -> dict:
|
||||
"""Compact Knowledge Base panel: known products, statuses of the NCM knowledge, hits today, and the latest conflicts with their evidence (no dashboard of its own)."""
|
||||
from ..models import KbKnowledge, KbProduct
|
||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
@@ -102,4 +134,4 @@ def kb_summary(session) -> dict:
|
||||
"candidates": [{"ncm": i.value, "clients": sorted({c for c in (i.evidence or {}).get("clients", [])}), "documents": sum(int(l.get("docs", 0)) for l in ((i.evidence or {}).get("lineages") or {}).values()),
|
||||
"providers": sorted(f for f in ((i.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:"))} for i in items]})
|
||||
return {"products": products, "verified": by.get("VERIFIED_KB", 0), "corroborated": by.get("CORROBORATED", 0) + by.get("EXTERNAL_CORROBORATED", 0), "observed": by.get("OBSERVED", 0),
|
||||
"conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts}
|
||||
"conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts, "search": kb_search(session, query) if query else []}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Human-facing status of every tracked product: nothing may silently 'do nothing'. Computed from the resolver result and refined when the Agent reports the write outcome."""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select, update
|
||||
|
||||
LABELS = {
|
||||
"ENRIQUECIDO": "Enriquecido: campos seguros preenchidos.",
|
||||
"PRONTO_PARA_PREENCHER": "Evidência suficiente: aguardando a escrita segura pelo Agent.",
|
||||
"SIMULADO": "Simulação (DRY_RUN): há candidato, nada foi gravado.",
|
||||
"AGUARDANDO_EVIDENCIA": "Aguardando evidência: nenhuma fonte confiável ainda.",
|
||||
"EAN_AUSENTE": "Sem EAN e sem produto único verificado na base: informe o código de barras ou peça a decisão de um administrador.",
|
||||
"EAN_INVALIDO": "Código de barras inválido: nenhuma fonte externa foi consultada.",
|
||||
"NOME_AMBIGUO": "Nome genérico ou ambíguo demais para identificar o produto com segurança.",
|
||||
"SEM_MATCH_DE_PRODUTO": "Nenhum produto conhecido corresponde a este nome/EAN.",
|
||||
"NCM_NAO_VERIFICADO": "Produto identificado, mas o NCM ainda não foi verificado (falta segunda fonte independente ou decisão de administrador).",
|
||||
"CONFLITO_DE_NCM": "As fontes divergem sobre o NCM: nada é preenchido até um administrador decidir.",
|
||||
"PROVIDER_SEM_QUOTA": "Limite diário das fontes externas atingido; será repetido após o reset.",
|
||||
"PROVIDER_NAO_CONFIGURADO": "Nenhuma fonte externa com NCM está configurada.",
|
||||
"AGUARDANDO_RETRY": "Fontes indisponíveis agora; nova tentativa automática agendada.",
|
||||
"HUMAN_OVERRIDE": "Alterado por uma pessoa: nunca mais preenchido automaticamente.",
|
||||
"TECHNICAL_PLACEHOLDER": "NCM provisório (00000000) reconhecido como 'ainda não classificado'.",
|
||||
"WRITE_FAILED": "A escrita segura foi recusada ou falhou (veja o motivo).",
|
||||
"VERIFICACAO_POS_WRITE_FALHOU": "A escrita foi feita mas a releitura não confirmou o valor.",
|
||||
"NADA_A_PREENCHER": "Todos os campos elegíveis já estão preenchidos.",
|
||||
"FUNCAO_DESATIVADA": "Enriquecimento desativado para esta instalação.",
|
||||
}
|
||||
|
||||
|
||||
def classify(result: dict, *, has_gtin: bool, specific_name: bool) -> tuple[str, str]:
|
||||
"""(ui_status, reason_code) from a resolve() result."""
|
||||
fields = result.get("fields") or {}
|
||||
ncm = fields.get("CLASSIFICACAO_FISCAL") or {}
|
||||
state = result.get("state")
|
||||
if state == "FEATURE_DISABLED":
|
||||
return "FUNCAO_DESATIVADA", state
|
||||
if state == "INVALID_GTIN":
|
||||
return "EAN_INVALIDO", state
|
||||
if state == "PENDING_PROVIDER_QUOTA":
|
||||
return "PROVIDER_SEM_QUOTA", state
|
||||
if state in ("PENDING_PROVIDERS_OFFLINE", "PENDING_TIME_BUDGET"):
|
||||
return "AGUARDANDO_RETRY", state
|
||||
if state == "CONFLICT" or ncm.get("status") == "CONFLICT":
|
||||
return "CONFLITO_DE_NCM", "CONFLICT"
|
||||
if ncm.get("status") == "HUMAN_OVERRIDDEN":
|
||||
return "HUMAN_OVERRIDE", "HUMAN_OVERRIDDEN"
|
||||
if any(f.get("status") == "AUTO_FILL_SAFE" for f in fields.values()):
|
||||
return "PRONTO_PARA_PREENCHER", state or "RESOLVED"
|
||||
if ncm.get("status") == "ALREADY_SET" and not any(f.get("status") in ("AUTO_FILL_SAFE", "SUGGESTION_ONLY") for f in fields.values()):
|
||||
return "NADA_A_PREENCHER", "ALL_FIELDS_SET"
|
||||
if not has_gtin:
|
||||
if state == "AMBIGUOUS" or not specific_name:
|
||||
return "NOME_AMBIGUO", "GENERIC_OR_AMBIGUOUS_NAME"
|
||||
return "EAN_AUSENTE", "NAME_ONLY_NO_VERIFIED_MATCH"
|
||||
if ncm.get("status") == "SUGGESTION_ONLY":
|
||||
return "NCM_NAO_VERIFICADO", ncm.get("rule") or "NCM_NOT_VERIFIED"
|
||||
if state == "NOT_FOUND":
|
||||
return "SEM_MATCH_DE_PRODUTO", "GTIN_UNKNOWN"
|
||||
return "AGUARDANDO_EVIDENCIA", state or "INSUFFICIENT_EVIDENCE"
|
||||
|
||||
|
||||
def after_write(outcomes: list[dict], mode: str) -> tuple[str | None, str | None]:
|
||||
"""Refines the status with what the Agent reported (APPLIED / DRY_RUN / ABORTED / FAILED / HUMAN_OVERRIDDEN)."""
|
||||
kinds = {o.get("outcome") for o in outcomes}
|
||||
reasons = [o.get("reason") for o in outcomes if o.get("reason")]
|
||||
if "HUMAN_OVERRIDDEN" in kinds and "APPLIED" not in kinds:
|
||||
return "HUMAN_OVERRIDE", "HUMAN_OVERRIDDEN"
|
||||
if "APPLIED" in kinds:
|
||||
return "ENRIQUECIDO", None
|
||||
if "FAILED" in kinds:
|
||||
return ("VERIFICACAO_POS_WRITE_FALHOU" if "POST_WRITE_VERIFICATION_FAILED" in reasons else "WRITE_FAILED"), (reasons[0] if reasons else "FAILED")
|
||||
if "ABORTED" in kinds:
|
||||
return "WRITE_FAILED", (reasons[0] if reasons else "ABORTED")
|
||||
if "DRY_RUN" in kinds:
|
||||
return "SIMULADO", "DRY_RUN_NO_WRITE"
|
||||
return None, None
|
||||
|
||||
|
||||
def mark_for_reevaluation(session, installation_id: int, *, codes: list[str] | None = None, gtin: str | None = None) -> int:
|
||||
"""Flags tracked, NOT-yet-enriched products for a new evaluation (KB/cache first; providers only when really needed). Idempotent; never touches enriched / human-overridden ones."""
|
||||
from ..models import ProductEnrichment
|
||||
q = update(ProductEnrichment).where(ProductEnrichment.installation_id == installation_id, ProductEnrichment.ui_status.notin_(["ENRIQUECIDO", "HUMAN_OVERRIDE"]) | ProductEnrichment.ui_status.is_(None))
|
||||
if codes:
|
||||
q = q.where(ProductEnrichment.product_code.in_(codes))
|
||||
if gtin:
|
||||
q = q.where(ProductEnrichment.gtin == gtin)
|
||||
return session.execute(q.values(reevaluate=True)).rowcount or 0
|
||||
@@ -30,6 +30,9 @@ def create_job(session, agent, kind, payload=None):
|
||||
(no duplicates, never two full jobs at once)."""
|
||||
if kind not in _VALID_KINDS:
|
||||
raise ValueError(f"unsupported job kind: {kind}")
|
||||
if kind in ("REPROCESS_ALL", "REPROCESS_PRODUCTS", "SYNC_INCREMENTAL"):
|
||||
_flag_enrichment_reevaluation(session, agent) # "Reprocessar"/"Sincronizar" also re-evaluate the tracked, not-yet-enriched products (KB/cache first), even when an equivalent job is already queued
|
||||
session.commit()
|
||||
if kind in SYNC_KINDS:
|
||||
# A full job subsumes an incremental one, not the other way round: asking for a
|
||||
# full reprocess while only an incremental is queued must still queue the full.
|
||||
@@ -43,6 +46,16 @@ def create_job(session, agent, kind, payload=None):
|
||||
return job
|
||||
|
||||
|
||||
def _flag_enrichment_reevaluation(session, agent):
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is not None and installation.features_configured and installation.enrichment_mode != "DISABLED":
|
||||
from .enrichment.status import mark_for_reevaluation
|
||||
mark_for_reevaluation(session, installation.id)
|
||||
except Exception: # noqa: BLE001 - never let the enrichment bookkeeping break job creation
|
||||
pass
|
||||
|
||||
|
||||
def recover_stale_running(session, agent, now=None):
|
||||
now = now or datetime.now(timezone.utc)
|
||||
cutoff = now - STALE_RUNNING_AFTER
|
||||
|
||||
@@ -816,6 +816,10 @@ class ProductEnrichment(Base):
|
||||
attempts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
||||
latency_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
reevaluate: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false", index=True) # set by panel / REPROCESS / SYNC / KB change; consumed by the Agent
|
||||
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
ui_status: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) # human-facing state (never a silent product)
|
||||
reason_code: Mapped[str | None] = mapped_column(String(48), nullable=True)
|
||||
detected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
|
||||
once published — never a second, independent version number. Fiscal semantics
|
||||
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
|
||||
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
|
||||
GTI_FISCAL_VERSION = "1.3.4"
|
||||
GTI_FISCAL_VERSION = "1.3.5"
|
||||
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
|
||||
|
||||
+60
-3
@@ -22,6 +22,7 @@ from .panel_presentation import format_ui_datetime
|
||||
from .models import AdminUser, Agent, AgentEnrollment, AgentRelease, AgentSnapshot, BlockedProduct, FiscalContextEntry, Installation, Job, ProductEnrichment, ProductFieldAudit, RtcCandidate, RtcWriteAudit, Tenant
|
||||
from .jobs import create_job
|
||||
from .audit import record_audit
|
||||
from .enrichment import kb as kb_mod, status as enrichment_status
|
||||
from .enrichment.observability import coverage_metrics, kb_summary, official_sources, provider_sources
|
||||
|
||||
|
||||
@@ -158,10 +159,10 @@ def router(session_factory):
|
||||
return {key: items[0] for key, items in parse_qs((await request.body()).decode()).items() if items}
|
||||
|
||||
@routes.get("/", response_class=HTMLResponse)
|
||||
def dashboard(request: Request):
|
||||
def dashboard(request: Request, kb_q: str | None = None):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session), "kb": kb_summary(session)})
|
||||
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "kb_q": kb_q or "", "is_admin": _is_privileged(auth_data[1]), "csrf_token": request.cookies.get("gti_csrf", ""), "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session), "kb": kb_summary(session, kb_q)})
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents", response_class=HTMLResponse)
|
||||
@@ -302,10 +303,18 @@ def router(session_factory):
|
||||
row = session.scalar(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.product_code == enrich_code.strip()[:64]))
|
||||
audits = session.scalars(select(ProductFieldAudit).where(ProductFieldAudit.installation_id == installation.id, ProductFieldAudit.product_code == enrich_code.strip()[:64]).order_by(ProductFieldAudit.id.desc()).limit(20)).all() if row else []
|
||||
lookup = {"found": row is not None, "row": row, "audits": audits}
|
||||
tracked = []
|
||||
for r in session.scalars(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id).order_by(ProductEnrichment.id.desc()).limit(15)):
|
||||
fields = ((r.decisions or {}).get("fields") or {})
|
||||
ncm = fields.get("CLASSIFICACAO_FISCAL") or {}
|
||||
tracked.append({"code": r.product_code, "description": r.description, "gtin": r.gtin, "ui_status": r.ui_status or "AGUARDANDO_EVIDENCIA", "label": enrichment_status.LABELS.get(r.ui_status or "AGUARDANDO_EVIDENCIA", ""),
|
||||
"reason": r.reason_code, "detected": r.detected_at, "last_attempt": r.last_attempt_at, "next_retry": r.next_attempt_at, "kb_status": ncm.get("kb_status"), "ncm_candidate": ncm.get("value"),
|
||||
"cest_candidate": (fields.get("CEST") or {}).get("value"), "ipi_candidate": (fields.get("IPI") or {}).get("value"), "applied": r.fields_applied or {}, "placeholder": ncm.get("technical_placeholder"),
|
||||
"reevaluate": bool(r.reevaluate), "mode": installation.enrichment_mode})
|
||||
mode = installation.enrichment_mode
|
||||
waiting = not installation.features_configured
|
||||
return {"configured": bool(installation.features_configured), "waiting": waiting, "corrector_enabled": bool(installation.corrector_enabled), "enrichment_mode": mode,
|
||||
"mode_label": ENRICHMENT_MODE_LABELS.get(mode, mode), "counts": counts, "total": sum(counts.values()), "applied": applied, "lookup": lookup,
|
||||
"mode_label": ENRICHMENT_MODE_LABELS.get(mode, mode), "counts": counts, "total": sum(counts.values()), "applied": applied, "lookup": lookup, "tracked": tracked,
|
||||
"state_label": "Aguardando configuração de funcionalidades" if waiting else ENRICHMENT_MODE_LABELS.get(mode, mode)}
|
||||
|
||||
@routes.post("/agents/{agent_id}/features")
|
||||
@@ -335,6 +344,54 @@ def router(session_factory):
|
||||
return RedirectResponse(f"/agents/{agent_id}", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/{agent_id}/enrichment/reevaluate")
|
||||
async def reevaluate_enrichment(request: Request, agent_id: int):
|
||||
"""Explicit, idempotent 'evaluate again' for one product (code=...) or every tracked product that is not enriched yet. KB/cache first; providers only when really needed."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
agent, installation = _rtc_target(session, agent_id)
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
if not installation.features_configured or installation.enrichment_mode == "DISABLED":
|
||||
return HTMLResponse("O enriquecimento está desativado nesta instalação (Funcionalidades).", status_code=409)
|
||||
code = (values.get("code") or "").strip()[:64]
|
||||
n = enrichment_status.mark_for_reevaluation(session, installation.id, codes=[code] if code else None)
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="ENRICHMENT_REEVALUATE", target_type="installation", target_id=installation.installation_id,
|
||||
context={"product": code or "ALL_UNRESOLVED", "flagged": n, "operator": admin.display_name or admin.username})
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}#enriquecimento", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/kb/attest")
|
||||
async def kb_attest(request: Request):
|
||||
"""A human fiscal decision (ADMIN only): choose which candidate NCM is right for a GTIN. Only an existing, vigente candidate can be chosen; the others are recorded as rejected."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
if not _is_privileged(admin):
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=None, action="KB_ATTEST_DENIED", target_type="kb_product", target_id=values.get("gtin") or "", context={"role": admin.external_role or admin.auth_source}, severity="WARNING")
|
||||
session.commit()
|
||||
return HTMLResponse("Ação restrita a administradores do GTI Gerenciador (perfil ADMIN).", status_code=403)
|
||||
gtin, ncm = (values.get("gtin") or "").strip(), (values.get("ncm") or "").strip()
|
||||
from .enrichment import history
|
||||
current = history.official_sets(session)
|
||||
try:
|
||||
result = kb_mod.attest(session, gtin, ncm, admin.display_name or admin.username, lambda v: v in current, note=(values.get("note") or "")[:200])
|
||||
except ValueError as error:
|
||||
return HTMLResponse(f"Decisão recusada: {error}", status_code=422)
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=None, action="KB_NCM_ATTESTED", target_type="kb_product", target_id=gtin,
|
||||
context={"ncm": ncm, "status": result["status"], "families": result["families"], "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source}, severity="WARNING")
|
||||
for inst in session.scalars(select(Installation).where(Installation.enrichment_mode != "DISABLED", Installation.features_configured.is_(True))):
|
||||
enrichment_status.mark_for_reevaluation(session, inst.id, gtin=gtin)
|
||||
enrichment_status.mark_for_reevaluation(session, inst.id) # products identified by name are found through the KB aliases
|
||||
session.commit()
|
||||
return RedirectResponse("/#kb-resumo", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
def _ops_view(session, installation, agent):
|
||||
"""Processing status shown on the Agent detail (no clutter: 5 facts)."""
|
||||
last_job = session.scalar(select(Job).where(Job.agent_id == agent.id).order_by(Job.id.desc()).limit(1))
|
||||
|
||||
Reference in New Issue
Block a user