feat: visible per-product enrichment status, flagged re-evaluation (panel/Reprocessar/Sincronizar/KB decision), operator NCM decision, deterministic name-only match, stale-write guard; 1.3.5
This commit is contained in:
@@ -0,0 +1,28 @@
|
||||
"""Enrichment re-evaluation + visible per-product status (no silent failures).
|
||||
|
||||
Revision ID: 0031_enrichment_reevaluate
|
||||
Revises: 0030_kb_evidence_model
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0031_enrichment_reevaluate"
|
||||
down_revision = "0030_kb_evidence_model"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("product_enrichments", sa.Column("reevaluate", sa.Boolean(), nullable=False, server_default=sa.text("false")))
|
||||
op.add_column("product_enrichments", sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("product_enrichments", sa.Column("ui_status", sa.String(32), nullable=True))
|
||||
op.add_column("product_enrichments", sa.Column("reason_code", sa.String(48), nullable=True))
|
||||
op.create_index("ix_product_enrichments_reevaluate", "product_enrichments", ["reevaluate"])
|
||||
op.create_index("ix_product_enrichments_ui_status", "product_enrichments", ["ui_status"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index("ix_product_enrichments_ui_status", table_name="product_enrichments")
|
||||
op.drop_index("ix_product_enrichments_reevaluate", table_name="product_enrichments")
|
||||
for name in ("reason_code", "ui_status", "last_attempt_at", "reevaluate"):
|
||||
op.drop_column("product_enrichments", name)
|
||||
@@ -10,6 +10,7 @@ from sqlalchemy import select
|
||||
from ..blocked_products import upsert_blocker
|
||||
from ..config import settings
|
||||
from ..models import Installation, ProductEnrichment, ProductFieldAudit
|
||||
from . import status as enrichment_status, textnorm
|
||||
from .placeholders import is_placeholder
|
||||
from .providers import build_providers
|
||||
from .quota import ProviderGateway
|
||||
@@ -80,6 +81,8 @@ def router(principal, providers_factory=None):
|
||||
decision["technical_placeholder"] = str(current.get(name))
|
||||
row.gtin, row.description, row.latency_ms = result.get("gtin"), description, latency
|
||||
row.attempts = (row.attempts or 0) + 1
|
||||
row.ui_status, row.reason_code = enrichment_status.classify(result, has_gtin=bool(result.get("gtin")), specific_name=textnorm.is_specific(description))
|
||||
row.last_attempt_at, row.reevaluate = datetime.now(timezone.utc), False
|
||||
row.status = "PENDING" if pending else "RESOLVED"
|
||||
row.decisions = {"state": result["state"], "fields": {k: {kk: vv for kk, vv in v.items() if kk != "evidence"} for k, v in result["fields"].items()}}
|
||||
row.next_attempt_at = _aware(result["retry_at"]) if pending else None
|
||||
@@ -124,6 +127,10 @@ def router(principal, providers_factory=None):
|
||||
elif outcome == "HUMAN_OVERRIDDEN":
|
||||
overridden[item["field"]] = True
|
||||
row.fields_applied, row.human_overridden = applied_fields, overridden
|
||||
ui, why = enrichment_status.after_write([i for i in (payload.get("fields") or []) if isinstance(i, dict)], installation.enrichment_mode)
|
||||
if ui:
|
||||
row.ui_status, row.reason_code = ui, why
|
||||
row.reevaluate = False
|
||||
row.status = "APPLIED" if applied_fields and not overridden else ("PARTIAL" if applied_fields else row.status if row.status != "PENDING" else "RESOLVED")
|
||||
row.updated_at = datetime.now(timezone.utc)
|
||||
session.commit()
|
||||
@@ -131,6 +138,19 @@ def router(principal, providers_factory=None):
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.get("/enrichment/pending")
|
||||
def pending_reevaluation(identity=Depends(principal)):
|
||||
"""Tracked products flagged for a new evaluation (panel button, REPROCESS/SYNC, KB decision). The Agent polls this cheaply and re-runs resolve for each code."""
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is None or not installation.features_configured or installation.enrichment_mode == "DISABLED":
|
||||
return {"codes": []}
|
||||
codes = list(session.scalars(select(ProductEnrichment.product_code).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.reevaluate.is_(True)).order_by(ProductEnrichment.id).limit(25)))
|
||||
return {"codes": codes}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.get("/enrichment/authorization")
|
||||
def authorization(identity=Depends(principal)):
|
||||
"""Fresh central authorization read by the Agent right before EVERY field write."""
|
||||
|
||||
@@ -36,13 +36,39 @@ def find_by_gtin(session, key: str | None) -> KbProduct | None:
|
||||
return session.scalar(select(KbProduct).where(KbProduct.gtin == key)) if key else None
|
||||
|
||||
|
||||
NEUTRAL_EXTRAS = frozenset({"REFRIGERANTE", "REFRIG", "REFR", "BEBIDA", "PRODUTO", "PROD"}) # category nouns only: ZERO / DIET / a flavour / another size are NEVER neutral
|
||||
|
||||
|
||||
def _tokset(text) -> frozenset:
|
||||
return frozenset(t for t in textnorm.tokens(text) if t not in textnorm.MARKETING)
|
||||
|
||||
|
||||
def alias_match(query: frozenset, alias: frozenset) -> bool:
|
||||
"""Deterministic, conservative: same token set, or one set contains the other and the difference is only neutral category nouns (so "COCA COLA LATA 350 ML" = "REFRIGERANTE COCA COLA LATA 350 ML", but not "... ZERO ...")."""
|
||||
if not query or not alias:
|
||||
return False
|
||||
if query == alias:
|
||||
return True
|
||||
if query < alias:
|
||||
return (alias - query) <= NEUTRAL_EXTRAS
|
||||
if alias < query:
|
||||
return (query - alias) <= NEUTRAL_EXTRAS
|
||||
return False
|
||||
|
||||
|
||||
def name_candidates(session, description) -> list[KbProduct]:
|
||||
"""Exact-fingerprint matches (0, 1 or several)."""
|
||||
"""Every KB product with an alias that matches the description under `alias_match` (exact fingerprint included). 0, 1 or several - the caller decides (unique VERIFIED only)."""
|
||||
fp = textnorm.fingerprint(description)
|
||||
if not fp:
|
||||
return []
|
||||
ids = list(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "name_fingerprint", KbKnowledge.value == fp).distinct()))
|
||||
return [p for p in (session.get(KbProduct, i) for i in ids) if p]
|
||||
ids = set(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "name_fingerprint", KbKnowledge.value == fp).distinct()))
|
||||
query = _tokset(description)
|
||||
anchor = max((t for t in query if not t.replace(".", "").isdigit() and len(t) > 2), key=len, default=None) # most selective informative token, then verify in Python
|
||||
if anchor:
|
||||
for row in session.execute(select(KbKnowledge.kb_product_id, KbKnowledge.value).where(KbKnowledge.knowledge_key == "alias", KbKnowledge.value.contains(anchor))):
|
||||
if alias_match(query, _tokset(row[1])):
|
||||
ids.add(row[0])
|
||||
return [p for p in (session.get(KbProduct, i) for i in sorted(ids)) if p]
|
||||
|
||||
|
||||
def upsert_product(session, key: str | None, description, *, brand=None, quantity=None) -> KbProduct:
|
||||
@@ -140,6 +166,28 @@ def _mark(row, state):
|
||||
row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "historical_state": state}}
|
||||
|
||||
|
||||
OPERATOR_FAMILY = "OPERATOR_ATTESTATION"
|
||||
|
||||
|
||||
def attest(session, gtin_key: str, ncm: str, operator: str, is_official, note: str = "") -> dict:
|
||||
"""A human fiscal decision by an ADMIN (audited by the caller). Only a value that is ALREADY a candidate in the KB (some local/provider evidence) and vigente in the official dataset can be attested;
|
||||
the dissenting candidates become STALE (REJECTED_BY_OPERATOR) and the product is promoted with the promotion audit naming the operator."""
|
||||
product = find_by_gtin(session, gtin_key)
|
||||
if product is None:
|
||||
raise ValueError("GTIN_UNKNOWN_IN_KB")
|
||||
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm", KbKnowledge.value == ncm))
|
||||
if row is None:
|
||||
raise ValueError("NCM_IS_NOT_A_CANDIDATE_OF_THIS_GTIN")
|
||||
if not is_official(ncm):
|
||||
raise ValueError("NCM_NOT_VIGENTE_IN_OFFICIAL_DATASET")
|
||||
add_evidence(session, product, "ncm", ncm, f"operator:{operator}"[:32], family=OPERATOR_FAMILY, info={"attested_by": operator, "attested_at": _now().isoformat(), "note": note[:200]})
|
||||
for other in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm", KbKnowledge.value != ncm)):
|
||||
if OPERATOR_FAMILY not in ((other.evidence or {}).get("families") or {}):
|
||||
other.status = STALE
|
||||
_mark(other, "REJECTED_BY_OPERATOR")
|
||||
return evaluate(session, product, "ncm", is_official)
|
||||
|
||||
|
||||
def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
|
||||
"""Recomputes and PERSISTS the status of one field. Never 'majority wins': two current-valid values => CONFLICT."""
|
||||
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key)))
|
||||
@@ -152,6 +200,14 @@ def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
|
||||
state = _official_period(session, r.value) if key == "ncm" else "HISTORICAL_INVALID"
|
||||
r.status = STALE if state == "HISTORICAL_EXPIRED" else OBSERVED
|
||||
_mark(r, state)
|
||||
attested = {r.value for r in valid if OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {})}
|
||||
if len(attested) == 1: # a human decision resolves the candidates: the others are recorded as rejected, not "outvoted"
|
||||
winner = next(iter(attested))
|
||||
for r in valid:
|
||||
if r.value != winner:
|
||||
r.status = STALE
|
||||
_mark(r, "REJECTED_BY_OPERATOR")
|
||||
valid = [r for r in valid if r.value == winner]
|
||||
values = sorted({r.value for r in valid})
|
||||
if len(values) > 1:
|
||||
for r in valid:
|
||||
@@ -169,6 +225,9 @@ def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
|
||||
if superseded and CLIENT_FAMILY in names:
|
||||
names.remove(CLIENT_FAMILY)
|
||||
count -= 1
|
||||
if any(OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen):
|
||||
names = sorted(set(names) | {OPERATOR_FAMILY} | ({CLIENT_FAMILY} if any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen) else set()))
|
||||
count = len(names)
|
||||
providers = [n for n in names if n.startswith("PROVIDER:")]
|
||||
already = any(r.status == VERIFIED_KB and r.validated_at for r in chosen)
|
||||
has_client_data = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen)
|
||||
@@ -188,7 +247,8 @@ def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
|
||||
if status == VERIFIED_KB and old != VERIFIED_KB and key == "ncm":
|
||||
from .resolver import dataset_versions
|
||||
session.add(KbPromotionAudit(kb_product_id=product.id, gtin=product.gtin, field=key, value=values[0], old_status=old, new_status=status, families=names,
|
||||
sources={"clients": summary["clients"], "documents": summary["docs"], "years": summary["years"], "providers": providers},
|
||||
sources={"clients": summary["clients"], "documents": summary["docs"], "years": summary["years"], "providers": providers,
|
||||
"operator": next((((r.evidence or {}).get("info") or {}).get("attested_by") for r in chosen if OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {})), None)},
|
||||
official_dataset=dataset_versions(session).get("NCM", {}), rule_version=RULE_VERSION))
|
||||
return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected, "detail": summary}
|
||||
|
||||
|
||||
@@ -84,7 +84,39 @@ def coverage_metrics(session) -> dict:
|
||||
"provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}
|
||||
|
||||
|
||||
def kb_summary(session) -> dict:
|
||||
def _candidate_cards(session, products) -> list[dict]:
|
||||
from ..models import Dataset, KbKnowledge, NcmEntry
|
||||
out = []
|
||||
for prod in products:
|
||||
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == prod.id, KbKnowledge.knowledge_key == "ncm")))
|
||||
cands = []
|
||||
for r in rows:
|
||||
ev = r.evidence or {}
|
||||
text = session.scalar(select(NcmEntry.description).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == r.value).limit(1))
|
||||
cands.append({"ncm": r.value, "official_text": text, "status": r.status, "clients": sorted(ev.get("clients", [])), "documents": sum(int(l.get("docs", 0)) for l in (ev.get("lineages") or {}).values()),
|
||||
"providers": sorted(f for f in (ev.get("families") or {}) if f.startswith("PROVIDER:")), "attested": (ev.get("info") or {}).get("attested_by")})
|
||||
status = "VERIFIED_KB" if any(c["status"] == "VERIFIED_KB" for c in cands) else ("CONFLICT" if any(c["status"] == "CONFLICT" for c in cands) else (cands[0]["status"] if cands else "OBSERVED"))
|
||||
out.append({"gtin": (prod.gtin or "")[-13:], "gtin_key": prod.gtin, "description": (prod.normalized_description or "")[:70], "status": status, "candidates": [c for c in cands if c["status"] != "STALE" or c["attested"] is None][:4]})
|
||||
return out
|
||||
|
||||
|
||||
def kb_search(session, query: str, limit: int = 6) -> list[dict]:
|
||||
from ..models import KbProduct
|
||||
from . import kb as kbm
|
||||
q = (query or "").strip()
|
||||
if not q:
|
||||
return []
|
||||
digits = "".join(ch for ch in q if ch.isdigit())
|
||||
products = []
|
||||
if len(digits) >= 8 and len(digits) == len(q.replace(" ", "")):
|
||||
p = kbm.find_by_gtin(session, digits.zfill(14))
|
||||
products = [p] if p else []
|
||||
else:
|
||||
products = kbm.name_candidates(session, q)[:limit]
|
||||
return _candidate_cards(session, products)
|
||||
|
||||
|
||||
def kb_summary(session, query: str | None = None) -> dict:
|
||||
"""Compact Knowledge Base panel: known products, statuses of the NCM knowledge, hits today, and the latest conflicts with their evidence (no dashboard of its own)."""
|
||||
from ..models import KbKnowledge, KbProduct
|
||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
@@ -102,4 +134,4 @@ def kb_summary(session) -> dict:
|
||||
"candidates": [{"ncm": i.value, "clients": sorted({c for c in (i.evidence or {}).get("clients", [])}), "documents": sum(int(l.get("docs", 0)) for l in ((i.evidence or {}).get("lineages") or {}).values()),
|
||||
"providers": sorted(f for f in ((i.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:"))} for i in items]})
|
||||
return {"products": products, "verified": by.get("VERIFIED_KB", 0), "corroborated": by.get("CORROBORATED", 0) + by.get("EXTERNAL_CORROBORATED", 0), "observed": by.get("OBSERVED", 0),
|
||||
"conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts}
|
||||
"conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts, "search": kb_search(session, query) if query else []}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Human-facing status of every tracked product: nothing may silently 'do nothing'. Computed from the resolver result and refined when the Agent reports the write outcome."""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select, update
|
||||
|
||||
LABELS = {
|
||||
"ENRIQUECIDO": "Enriquecido: campos seguros preenchidos.",
|
||||
"PRONTO_PARA_PREENCHER": "Evidência suficiente: aguardando a escrita segura pelo Agent.",
|
||||
"SIMULADO": "Simulação (DRY_RUN): há candidato, nada foi gravado.",
|
||||
"AGUARDANDO_EVIDENCIA": "Aguardando evidência: nenhuma fonte confiável ainda.",
|
||||
"EAN_AUSENTE": "Sem EAN e sem produto único verificado na base: informe o código de barras ou peça a decisão de um administrador.",
|
||||
"EAN_INVALIDO": "Código de barras inválido: nenhuma fonte externa foi consultada.",
|
||||
"NOME_AMBIGUO": "Nome genérico ou ambíguo demais para identificar o produto com segurança.",
|
||||
"SEM_MATCH_DE_PRODUTO": "Nenhum produto conhecido corresponde a este nome/EAN.",
|
||||
"NCM_NAO_VERIFICADO": "Produto identificado, mas o NCM ainda não foi verificado (falta segunda fonte independente ou decisão de administrador).",
|
||||
"CONFLITO_DE_NCM": "As fontes divergem sobre o NCM: nada é preenchido até um administrador decidir.",
|
||||
"PROVIDER_SEM_QUOTA": "Limite diário das fontes externas atingido; será repetido após o reset.",
|
||||
"PROVIDER_NAO_CONFIGURADO": "Nenhuma fonte externa com NCM está configurada.",
|
||||
"AGUARDANDO_RETRY": "Fontes indisponíveis agora; nova tentativa automática agendada.",
|
||||
"HUMAN_OVERRIDE": "Alterado por uma pessoa: nunca mais preenchido automaticamente.",
|
||||
"TECHNICAL_PLACEHOLDER": "NCM provisório (00000000) reconhecido como 'ainda não classificado'.",
|
||||
"WRITE_FAILED": "A escrita segura foi recusada ou falhou (veja o motivo).",
|
||||
"VERIFICACAO_POS_WRITE_FALHOU": "A escrita foi feita mas a releitura não confirmou o valor.",
|
||||
"NADA_A_PREENCHER": "Todos os campos elegíveis já estão preenchidos.",
|
||||
"FUNCAO_DESATIVADA": "Enriquecimento desativado para esta instalação.",
|
||||
}
|
||||
|
||||
|
||||
def classify(result: dict, *, has_gtin: bool, specific_name: bool) -> tuple[str, str]:
|
||||
"""(ui_status, reason_code) from a resolve() result."""
|
||||
fields = result.get("fields") or {}
|
||||
ncm = fields.get("CLASSIFICACAO_FISCAL") or {}
|
||||
state = result.get("state")
|
||||
if state == "FEATURE_DISABLED":
|
||||
return "FUNCAO_DESATIVADA", state
|
||||
if state == "INVALID_GTIN":
|
||||
return "EAN_INVALIDO", state
|
||||
if state == "PENDING_PROVIDER_QUOTA":
|
||||
return "PROVIDER_SEM_QUOTA", state
|
||||
if state in ("PENDING_PROVIDERS_OFFLINE", "PENDING_TIME_BUDGET"):
|
||||
return "AGUARDANDO_RETRY", state
|
||||
if state == "CONFLICT" or ncm.get("status") == "CONFLICT":
|
||||
return "CONFLITO_DE_NCM", "CONFLICT"
|
||||
if ncm.get("status") == "HUMAN_OVERRIDDEN":
|
||||
return "HUMAN_OVERRIDE", "HUMAN_OVERRIDDEN"
|
||||
if any(f.get("status") == "AUTO_FILL_SAFE" for f in fields.values()):
|
||||
return "PRONTO_PARA_PREENCHER", state or "RESOLVED"
|
||||
if ncm.get("status") == "ALREADY_SET" and not any(f.get("status") in ("AUTO_FILL_SAFE", "SUGGESTION_ONLY") for f in fields.values()):
|
||||
return "NADA_A_PREENCHER", "ALL_FIELDS_SET"
|
||||
if not has_gtin:
|
||||
if state == "AMBIGUOUS" or not specific_name:
|
||||
return "NOME_AMBIGUO", "GENERIC_OR_AMBIGUOUS_NAME"
|
||||
return "EAN_AUSENTE", "NAME_ONLY_NO_VERIFIED_MATCH"
|
||||
if ncm.get("status") == "SUGGESTION_ONLY":
|
||||
return "NCM_NAO_VERIFICADO", ncm.get("rule") or "NCM_NOT_VERIFIED"
|
||||
if state == "NOT_FOUND":
|
||||
return "SEM_MATCH_DE_PRODUTO", "GTIN_UNKNOWN"
|
||||
return "AGUARDANDO_EVIDENCIA", state or "INSUFFICIENT_EVIDENCE"
|
||||
|
||||
|
||||
def after_write(outcomes: list[dict], mode: str) -> tuple[str | None, str | None]:
|
||||
"""Refines the status with what the Agent reported (APPLIED / DRY_RUN / ABORTED / FAILED / HUMAN_OVERRIDDEN)."""
|
||||
kinds = {o.get("outcome") for o in outcomes}
|
||||
reasons = [o.get("reason") for o in outcomes if o.get("reason")]
|
||||
if "HUMAN_OVERRIDDEN" in kinds and "APPLIED" not in kinds:
|
||||
return "HUMAN_OVERRIDE", "HUMAN_OVERRIDDEN"
|
||||
if "APPLIED" in kinds:
|
||||
return "ENRIQUECIDO", None
|
||||
if "FAILED" in kinds:
|
||||
return ("VERIFICACAO_POS_WRITE_FALHOU" if "POST_WRITE_VERIFICATION_FAILED" in reasons else "WRITE_FAILED"), (reasons[0] if reasons else "FAILED")
|
||||
if "ABORTED" in kinds:
|
||||
return "WRITE_FAILED", (reasons[0] if reasons else "ABORTED")
|
||||
if "DRY_RUN" in kinds:
|
||||
return "SIMULADO", "DRY_RUN_NO_WRITE"
|
||||
return None, None
|
||||
|
||||
|
||||
def mark_for_reevaluation(session, installation_id: int, *, codes: list[str] | None = None, gtin: str | None = None) -> int:
|
||||
"""Flags tracked, NOT-yet-enriched products for a new evaluation (KB/cache first; providers only when really needed). Idempotent; never touches enriched / human-overridden ones."""
|
||||
from ..models import ProductEnrichment
|
||||
q = update(ProductEnrichment).where(ProductEnrichment.installation_id == installation_id, ProductEnrichment.ui_status.notin_(["ENRIQUECIDO", "HUMAN_OVERRIDE"]) | ProductEnrichment.ui_status.is_(None))
|
||||
if codes:
|
||||
q = q.where(ProductEnrichment.product_code.in_(codes))
|
||||
if gtin:
|
||||
q = q.where(ProductEnrichment.gtin == gtin)
|
||||
return session.execute(q.values(reevaluate=True)).rowcount or 0
|
||||
@@ -30,6 +30,9 @@ def create_job(session, agent, kind, payload=None):
|
||||
(no duplicates, never two full jobs at once)."""
|
||||
if kind not in _VALID_KINDS:
|
||||
raise ValueError(f"unsupported job kind: {kind}")
|
||||
if kind in ("REPROCESS_ALL", "REPROCESS_PRODUCTS", "SYNC_INCREMENTAL"):
|
||||
_flag_enrichment_reevaluation(session, agent) # "Reprocessar"/"Sincronizar" also re-evaluate the tracked, not-yet-enriched products (KB/cache first), even when an equivalent job is already queued
|
||||
session.commit()
|
||||
if kind in SYNC_KINDS:
|
||||
# A full job subsumes an incremental one, not the other way round: asking for a
|
||||
# full reprocess while only an incremental is queued must still queue the full.
|
||||
@@ -43,6 +46,16 @@ def create_job(session, agent, kind, payload=None):
|
||||
return job
|
||||
|
||||
|
||||
def _flag_enrichment_reevaluation(session, agent):
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is not None and installation.features_configured and installation.enrichment_mode != "DISABLED":
|
||||
from .enrichment.status import mark_for_reevaluation
|
||||
mark_for_reevaluation(session, installation.id)
|
||||
except Exception: # noqa: BLE001 - never let the enrichment bookkeeping break job creation
|
||||
pass
|
||||
|
||||
|
||||
def recover_stale_running(session, agent, now=None):
|
||||
now = now or datetime.now(timezone.utc)
|
||||
cutoff = now - STALE_RUNNING_AFTER
|
||||
|
||||
@@ -816,6 +816,10 @@ class ProductEnrichment(Base):
|
||||
attempts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
||||
latency_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
reevaluate: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false", index=True) # set by panel / REPROCESS / SYNC / KB change; consumed by the Agent
|
||||
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
ui_status: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) # human-facing state (never a silent product)
|
||||
reason_code: Mapped[str | None] = mapped_column(String(48), nullable=True)
|
||||
detected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
|
||||
once published — never a second, independent version number. Fiscal semantics
|
||||
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
|
||||
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
|
||||
GTI_FISCAL_VERSION = "1.3.4"
|
||||
GTI_FISCAL_VERSION = "1.3.5"
|
||||
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
|
||||
|
||||
+60
-3
@@ -22,6 +22,7 @@ from .panel_presentation import format_ui_datetime
|
||||
from .models import AdminUser, Agent, AgentEnrollment, AgentRelease, AgentSnapshot, BlockedProduct, FiscalContextEntry, Installation, Job, ProductEnrichment, ProductFieldAudit, RtcCandidate, RtcWriteAudit, Tenant
|
||||
from .jobs import create_job
|
||||
from .audit import record_audit
|
||||
from .enrichment import kb as kb_mod, status as enrichment_status
|
||||
from .enrichment.observability import coverage_metrics, kb_summary, official_sources, provider_sources
|
||||
|
||||
|
||||
@@ -158,10 +159,10 @@ def router(session_factory):
|
||||
return {key: items[0] for key, items in parse_qs((await request.body()).decode()).items() if items}
|
||||
|
||||
@routes.get("/", response_class=HTMLResponse)
|
||||
def dashboard(request: Request):
|
||||
def dashboard(request: Request, kb_q: str | None = None):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session), "kb": kb_summary(session)})
|
||||
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "kb_q": kb_q or "", "is_admin": _is_privileged(auth_data[1]), "csrf_token": request.cookies.get("gti_csrf", ""), "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session), "kb": kb_summary(session, kb_q)})
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents", response_class=HTMLResponse)
|
||||
@@ -302,10 +303,18 @@ def router(session_factory):
|
||||
row = session.scalar(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.product_code == enrich_code.strip()[:64]))
|
||||
audits = session.scalars(select(ProductFieldAudit).where(ProductFieldAudit.installation_id == installation.id, ProductFieldAudit.product_code == enrich_code.strip()[:64]).order_by(ProductFieldAudit.id.desc()).limit(20)).all() if row else []
|
||||
lookup = {"found": row is not None, "row": row, "audits": audits}
|
||||
tracked = []
|
||||
for r in session.scalars(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id).order_by(ProductEnrichment.id.desc()).limit(15)):
|
||||
fields = ((r.decisions or {}).get("fields") or {})
|
||||
ncm = fields.get("CLASSIFICACAO_FISCAL") or {}
|
||||
tracked.append({"code": r.product_code, "description": r.description, "gtin": r.gtin, "ui_status": r.ui_status or "AGUARDANDO_EVIDENCIA", "label": enrichment_status.LABELS.get(r.ui_status or "AGUARDANDO_EVIDENCIA", ""),
|
||||
"reason": r.reason_code, "detected": r.detected_at, "last_attempt": r.last_attempt_at, "next_retry": r.next_attempt_at, "kb_status": ncm.get("kb_status"), "ncm_candidate": ncm.get("value"),
|
||||
"cest_candidate": (fields.get("CEST") or {}).get("value"), "ipi_candidate": (fields.get("IPI") or {}).get("value"), "applied": r.fields_applied or {}, "placeholder": ncm.get("technical_placeholder"),
|
||||
"reevaluate": bool(r.reevaluate), "mode": installation.enrichment_mode})
|
||||
mode = installation.enrichment_mode
|
||||
waiting = not installation.features_configured
|
||||
return {"configured": bool(installation.features_configured), "waiting": waiting, "corrector_enabled": bool(installation.corrector_enabled), "enrichment_mode": mode,
|
||||
"mode_label": ENRICHMENT_MODE_LABELS.get(mode, mode), "counts": counts, "total": sum(counts.values()), "applied": applied, "lookup": lookup,
|
||||
"mode_label": ENRICHMENT_MODE_LABELS.get(mode, mode), "counts": counts, "total": sum(counts.values()), "applied": applied, "lookup": lookup, "tracked": tracked,
|
||||
"state_label": "Aguardando configuração de funcionalidades" if waiting else ENRICHMENT_MODE_LABELS.get(mode, mode)}
|
||||
|
||||
@routes.post("/agents/{agent_id}/features")
|
||||
@@ -335,6 +344,54 @@ def router(session_factory):
|
||||
return RedirectResponse(f"/agents/{agent_id}", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/{agent_id}/enrichment/reevaluate")
|
||||
async def reevaluate_enrichment(request: Request, agent_id: int):
|
||||
"""Explicit, idempotent 'evaluate again' for one product (code=...) or every tracked product that is not enriched yet. KB/cache first; providers only when really needed."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
agent, installation = _rtc_target(session, agent_id)
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
if not installation.features_configured or installation.enrichment_mode == "DISABLED":
|
||||
return HTMLResponse("O enriquecimento está desativado nesta instalação (Funcionalidades).", status_code=409)
|
||||
code = (values.get("code") or "").strip()[:64]
|
||||
n = enrichment_status.mark_for_reevaluation(session, installation.id, codes=[code] if code else None)
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="ENRICHMENT_REEVALUATE", target_type="installation", target_id=installation.installation_id,
|
||||
context={"product": code or "ALL_UNRESOLVED", "flagged": n, "operator": admin.display_name or admin.username})
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}#enriquecimento", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/kb/attest")
|
||||
async def kb_attest(request: Request):
|
||||
"""A human fiscal decision (ADMIN only): choose which candidate NCM is right for a GTIN. Only an existing, vigente candidate can be chosen; the others are recorded as rejected."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
if not _is_privileged(admin):
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=None, action="KB_ATTEST_DENIED", target_type="kb_product", target_id=values.get("gtin") or "", context={"role": admin.external_role or admin.auth_source}, severity="WARNING")
|
||||
session.commit()
|
||||
return HTMLResponse("Ação restrita a administradores do GTI Gerenciador (perfil ADMIN).", status_code=403)
|
||||
gtin, ncm = (values.get("gtin") or "").strip(), (values.get("ncm") or "").strip()
|
||||
from .enrichment import history
|
||||
current = history.official_sets(session)
|
||||
try:
|
||||
result = kb_mod.attest(session, gtin, ncm, admin.display_name or admin.username, lambda v: v in current, note=(values.get("note") or "")[:200])
|
||||
except ValueError as error:
|
||||
return HTMLResponse(f"Decisão recusada: {error}", status_code=422)
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=None, action="KB_NCM_ATTESTED", target_type="kb_product", target_id=gtin,
|
||||
context={"ncm": ncm, "status": result["status"], "families": result["families"], "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source}, severity="WARNING")
|
||||
for inst in session.scalars(select(Installation).where(Installation.enrichment_mode != "DISABLED", Installation.features_configured.is_(True))):
|
||||
enrichment_status.mark_for_reevaluation(session, inst.id, gtin=gtin)
|
||||
enrichment_status.mark_for_reevaluation(session, inst.id) # products identified by name are found through the KB aliases
|
||||
session.commit()
|
||||
return RedirectResponse("/#kb-resumo", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
def _ops_view(session, installation, agent):
|
||||
"""Processing status shown on the Agent detail (no clutter: 5 facts)."""
|
||||
last_job = session.scalar(select(Job).where(Job.agent_id == agent.id).order_by(Job.id.desc()).limit(1))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "gti-fiscal-hub"
|
||||
version = "1.3.4"
|
||||
version = "1.3.5"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"fastapi>=0.115,<1",
|
||||
|
||||
@@ -9,14 +9,22 @@
|
||||
<a class="botao secondary" href="/agents/{{ detail.agent_id }}/reprocess-confirmation" title="Recalcula todos os produtos (pede confirmação)">Reprocessar tudo</a>
|
||||
{% if ops.enabled %}<form method="post" action="/agents/{{ detail.agent_id }}/pause" style="display:inline"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><button type="submit" class="caution" title="Mantém o heartbeat, mas não executa jobs">Pausar processamento</button></form>{% else %}<form method="post" action="/agents/{{ detail.agent_id }}/resume" style="display:inline"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><button type="submit" class="primary-action" title="Volta a executar jobs">Retomar processamento</button></form>{% endif %}
|
||||
</div></div><dl class="facts"><dt>Processamento</dt><dd><span class="badge {{ '' if ops.enabled else 'critical' }}">{{ "Ativo" if ops.enabled else "Pausado" }}</span> · sincronização automática a cada {{ ops.interval_minutes }} min</dd><dt>Última sincronização</dt><dd>{{ ops.last_sync or "ainda não sincronizado" }}{% if ops.last_changed is not none %} · {{ ops.last_changed }} produto(s) alterado(s){% endif %}</dd><dt>Sincronização agendada</dt><dd>{{ ops.next_sync or ("aguardando o primeiro sync" if ops.enabled else "pausado") }}</dd>{% if ops.rtc %}<dt>Reforma Tributária</dt><dd><span class="badge {{ "" if ops.rtc.enabled else "muted" }}" title="{{ ops.rtc.title }}">{{ ops.rtc.label }}</span></dd>{% endif %}{% if ops.rtc and ops.rtc.enabled %}{% if ops.rtc.products and not ops.rtc.products.applicable %}<dt>cClassTrib</dt><dd>{{ ops.rtc.products.label }}</dd>{% elif ops.rtc.products %}<dt>Produtos RTC</dt><dd><b>{{ ops.rtc.products.headline }}</b><br><small>Total {{ ops.rtc.products.total }} · com cClassTrib válido {{ ops.rtc.products.valid }} · prontos para preenchimento {{ ops.rtc.products.ready }} · ambíguos/bloqueados {{ ops.rtc.products.blocked }}</small></dd>{% endif %}
|
||||
<dt>Preenchimento automático</dt><dd>{{ ops.rtc.apply.state }}{% if ops.rtc.apply.applied_last_job %} · {{ ops.rtc.apply.applied_last_job }} preenchido(s) no último job{% endif %} · <a href="/agents/{{ detail.agent_id }}/rtc">Controle e candidatos cClassTrib</a></dd>{% endif %}<dt>Último job</dt><dd>{% if ops.last_job %}<a href="/jobs?scope=all&job_id={{ ops.last_job.id }}">{{ ops.last_job.kind_label }}</a> · {{ ops.last_job.status_label }}{% else %}nenhum{% endif %}</dd></dl></section>
|
||||
<section class="panel" id="funcionalidades"><div class="panel-title"><div><h2>Funcionalidades</h2><p class="muted">Decididas no Hub para esta instalação — o Agent não tem chave local.</p></div></div>
|
||||
<dt>Preenchimento cClassTrib (RTC) — não é o Enriquecimento</dt><dd>{{ ops.rtc.apply.state }}{% if ops.rtc.apply.applied_last_job %} · {{ ops.rtc.apply.applied_last_job }} preenchido(s) no último job{% endif %} · <a href="/agents/{{ detail.agent_id }}/rtc">Controle e candidatos cClassTrib</a></dd>{% endif %}<dt>Último job</dt><dd>{% if ops.last_job %}<a href="/jobs?scope=all&job_id={{ ops.last_job.id }}">{{ ops.last_job.kind_label }}</a> · {{ ops.last_job.status_label }}{% else %}nenhum{% endif %}</dd></dl></section>
|
||||
<section class="panel" id="funcionalidades"><div class="panel-title"><div><h2>Funcionalidades (Enriquecimento de produtos novos)</h2><p class="muted">Decididas no Hub para esta instalação — o Agent não tem chave local. O Enriquecimento de produtos novos NÃO é o Preenchimento cClassTrib (RTC), que tem controle próprio em <a href="/agents/{{ detail.agent_id }}/rtc">Controle e candidatos cClassTrib</a>.</p></div></div>
|
||||
{% if features.waiting %}<p class="coverage-status error">Aguardando configuração de funcionalidades: esta instalação ainda não processa nada.</p>{% endif %}
|
||||
<dl class="facts"><dt>Corretor fiscal</dt><dd><span class="badge {{ '' if features.corrector_enabled else 'critical' }}">{{ "Ativo" if features.corrector_enabled else "Desativado" }}</span></dd><dt>Enriquecimento de produtos novos</dt><dd><span class="badge {{ '' if features.enrichment_mode != 'DISABLED' else 'critical' }}">{{ features.state_label }}</span></dd></dl>
|
||||
{% if is_admin %}<form method="post" action="/agents/{{ detail.agent_id }}/features" class="filters compact-filters"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><label><input type="checkbox" name="corrector_enabled" value="1" {% if features.corrector_enabled or features.waiting %}checked{% endif %}> Corretor fiscal</label><label>Enriquecimento<select name="enrichment_mode">{% for value, label in [("DISABLED","Desativado"),("DRY_RUN","Simulação (não grava)"),("ENABLED","Automático")] %}<option value="{{ value }}" {% if features.enrichment_mode==value %}selected{% endif %}>{{ label }}</option>{% endfor %}</select></label><label>Para o modo automático digite PREENCHER<input name="confirm" autocomplete="off"></label><button class="secondary" type="submit">Salvar funcionalidades</button></form>{% else %}<p class="muted">Somente administradores alteram as funcionalidades.</p>{% endif %}</section>
|
||||
<section class="panel" id="enriquecimento"><div class="panel-title"><div><h2>Enriquecimento</h2><p class="muted">Produtos novos cadastrados no SGV; o catálogo histórico nunca é enviado a fontes externas.</p></div></div>
|
||||
<dl class="facts"><dt>Produtos acompanhados</dt><dd>{{ features.total }}</dd><dt>Campos preenchidos</dt><dd>{{ features.applied }}</dd>{% for status, n in features.counts.items() %}<dt>{{ status }}</dt><dd>{{ n }}</dd>{% endfor %}</dl>
|
||||
<form method="get" class="filters compact-filters"><label>Consultar produto<input name="enrich_code" value="{{ enrich_code }}" placeholder="código do produto"></label><button class="secondary" type="submit">Ver origem dos campos</button></form>
|
||||
<div class="enrich-list" style="margin-top:12px">{% if features.tracked %}<table class="data-table"><thead><tr><th>Produto</th><th>Estado</th><th>Motivo / candidatos</th><th>Última tentativa</th><th></th></tr></thead><tbody>
|
||||
{% for r in features.tracked %}<tr><td><b>{{ r.code }}</b> {{ r.description }}<br><small class="muted">EAN: {{ r.gtin or "não informado" }} · detectado {{ r.detected.strftime("%d/%m %H:%M") if r.detected else "—" }}</small></td>
|
||||
<td><em class="badge {{ '' if r.ui_status in ['ENRIQUECIDO','PRONTO_PARA_PREENCHER','SIMULADO','NADA_A_PREENCHER'] else 'critical' }}">{{ r.ui_status }}</em>{% if r.placeholder %}<br><small class="muted">NCM provisório {{ r.placeholder }}</small>{% endif %}</td>
|
||||
<td><small>{{ r.label }}{% if r.kb_status %} · base: {{ r.kb_status }}{% endif %}{% if r.ncm_candidate %} · NCM candidato {{ r.ncm_candidate }}{% endif %}{% if r.cest_candidate %} · CEST {{ r.cest_candidate }}{% endif %}{% if r.ipi_candidate %} · IPI {{ r.ipi_candidate }}{% endif %}{% if r.applied %} · gravado: {{ r.applied.keys()|join(", ") }}{% endif %}</small></td>
|
||||
<td><small>{{ r.last_attempt.strftime("%d/%m %H:%M:%S") if r.last_attempt else "—" }}{% if r.next_retry %}<br>próximo retry {{ r.next_retry.strftime("%d/%m %H:%M") }}{% endif %}{% if r.reevaluate %}<br>reavaliação pedida{% endif %}</small></td>
|
||||
<td>{% if features.enrichment_mode != "DISABLED" and r.ui_status not in ["ENRIQUECIDO","HUMAN_OVERRIDE"] %}<form method="post" action="/agents/{{ detail.agent_id }}/enrichment/reevaluate"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><input type="hidden" name="code" value="{{ r.code }}"><button class="secondary" type="submit">Reavaliar</button></form>{% endif %}</td></tr>{% endfor %}</tbody></table>
|
||||
{% if features.enrichment_mode != "DISABLED" %}<form method="post" action="/agents/{{ detail.agent_id }}/enrichment/reevaluate"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><button class="secondary" type="submit">Reavaliar todos os pendentes</button> <small class="muted">Só reavalia (base/cache primeiro); preço editado não é produto novo.</small></form>{% endif %}
|
||||
{% else %}<p class="empty-state">Nenhum produto novo acompanhado ainda. Só entram produtos cadastrados depois da ativação.</p>{% endif %}</div>
|
||||
{% if features.lookup %}{% if features.lookup.found %}<dl class="facts"><dt>Produto</dt><dd>{{ features.lookup.row.product_code }} · {{ features.lookup.row.description }}</dd><dt>GTIN</dt><dd>{{ features.lookup.row.gtin or "sem GTIN válido" }}</dd><dt>Estado</dt><dd>{{ features.lookup.row.status }}{% if features.lookup.row.decisions.state %} · {{ features.lookup.row.decisions.state }}{% endif %}</dd></dl>{% for a in features.lookup.audits %}<div class="list-row"><span><b>{{ a.field }}</b>: {{ a.old_value or "vazio" }} → {{ a.new_value or "—" }}<small>{{ a.outcome }} · regra {{ a.rule or "—" }} · base {{ a.kb_status or "—" }}{% if a.evidence %} · {{ a.evidence|join(", ") }}{% endif %}</small></span></div>{% endfor %}{% else %}<p class="empty-state">Produto sem registro de enriquecimento.</p>{% endif %}{% endif %}</section>
|
||||
<nav class="tabs" aria-label="Seções da instalação">{% for name,label in [("overview","Resumo"),("fiscal","Fiscal"),("blockers","Blockers"),("jobs","Jobs"),("events","Eventos")] %}<a class="{% if tab==name %}active{% endif %}" href="?tab={{ name }}">{{ label }}</a>{% endfor %}</nav>
|
||||
<section class="split"><article class="panel"><div class="panel-title"><h2>Contexto da empresa</h2></div><dl class="facts"><dt>Cliente</dt><dd>{{ detail.client or "Não informado" }}</dd><dt>Nome de exibição</dt><dd>{{ detail.display_name or "Não informado" }}</dd><dt>Hostname</dt><dd>{{ detail.hostname or "Não informado" }}</dd><dt>Empresa</dt><dd>{{ detail.company_name or detail.company_code or "Não informado" }}</dd><dt>UF</dt><dd>{{ detail.company_uf or "Não informado" }}</dd><dt>CRT</dt><dd>{{ detail.company_crt or "Não informado" }}</dd><dt>Versão do agent</dt><dd>{{ detail.agent_version_display }}</dd><dt>Último heartbeat</dt><dd title="{{ detail.last_seen_at }}">{{ detail.last_seen_display }} · {{ detail.last_seen_age }}</dd><dt>Firebird</dt><dd>{{ detail.firebird_version or "Não informado" }} · ODS {{ detail.firebird_ods or "Não informado" }}</dd><dt>IDs técnicos</dt><dd>Agent {{ detail.technical_ids.agent_id }} · instalação {{ detail.technical_ids.installation_db_id }}</dd></dl></article>
|
||||
|
||||
@@ -30,6 +30,10 @@
|
||||
</section>
|
||||
<section class="panel" id="kb-resumo"><div class="panel-title"><div><h2>Base de conhecimento de produtos</h2><p class="muted">Evidência local, histórica e externa; nada vira verdade fiscal sem regra de promoção.</p></div></div>
|
||||
<div class="coverage"><div class="coverage-item"><strong>{{ kb.products }}</strong><span>Produtos conhecidos</span></div><div class="coverage-item"><strong>{{ kb.verified }}</strong><span>GTINs verificados</span></div><div class="coverage-item"><strong>{{ kb.corroborated }}</strong><span>Corroborados</span></div><div class="coverage-item"><strong>{{ kb.conflicts }}</strong><span>Conflitos</span></div><div class="coverage-item"><strong>{{ kb.hits_today }}</strong><span>Resolvidos pela base hoje</span></div></div>
|
||||
<form method="get" action="/" class="filters compact-filters"><label>Buscar produto na base (EAN ou nome)<input name="kb_q" value="{{ kb_q or '' }}" placeholder="ex.: COCA COLA LATA 350"></label><button class="secondary" type="submit">Buscar</button></form>
|
||||
{% for c in kb.search %}<div class="list-row"><span><b>{{ c.gtin }}</b> {{ c.description }} · <em class="badge">{{ c.status }}</em>
|
||||
<small>{% for x in c.candidates %}NCM {{ x.ncm }} — <i>{{ x.official_text or "fora do dataset oficial vigente" }}</i> ({{ x.clients|length }} cliente(s), {{ x.documents }} doc.{% if x.providers %}, {{ x.providers|join("/") }}{% endif %}{% if x.attested %}, decidido por {{ x.attested }}{% endif %}){% if is_admin and x.official_text and c.status != "VERIFIED_KB" %}
|
||||
<form method="post" action="/kb/attest" style="display:inline"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><input type="hidden" name="gtin" value="{{ c.gtin_key }}"><input type="hidden" name="ncm" value="{{ x.ncm }}"><button class="secondary" type="submit" title="Decisão fiscal de um administrador: este é o NCM correto deste EAN">Confirmar este NCM</button></form>{% endif %}<br>{% endfor %}</small></span></div>{% endfor %}
|
||||
{% for c in kb.recent_conflicts %}<div class="list-row"><span><b>{{ c.gtin }}</b> {{ c.description }}<small>NCMs divergentes, nada é preenchido: {% for x in c.candidates %}{{ x.ncm }} ({{ x.clients|length }} cliente(s), {{ x.documents }} doc.{% if x.providers %}, {{ x.providers|join("/") }}{% endif %}){% if not loop.last %} × {% endif %}{% endfor %}</small></span></div>{% endfor %}</section>
|
||||
<section class="panel" id="enriquecimento-resumo"><div class="panel-title"><div><h2>Enriquecimento de produtos novos</h2><p class="muted">Cobertura medida sobre os produtos novos acompanhados (o catálogo histórico não entra).</p></div></div>
|
||||
{% if enrich_metrics.tracked %}<div class="coverage">{% for key,label in [("GTIN_VALID_RATE","Com GTIN válido"),("KB_HIT_RATE","Resolvidos pela base"),("PRODUCT_FOUND_RATE","Produto identificado"),("NCM_AUTO_FILL_SAFE_RATE","NCM preenchível com segurança"),("AMBIGUOUS_RATE","Ambíguos"),("CONFLICT_RATE","Em conflito")] %}<div class="coverage-item"><strong>{{ enrich_metrics[key] if enrich_metrics[key] is not none else "—" }}{% if enrich_metrics[key] is not none %}%{% endif %}</strong><span>{{ label }}</span></div>{% endfor %}</div>{% else %}<p class="empty-state">Nenhum produto novo acompanhado ainda.</p>{% endif %}</section>
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Visible status, re-evaluation (panel / Reprocessar / Sincronizar / KB decision), operator attestation, deterministic name-only match, RTC != Enrichment."""
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from central.app.enrichment import api as enrich_api, history, kb, status
|
||||
from central.app.enrichment.quota import ProviderGateway
|
||||
from central.app.enrichment.resolver import resolve
|
||||
from central.app.jobs import create_job
|
||||
from central.app.models import Agent, AuditEvent, Installation, KbKnowledge, KbProduct, KbPromotionAudit, ProductEnrichment
|
||||
from central.tests.test_control_panel_pages import _panel_client
|
||||
from central.tests.test_enrichment_history import G, run as seed_history, staging
|
||||
from central.tests.test_product_enrichment_api import H, _second_agent, resolve as api_resolve, setup
|
||||
from central.tests.test_product_enrichment_resolver import COSMOS, Fake, G1, env, gtin # noqa: F401
|
||||
from central.tests.test_rtc_security import _become, _post
|
||||
|
||||
|
||||
def _kb_conflict(env):
|
||||
seed_history(env, [staging("RINCAO", G1, "19059090", years={2024: 900, 2025: 900}, desc="COCA LATA 350 ML"), staging("UNIAO", G1, "22021000", years={2024: 20, 2025: 13}, desc="REFRIGERANTE COCA COLA LATA 350 ML")])
|
||||
|
||||
|
||||
def test_attestation_resolves_a_conflict_with_audit_and_only_among_candidates(env):
|
||||
_kb_conflict(env)
|
||||
cur = history.official_sets(env)
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: v in cur)["status"] == kb.CONFLICT
|
||||
for bad, why in (("21069090", "NCM_IS_NOT_A_CANDIDATE_OF_THIS_GTIN"), ("22030000x", "NCM_IS_NOT_A_CANDIDATE_OF_THIS_GTIN")):
|
||||
try:
|
||||
kb.attest(env, G, bad, "Rafael", lambda v: v in cur)
|
||||
assert False
|
||||
except ValueError as error:
|
||||
assert str(error) == why
|
||||
res = kb.attest(env, G, "22021000", "Rafael", lambda v: v in cur, note="lata 350ml = refrigerante")
|
||||
assert res["status"] == kb.VERIFIED_KB and res["value"] == "22021000" and set(res["families"]) == {"OPERATOR_ATTESTATION", "CLIENT_DATA"}
|
||||
audit = env.scalars(select(KbPromotionAudit)).one()
|
||||
assert audit.sources["operator"] == "Rafael" and audit.new_status == "VERIFIED_KB"
|
||||
rejected = env.scalar(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.value == "19059090"))
|
||||
assert rejected.status == kb.STALE and rejected.evidence["info"]["historical_state"] == "REJECTED_BY_OPERATOR"
|
||||
|
||||
|
||||
def test_name_only_match_is_deterministic_and_conservative(env):
|
||||
_kb_conflict(env)
|
||||
cur = history.official_sets(env)
|
||||
kb.attest(env, G, "22021000", "Rafael", lambda v: v in cur)
|
||||
zero = gtin("789490070001")
|
||||
seed_history(env, [staging("UNIAO", zero, "22021000", years={2024: 20, 2025: 13}, desc="REFRIGERANTE COCA COLA ZERO LATA 350 ML", code="9")])
|
||||
env.commit()
|
||||
r = resolve(env, ProviderGateway(env, []), gtin_raw="", description="COCA-COLA ORIGINAL LATA 350 ML", current={"CLASSIFICACAO_FISCAL": "00000000"})
|
||||
assert r["state"] == "EXACT_KB_NAME_VERIFIED" and r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "AUTO_FILL_SAFE" and r["fields"]["CLASSIFICACAO_FISCAL"]["value"] == "22021000"
|
||||
ambiguous = resolve(env, ProviderGateway(env, []), gtin_raw="", description="COCA COLA ZERO LATA 350 ML", current={})
|
||||
assert ambiguous["fields"]["CLASSIFICACAO_FISCAL"]["status"] != "AUTO_FILL_SAFE" # ZERO is not a neutral extra word: no cross-match
|
||||
generic = resolve(env, ProviderGateway(env, []), gtin_raw="", description="REFRIGERANTE", current={})
|
||||
assert generic["state"] == "AMBIGUOUS"
|
||||
assert not kb.alias_match(frozenset({"COCA", "COLA", "LATA", "350", "ML"}), frozenset({"COCA", "COLA", "ZERO", "LATA", "350", "ML"}))
|
||||
assert kb.alias_match(frozenset({"COCA", "COLA", "LATA", "350", "ML"}), frozenset({"REFRIGERANTE", "COCA", "COLA", "LATA", "350", "ML"}))
|
||||
|
||||
|
||||
def test_every_tracked_product_has_a_visible_status(monkeypatch):
|
||||
client, engine = setup(monkeypatch)
|
||||
out = api_resolve(client, "N1", "", "COCA-COLA ORIGINAL LATA 350 ML", {"CLASSIFICACAO_FISCAL": "00000000"})
|
||||
with Session(engine) as s:
|
||||
row = s.scalars(select(ProductEnrichment)).one()
|
||||
assert row.ui_status == "EAN_AUSENTE" and row.reason_code == "NAME_ONLY_NO_VERIFIED_MATCH" and row.last_attempt_at is not None
|
||||
assert out["fields"]["CLASSIFICACAO_FISCAL"]["technical_placeholder"] == "00000000"
|
||||
generic = api_resolve(client, "N2", "", "REFRIGERANTE", {})
|
||||
invalid = api_resolve(client, "N3", "7891000100104", "PRODUTO X", {})
|
||||
with Session(engine) as s:
|
||||
st = {r.product_code: r.ui_status for r in s.scalars(select(ProductEnrichment))}
|
||||
assert st["N2"] == "NOME_AMBIGUO" and st["N3"] == "EAN_INVALIDO"
|
||||
assert all(status.LABELS.get(v) for v in st.values())
|
||||
from central.tests.test_product_enrichment_resolver import G1 as g
|
||||
done = api_resolve(client, "N4", g, "REFRIGERANTE COLA LATA 350ML", {})
|
||||
with Session(engine) as s:
|
||||
assert s.scalars(select(ProductEnrichment).where(ProductEnrichment.product_code == "N4")).one().ui_status == "PRONTO_PARA_PREENCHER"
|
||||
client.post("/v1/products/enrichment/applied", json={"product_code": "N4", "fields": [{"field": "CLASSIFICACAO_FISCAL", "outcome": "APPLIED", "old": "00000000", "new": "22021000"}]}, headers=H)
|
||||
client.post("/v1/products/enrichment/applied", json={"product_code": "N1", "fields": [{"field": "CLASSIFICACAO_FISCAL", "outcome": "ABORTED", "reason": "STALE_CONFLICT"}]}, headers=H)
|
||||
with Session(engine) as s:
|
||||
st = {r.product_code: (r.ui_status, r.reason_code) for r in s.scalars(select(ProductEnrichment))}
|
||||
assert st["N4"][0] == "ENRIQUECIDO" and st["N1"] == ("WRITE_FAILED", "STALE_CONFLICT")
|
||||
|
||||
|
||||
def test_reprocess_sync_and_panel_flag_only_unresolved_products_for_reevaluation(monkeypatch):
|
||||
client, engine = setup(monkeypatch)
|
||||
api_resolve(client, "U1", "", "COCA-COLA ORIGINAL LATA 350 ML", {"CLASSIFICACAO_FISCAL": "00000000"})
|
||||
api_resolve(client, "E1", G1[:0] or gtin("789100010010"), "REFRIGERANTE COLA LATA 350ML", {})
|
||||
client.post("/v1/products/enrichment/applied", json={"product_code": "E1", "fields": [{"field": "CLASSIFICACAO_FISCAL", "outcome": "APPLIED", "old": None, "new": "22021000"}]}, headers=H)
|
||||
assert client.get("/v1/products/enrichment/pending", headers=H).json() == {"codes": []}
|
||||
with Session(engine) as s:
|
||||
agent = s.scalars(select(Agent)).first()
|
||||
for kind in ("REPROCESS_ALL", "SYNC_INCREMENTAL"):
|
||||
create_job(s, agent, kind, {})
|
||||
assert s.scalars(select(ProductEnrichment).where(ProductEnrichment.reevaluate.is_(True))).all()[0].product_code == "U1"
|
||||
for r in s.scalars(select(ProductEnrichment)):
|
||||
r.reevaluate = False
|
||||
s.commit()
|
||||
for j in s.scalars(select(__import__("central.app.models", fromlist=["Job"]).Job)):
|
||||
j.status = "COMPLETED"
|
||||
s.commit()
|
||||
with Session(engine) as s:
|
||||
create_job(s, s.scalars(select(Agent)).first(), "REPROCESS_PRODUCTS", {"codigos": ["U1"]})
|
||||
assert client.get("/v1/products/enrichment/pending", headers=H).json() == {"codes": ["U1"]} # the enriched product E1 is never re-evaluated
|
||||
api_resolve(client, "U1", "", "COCA-COLA ORIGINAL LATA 350 ML", {"CLASSIFICACAO_FISCAL": "00000000"})
|
||||
assert client.get("/v1/products/enrichment/pending", headers=H).json() == {"codes": []} # consumed by the new evaluation
|
||||
|
||||
|
||||
def test_panel_reevaluate_and_attest_routes_are_csrf_admin_and_audited(monkeypatch):
|
||||
client, agent_id, _ = _panel_client()
|
||||
with client._session_factory() as s:
|
||||
seed_history(s, [staging("RINCAO", G1, "19059090", years={2024: 900, 2025: 900}, desc="COCA LATA 350 ML"), staging("UNIAO", G1, "22021000", years={2024: 20, 2025: 13}, desc="REFRIGERANTE COCA COLA LATA 350 ML")])
|
||||
from central.tests.test_product_enrichment_resolver import seed
|
||||
seed(s)
|
||||
i = s.scalars(select(Installation)).one()
|
||||
i.enrichment_mode, i.features_configured = "ENABLED", True
|
||||
s.commit()
|
||||
_become(client, "PADRAO")
|
||||
assert _post(client, "/kb/attest", gtin="0" + G1, ncm="22021000").status_code == 403
|
||||
_become(client, "ADMIN")
|
||||
assert client.post("/kb/attest", data={"csrf_token": "bad", "gtin": "0" + G1, "ncm": "22021000"}, follow_redirects=False).status_code in (400, 403)
|
||||
assert _post(client, "/kb/attest", gtin="0" + G1, ncm="21069090").status_code == 422
|
||||
assert _post(client, "/kb/attest", gtin="0" + G1, ncm="22021000").status_code == 303
|
||||
assert _post(client, f"/agents/{agent_id}/enrichment/reevaluate", code="").status_code == 303
|
||||
with client._session_factory() as s:
|
||||
acts = [e.action for e in s.scalars(select(AuditEvent))]
|
||||
assert "KB_NCM_ATTESTED" in acts and "KB_ATTEST_DENIED" in acts and "ENRICHMENT_REEVALUATE" in acts
|
||||
page = client.get("/?kb_q=COCA COLA LATA 350 ML").text
|
||||
assert "VERIFIED_KB" in page and "decidido por" in page
|
||||
|
||||
|
||||
def test_rtc_mode_and_enrichment_mode_are_independent_switches_with_distinct_labels():
|
||||
client, agent_id, _ = _panel_client()
|
||||
_become(client, "ADMIN")
|
||||
assert _post(client, f"/agents/{agent_id}/features", enrichment_mode="DRY_RUN", corrector_enabled="1").status_code == 303
|
||||
with client._session_factory() as s:
|
||||
i = s.scalars(select(Installation)).one()
|
||||
assert i.enrichment_mode == "DRY_RUN" and i.rtc_write_mode == "DISABLED"
|
||||
assert _post(client, f"/agents/{agent_id}/rtc-mode", mode="DRY_RUN").status_code == 303
|
||||
assert _post(client, f"/agents/{agent_id}/rtc-mode", mode="ENABLED", confirm="PREENCHER").status_code == 303
|
||||
with client._session_factory() as s:
|
||||
i = s.scalars(select(Installation)).one()
|
||||
assert i.rtc_write_mode == "ENABLED" and i.enrichment_mode == "DRY_RUN" # turning cClassTrib to Automatic never touches the enrichment
|
||||
page = client.get(f"/agents/{agent_id}").text
|
||||
assert "Preenchimento cClassTrib (RTC)" in page and "Enriquecimento de produtos novos" in page
|
||||
@@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down():
|
||||
|
||||
def test_version_endpoint_reports_release_and_fiscal_versions():
|
||||
body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
|
||||
assert body["version"] == "1.3.4" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
||||
assert body["version"] == "1.3.5" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
||||
|
||||
|
||||
def test_production_refuses_the_dev_token(monkeypatch):
|
||||
|
||||
Reference in New Issue
Block a user