feat: coverage V2 - historical evidence miner, KB evidence model + promotion audit + revalidation, Kodebar/OSCBR limits, KB/provider panel, seed import; 1.3.2 (Hub only)
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
Claude Sonnet 5
@@ -0,0 +1,48 @@
|
||||
"""KB evidence model: wider per-field status (OBSERVED/CORROBORATED/EXTERNAL_CORROBORATED/...), promotion audit, provider counters for the panel.
|
||||
|
||||
Revision ID: 0030_kb_evidence_model
|
||||
Revises: 0029_product_enrichment
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0030_kb_evidence_model"
|
||||
down_revision = "0029_product_enrichment"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.alter_column("kb_knowledge", "status", existing_type=sa.String(16), type_=sa.String(24), existing_nullable=False, existing_server_default="EVIDENCE_ONLY")
|
||||
op.execute("UPDATE kb_knowledge SET status='OBSERVED' WHERE status='EVIDENCE_ONLY'")
|
||||
op.alter_column("kb_knowledge", "status", server_default="OBSERVED")
|
||||
op.create_table(
|
||||
"kb_promotion_audits",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("kb_product_id", sa.Integer(), sa.ForeignKey("kb_products.id"), nullable=False),
|
||||
sa.Column("gtin", sa.String(16), nullable=True),
|
||||
sa.Column("field", sa.String(24), nullable=False, server_default="ncm"),
|
||||
sa.Column("value", sa.String(64), nullable=True),
|
||||
sa.Column("old_status", sa.String(24), nullable=True),
|
||||
sa.Column("new_status", sa.String(24), nullable=False),
|
||||
sa.Column("families", sa.JSON(), nullable=False),
|
||||
sa.Column("sources", sa.JSON(), nullable=False),
|
||||
sa.Column("official_dataset", sa.JSON(), nullable=False),
|
||||
sa.Column("rule_version", sa.String(32), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index("ix_kb_promotion_audits_kb_product_id", "kb_promotion_audits", ["kb_product_id"])
|
||||
op.create_index("ix_kb_promotion_audits_gtin", "kb_promotion_audits", ["gtin"])
|
||||
op.add_column("provider_usage_daily", sa.Column("found", sa.Integer(), nullable=False, server_default="0"))
|
||||
op.add_column("provider_usage_daily", sa.Column("ncm_found", sa.Integer(), nullable=False, server_default="0"))
|
||||
op.add_column("provider_usage_daily", sa.Column("kb_hits", sa.Integer(), nullable=False, server_default="0"))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("provider_usage_daily", "kb_hits")
|
||||
op.drop_column("provider_usage_daily", "ncm_found")
|
||||
op.drop_column("provider_usage_daily", "found")
|
||||
op.drop_table("kb_promotion_audits")
|
||||
op.execute("UPDATE kb_knowledge SET status='EVIDENCE_ONLY' WHERE status IN ('OBSERVED','CORROBORATED','EXTERNAL_CORROBORATED')")
|
||||
op.alter_column("kb_knowledge", "status", server_default="EVIDENCE_ONLY")
|
||||
op.alter_column("kb_knowledge", "status", existing_type=sa.String(24), type_=sa.String(16), existing_nullable=False, existing_server_default="EVIDENCE_ONLY")
|
||||
@@ -81,6 +81,8 @@ class Settings:
|
||||
cosmos_token: str = os.getenv("COSMOS_TOKEN", "") # optional; without it Cosmos is NOT_CONFIGURED (not an error)
|
||||
oscbr_user: str = os.getenv("OSCBR_USER", "") # optional free-plan account of gtin.rscsistemas.com.br (never in logs/UI)
|
||||
oscbr_password: str = os.getenv("OSCBR_PASSWORD", "")
|
||||
kodebar_api_key: str = os.getenv("KODEBAR_API_KEY", "") # optional free-plan key (identity only); never in logs/UI
|
||||
provider_operational_reserve_percent: int = int(os.getenv("PROVIDER_OPERATIONAL_RESERVE_PERCENT", "20"))
|
||||
allow_paid_providers: bool = os.getenv("ALLOW_PAID_PROVIDERS", "0") == "1"
|
||||
official_refresh_enabled: bool = os.getenv("OFFICIAL_REFRESH_ENABLED", "1") == "1"
|
||||
enrichment_time_budget_seconds: float = float(os.getenv("ENRICHMENT_TIME_BUDGET_SECONDS", "4.5"))
|
||||
|
||||
@@ -46,6 +46,12 @@ def refresh_ncm(session, fetch=_fetch, url: str | None = None) -> SourceRefreshR
|
||||
raise ValueError(f"implausible record count {len(parsed.entries)} vs active {active.record_count}")
|
||||
DatasetService(session).import_payload(parsed, url, datetime.now(timezone.utc)) # atomic: one transaction, old ACTIVE superseded only at the end
|
||||
run.status = "UPDATED"
|
||||
try: # the KB is never deleted: only flagged STALE_REVIEW_REQUIRED when its NCM left the official dataset
|
||||
from ..enrichment import history, kb
|
||||
current = history.official_sets(session)
|
||||
run.error = f"kb_revalidated: {kb.revalidate(session, lambda v: v in current)}"[:300]
|
||||
except Exception as error: # noqa: BLE001 - revalidation trouble must not undo a valid dataset promotion
|
||||
run.error = f"kb_revalidate_failed: {type(error).__name__}"[:300]
|
||||
except Exception as error: # keep the last valid dataset
|
||||
session.rollback()
|
||||
run = session.get(SourceRefreshRun, run.id)
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Historical ERP evidence (product master + NFC-e/NF-e items) -> central KB evidence. Local data only: NO provider is contacted here.
|
||||
|
||||
Evidence model
|
||||
- One 'lineage' per client per (GTIN, value): master-data flag, fiscal-document count and documents per year. Master data and the fiscal documents of the SAME ERP are one family (CLIENT_DATA):
|
||||
the documents were issued from that very master, so they are never counted as two independent sources.
|
||||
- CLIENT_DATA alone can reach CORROBORATED (>= 20 documents over >= 2 years, or >= 2 distinct clients) but never VERIFIED_KB: that needs an independent NCM-bearing source (kb.evaluate).
|
||||
- NCM validity is judged against the ACTIVE official dataset today: CURRENT_VALID / HISTORICAL_EXPIRED / HISTORICAL_INVALID; a different NCM at least as recent -> HISTORICAL_CHANGED; two current values -> CONFLICT.
|
||||
- Company-specific tax data (CRT, CFOP, CSOSN, ...) is never read nor stored; only GTIN, descriptions/aliases, NCM, CEST and the source tenant label.
|
||||
Idempotent: a client's lineage for a (GTIN, value) is REPLACED on every import, never accumulated.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..models import KbKnowledge, KbProduct
|
||||
from . import kb, textnorm
|
||||
from .gtin import normalize_gtin
|
||||
|
||||
|
||||
def _digits(v) -> str:
|
||||
return re.sub(r"\D", "", str(v or ""))
|
||||
|
||||
|
||||
def build_observations(stagings: list[dict]) -> dict:
|
||||
"""stagings: miner output per client. Returns {"gtins": {key: {...}}, "stats": {...}}."""
|
||||
gtins: dict[str, dict] = {}
|
||||
stats = defaultdict(int)
|
||||
for st in stagings:
|
||||
client = st["client"]
|
||||
product_gtin: dict[str, set] = defaultdict(set)
|
||||
master_gtin: dict[str, str] = {}
|
||||
master_desc: dict[str, str] = {}
|
||||
for m in st["master"]:
|
||||
stats["master_rows"] += 1
|
||||
master_desc[m["code"]] = m["desc"]
|
||||
for raw in (m["gtin"], m["codbarra"]):
|
||||
g = normalize_gtin(raw)
|
||||
if g["status"] == "VALID_GTIN":
|
||||
master_gtin.setdefault(m["code"], g["key"])
|
||||
break
|
||||
# product<->EAN links seen in documents/movements are used ONLY when the master has none, and only when unambiguous
|
||||
seen: dict[str, set] = defaultdict(set)
|
||||
for row in st.get("nfe", []):
|
||||
g = normalize_gtin(row["codbarra"])
|
||||
if g["status"] == "VALID_GTIN":
|
||||
seen[row["code"]].add(g["key"])
|
||||
for row in st.get("barcodes", []):
|
||||
g = normalize_gtin(row["codbarra"])
|
||||
if g["status"] == "VALID_GTIN":
|
||||
seen[row["code"]].add(g["key"])
|
||||
link: dict[str, str] = dict(master_gtin)
|
||||
for code, keys in seen.items():
|
||||
if code in link:
|
||||
continue
|
||||
if len(keys) == 1:
|
||||
link[code] = next(iter(keys))
|
||||
stats["links_from_documents"] += 1
|
||||
else:
|
||||
stats["ambiguous_product_links_skipped"] += 1
|
||||
|
||||
def slot(key):
|
||||
return gtins.setdefault(key, {"aliases": set(), "ncm": defaultdict(lambda: defaultdict(lambda: {"master": 0, "docs": 0, "years": {}})), "cest": defaultdict(lambda: defaultdict(lambda: {"master": 0, "docs": 0, "years": {}})), "clients": set()})
|
||||
for m in st["master"]:
|
||||
key = link.get(m["code"])
|
||||
if not key:
|
||||
continue
|
||||
s = slot(key)
|
||||
s["clients"].add(client)
|
||||
if m["desc"]:
|
||||
s["aliases"].add(textnorm.normalized(m["desc"]))
|
||||
n = _digits(m["ncm"])
|
||||
if len(n) == 8:
|
||||
s["ncm"][n][client]["master"] = 1
|
||||
c = _digits(m["cest"])
|
||||
if len(c) == 7:
|
||||
s["cest"][c][client]["master"] = 1
|
||||
for row in st.get("nfce", []):
|
||||
key = link.get(row["code"])
|
||||
stats["nfce_groups"] += 1
|
||||
if not key:
|
||||
stats["nfce_groups_without_gtin"] += 1
|
||||
continue
|
||||
s = slot(key)
|
||||
n = _digits(row["ncm"])
|
||||
if len(n) == 8:
|
||||
lin = s["ncm"][n][client]
|
||||
lin["docs"] += row["n"]
|
||||
if row["year"]:
|
||||
lin["years"][str(row["year"])] = lin["years"].get(str(row["year"]), 0) + row["n"]
|
||||
stats["nfce_documents"] += row["n"]
|
||||
c = _digits(row["cest"])
|
||||
if len(c) == 7:
|
||||
lin = s["cest"][c][client]
|
||||
lin["docs"] += row["n"]
|
||||
if row["year"]:
|
||||
lin["years"][str(row["year"])] = lin["years"].get(str(row["year"]), 0) + row["n"]
|
||||
for row in st.get("nfe", []):
|
||||
key = link.get(row["code"])
|
||||
if not key:
|
||||
continue
|
||||
s = slot(key)
|
||||
if row["desc"]:
|
||||
s["aliases"].add(textnorm.normalized(row["desc"]))
|
||||
n = _digits(row["ncm"])
|
||||
if len(n) == 8:
|
||||
s["ncm"][n][client]["docs"] += row["n"]
|
||||
stats["nfe_documents"] += row["n"]
|
||||
stats["gtins"] = len(gtins)
|
||||
return {"gtins": gtins, "stats": dict(stats)}
|
||||
|
||||
|
||||
def apply(session, observations: dict, is_official, *, dry_run: bool = True, chunk: int = 500) -> dict:
|
||||
"""Bulk, idempotent import into kb_products / kb_knowledge. dry_run=True computes every status and rolls back."""
|
||||
keys = sorted(observations["gtins"])
|
||||
report = {"GTINS_FOUND": len(keys), "OBSERVATIONS": 0, "by_status": defaultdict(int), "CONFLICTS": 0, "WOULD_VERIFY": 0, "with_current_valid_ncm": 0, "stable_across_years": 0, "changed_over_time": 0,
|
||||
"with_historical_ncm": 0, "multi_client": 0}
|
||||
nested = session.begin_nested()
|
||||
try:
|
||||
for start in range(0, len(keys), chunk):
|
||||
part = keys[start:start + chunk]
|
||||
existing = {p.gtin: p for p in session.scalars(select(KbProduct).where(KbProduct.gtin.in_(part)))}
|
||||
for key in part:
|
||||
obs = observations["gtins"][key]
|
||||
product = existing.get(key)
|
||||
if product is None:
|
||||
alias = next(iter(sorted(obs["aliases"])), "")
|
||||
product = kb.upsert_product(session, key, alias)
|
||||
existing[key] = product
|
||||
for a in sorted(obs["aliases"])[:20]:
|
||||
kb.add_alias(session, product, a)
|
||||
fp = textnorm.fingerprint(a)
|
||||
if fp:
|
||||
kb.add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=kb.OBSERVED)
|
||||
for field, values in (("ncm", obs["ncm"]), ("cest", obs["cest"])):
|
||||
for value, per_client in values.items():
|
||||
for client, lin in per_client.items():
|
||||
kb.add_evidence(session, product, field, value, "client:history", family=kb.CLIENT_FAMILY, client_label=client, lineage=None)
|
||||
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == field, KbKnowledge.value == value))
|
||||
ev = dict(row.evidence or {})
|
||||
lineages = dict(ev.get("lineages") or {})
|
||||
lineages[client] = {"master": lin["master"], "docs": lin["docs"], "years": dict(lin["years"])} # REPLACE: idempotent
|
||||
row.evidence = {**ev, "lineages": lineages}
|
||||
report["OBSERVATIONS"] += 1
|
||||
session.flush()
|
||||
for key in part:
|
||||
product = existing[key]
|
||||
ev = kb.evaluate(session, product, "ncm", is_official)
|
||||
report["by_status"][ev["status"]] += 1
|
||||
if ev["status"] == kb.CONFLICT:
|
||||
report["CONFLICTS"] += 1
|
||||
if ev["status"] == kb.VERIFIED_KB:
|
||||
report["WOULD_VERIFY"] += 1
|
||||
if ev["value"]:
|
||||
report["with_current_valid_ncm"] += 1
|
||||
if observations["gtins"][key]["ncm"]:
|
||||
report["with_historical_ncm"] += 1
|
||||
if len(observations["gtins"][key]["clients"]) >= 2:
|
||||
report["multi_client"] += 1
|
||||
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm")))
|
||||
states = {(r.evidence.get("info") or {}).get("historical_state") for r in rows}
|
||||
if "HISTORICAL_STABLE_CURRENT" in states and ev["value"]:
|
||||
years = set()
|
||||
for r in rows:
|
||||
for lin in (r.evidence.get("lineages") or {}).values():
|
||||
years |= set(lin.get("years") or {})
|
||||
if len(years) >= 2:
|
||||
report["stable_across_years"] += 1
|
||||
if "HISTORICAL_CHANGED" in states or ("HISTORICAL_EXPIRED" in states and ev["value"]):
|
||||
report["changed_over_time"] += 1
|
||||
if dry_run:
|
||||
nested.rollback()
|
||||
else:
|
||||
nested.commit()
|
||||
session.commit()
|
||||
except Exception:
|
||||
nested.rollback()
|
||||
raise
|
||||
report["by_status"] = dict(report["by_status"])
|
||||
return report
|
||||
|
||||
|
||||
def official_sets(session):
|
||||
"""Precomputed 'vigente today' membership: one query instead of one per value."""
|
||||
from datetime import date
|
||||
from ..models import Dataset, NcmEntry
|
||||
from .resolver import _parse_br
|
||||
today = date.today()
|
||||
current = set()
|
||||
for row in session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")):
|
||||
start, end = _parse_br(row.valid_from), _parse_br(row.valid_to)
|
||||
if (start is None or start <= today) and (end is None or end >= today):
|
||||
current.add(row.code)
|
||||
return current
|
||||
+130
-31
@@ -12,11 +12,16 @@ from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..models import KbKnowledge, KbProduct
|
||||
from sqlalchemy import func
|
||||
|
||||
from ..models import KbKnowledge, KbProduct, KbPromotionAudit
|
||||
from . import textnorm
|
||||
|
||||
UNVERIFIED, EVIDENCE_ONLY, VERIFIED_KB, CONFLICT, STALE = "UNVERIFIED", "EVIDENCE_ONLY", "VERIFIED_KB", "CONFLICT", "STALE"
|
||||
CLIENT_FAMILY = "CLIENT_OBSERVATION"
|
||||
UNVERIFIED, OBSERVED, CORROBORATED, EXTERNAL_CORROBORATED, VERIFIED_KB, CONFLICT, STALE = "UNVERIFIED", "OBSERVED", "CORROBORATED", "EXTERNAL_CORROBORATED", "VERIFIED_KB", "CONFLICT", "STALE"
|
||||
EVIDENCE_ONLY = OBSERVED # legacy name
|
||||
CLIENT_FAMILY = "CLIENT_DATA" # every client-derived lineage (master data + fiscal documents of the same ERP are NOT independent of each other)
|
||||
RULE_VERSION = "kb-promotion-v2"
|
||||
STRONG_DOCS, STRONG_YEARS = 20, 2
|
||||
|
||||
|
||||
def _now():
|
||||
@@ -56,15 +61,29 @@ def upsert_product(session, key: str | None, description, *, brand=None, quantit
|
||||
if quantity and not product.package_quantity:
|
||||
product.package_quantity = str(quantity)[:32]
|
||||
fp = textnorm.fingerprint(description)
|
||||
norm = textnorm.normalized(description)
|
||||
if norm:
|
||||
add_alias(session, product, norm)
|
||||
if fp:
|
||||
add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=EVIDENCE_ONLY)
|
||||
add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=OBSERVED)
|
||||
return product
|
||||
|
||||
|
||||
def add_evidence(session, product: KbProduct, key: str, value: str, source: str, *, family: str, client_label: str | None = None, status: str = EVIDENCE_ONLY, info: dict | None = None) -> KbKnowledge:
|
||||
def add_alias(session, product: KbProduct, normalized: str, cap: int = 20):
|
||||
"""Non-fiscal aliases of the same GTIN (descriptions seen over the years). Helps name matching; never authority for a fiscal write."""
|
||||
if not normalized or session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "alias", KbKnowledge.value == normalized[:200])):
|
||||
return
|
||||
if (session.scalar(select(func.count()).select_from(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "alias")) or 0) >= cap:
|
||||
return
|
||||
session.add(KbKnowledge(kb_product_id=product.id, knowledge_key="alias", value=normalized[:200], source="ALIAS", status=OBSERVED, evidence={"families": {}, "clients": [], "info": {}}))
|
||||
|
||||
|
||||
def add_evidence(session, product: KbProduct, key: str, value: str, source: str, *, family: str, client_label: str | None = None, status: str = OBSERVED, info: dict | None = None,
|
||||
lineage: dict | None = None) -> KbKnowledge:
|
||||
"""lineage (client data only): {"master": 0/1, "docs": n, "years": {"2024": n}} of ONE client for this (GTIN, value); merged, never double counted."""
|
||||
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key, KbKnowledge.value == value))
|
||||
if row is None:
|
||||
row = KbKnowledge(kb_product_id=product.id, knowledge_key=key, value=value, source=source[:32], status=status, evidence={"families": {}, "clients": [], "info": info or {}})
|
||||
row = KbKnowledge(kb_product_id=product.id, knowledge_key=key, value=value, source=source[:32], status=status, evidence={"families": {}, "clients": [], "lineages": {}, "info": info or {}})
|
||||
session.add(row)
|
||||
session.flush()
|
||||
evidence = dict(row.evidence or {})
|
||||
@@ -73,48 +92,128 @@ def add_evidence(session, product: KbProduct, key: str, value: str, source: str,
|
||||
clients = list(evidence.get("clients") or [])
|
||||
if client_label and client_label not in clients:
|
||||
clients.append(client_label)
|
||||
row.evidence = {**evidence, "families": families, "clients": clients, "info": {**(evidence.get("info") or {}), **(info or {})}}
|
||||
lineages = dict(evidence.get("lineages") or {})
|
||||
if client_label and lineage:
|
||||
cur = dict(lineages.get(client_label) or {"master": 0, "docs": 0, "years": {}})
|
||||
cur["master"] = max(int(cur.get("master", 0)), int(lineage.get("master", 0)))
|
||||
cur["docs"] = int(cur.get("docs", 0)) + int(lineage.get("docs", 0))
|
||||
years = dict(cur.get("years") or {})
|
||||
for y, n in (lineage.get("years") or {}).items():
|
||||
years[str(y)] = int(years.get(str(y), 0)) + int(n)
|
||||
cur["years"] = years
|
||||
lineages[client_label] = cur
|
||||
row.evidence = {**evidence, "families": families, "clients": clients, "lineages": lineages, "info": {**(evidence.get("info") or {}), **(info or {})}}
|
||||
return row
|
||||
|
||||
|
||||
def _lineage_summary(rows):
|
||||
clients, docs, years = set(), 0, {}
|
||||
for r in rows:
|
||||
ev = r.evidence or {}
|
||||
for c, lin in (ev.get("lineages") or {}).items():
|
||||
clients.add(c)
|
||||
docs += int(lin.get("docs", 0))
|
||||
for y, n in (lin.get("years") or {}).items():
|
||||
years[y] = years.get(y, 0) + int(n)
|
||||
for c in ev.get("clients") or []:
|
||||
clients.add(c)
|
||||
return {"clients": sorted(clients), "docs": docs, "years": sorted(years), "last_year": max(int(y) for y in years) if years else None}
|
||||
|
||||
|
||||
def independent_count(rows) -> tuple[int, list[str]]:
|
||||
families = {}
|
||||
clients: set[str] = set()
|
||||
for row in rows:
|
||||
ev = row.evidence or {}
|
||||
for fam in (ev.get("families") or {}):
|
||||
if fam.startswith("PROVIDER:"):
|
||||
families[fam] = True
|
||||
elif fam == CLIENT_FAMILY:
|
||||
clients.update(ev.get("clients") or [])
|
||||
names = sorted(families)
|
||||
if len(clients) >= 2:
|
||||
names.append(f"{CLIENT_FAMILY}(x{len(clients)})")
|
||||
"""Independent families: each NCM-bearing provider = 1; ALL client-derived data = 1 and only when STRONG (>= STRONG_DOCS documents over >= STRONG_YEARS years, or >= 2 distinct clients)."""
|
||||
names = sorted({f for r in rows for f in ((r.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:")})
|
||||
summary = _lineage_summary(rows)
|
||||
has_client = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in rows)
|
||||
if has_client and (len(summary["clients"]) >= 2 or (summary["docs"] >= STRONG_DOCS and len(summary["years"]) >= STRONG_YEARS)):
|
||||
names.append(CLIENT_FAMILY)
|
||||
return len(names), names
|
||||
|
||||
|
||||
def _official_period(session, value):
|
||||
"""CURRENT_VALID / HISTORICAL_EXPIRED (exists in the dataset but not vigente today) / HISTORICAL_INVALID (never in the dataset)."""
|
||||
from .resolver import ncm_exists_any
|
||||
return "CURRENT_VALID" if ncm_exists_any(session, value, current=True) else ("HISTORICAL_EXPIRED" if ncm_exists_any(session, value, current=False) else "HISTORICAL_INVALID")
|
||||
|
||||
|
||||
def _mark(row, state):
|
||||
row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "historical_state": state}}
|
||||
|
||||
|
||||
def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
|
||||
"""Recomputes and PERSISTS the status of one field. Returns {value, status, families, candidates, rejected}."""
|
||||
"""Recomputes and PERSISTS the status of one field. Never 'majority wins': two current-valid values => CONFLICT."""
|
||||
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key)))
|
||||
if not rows:
|
||||
return {"value": None, "status": UNVERIFIED, "families": [], "candidates": [], "rejected": []}
|
||||
return {"value": None, "status": UNVERIFIED, "families": [], "candidates": [], "rejected": [], "detail": {}}
|
||||
valid = [r for r in rows if is_official(r.value)]
|
||||
rejected = [r.value for r in rows if not is_official(r.value)]
|
||||
for r in rows:
|
||||
if r not in valid:
|
||||
state = _official_period(session, r.value) if key == "ncm" else "HISTORICAL_INVALID"
|
||||
r.status = STALE if state == "HISTORICAL_EXPIRED" else OBSERVED
|
||||
_mark(r, state)
|
||||
values = sorted({r.value for r in valid})
|
||||
if len(values) > 1:
|
||||
for r in rows:
|
||||
for r in valid:
|
||||
r.status = CONFLICT
|
||||
return {"value": None, "status": CONFLICT, "families": [], "candidates": values, "rejected": rejected}
|
||||
_mark(r, "HISTORICAL_CONFLICT")
|
||||
return {"value": None, "status": CONFLICT, "families": [], "candidates": values, "rejected": rejected, "detail": {"clients": _lineage_summary(valid)["clients"]}}
|
||||
if not values:
|
||||
for r in rows:
|
||||
r.status = EVIDENCE_ONLY
|
||||
return {"value": None, "status": EVIDENCE_ONLY, "families": [], "candidates": [], "rejected": rejected}
|
||||
return {"value": None, "status": OBSERVED, "families": [], "candidates": [], "rejected": rejected, "detail": {}}
|
||||
chosen = [r for r in valid if r.value == values[0]]
|
||||
old = chosen[0].status
|
||||
count, names = independent_count(chosen)
|
||||
summary = _lineage_summary(chosen)
|
||||
other_last = _lineage_summary([r for r in rows if r not in chosen])["last_year"]
|
||||
superseded = other_last is not None and summary["last_year"] is not None and other_last >= summary["last_year"] # a different (expired) NCM is at least as recent: not stable
|
||||
if superseded and CLIENT_FAMILY in names:
|
||||
names.remove(CLIENT_FAMILY)
|
||||
count -= 1
|
||||
providers = [n for n in names if n.startswith("PROVIDER:")]
|
||||
already = any(r.status == VERIFIED_KB and r.validated_at for r in chosen)
|
||||
status = VERIFIED_KB if (count >= 2 or already) else EVIDENCE_ONLY
|
||||
for r in rows:
|
||||
r.status = status if r.value == values[0] else EVIDENCE_ONLY
|
||||
if status == VERIFIED_KB and r.value == values[0] and not r.validated_at:
|
||||
has_client_data = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen)
|
||||
if count >= 2 or already:
|
||||
status = VERIFIED_KB
|
||||
elif providers and has_client_data:
|
||||
status = EXTERNAL_CORROBORATED
|
||||
elif CLIENT_FAMILY in names:
|
||||
status = CORROBORATED
|
||||
else:
|
||||
status = OBSERVED
|
||||
for r in chosen:
|
||||
r.status = status
|
||||
_mark(r, "HISTORICAL_CHANGED" if superseded else "HISTORICAL_STABLE_CURRENT")
|
||||
if status == VERIFIED_KB and not r.validated_at:
|
||||
r.validated_at = _now()
|
||||
return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected}
|
||||
if status == VERIFIED_KB and old != VERIFIED_KB and key == "ncm":
|
||||
from .resolver import dataset_versions
|
||||
session.add(KbPromotionAudit(kb_product_id=product.id, gtin=product.gtin, field=key, value=values[0], old_status=old, new_status=status, families=names,
|
||||
sources={"clients": summary["clients"], "documents": summary["docs"], "years": summary["years"], "providers": providers},
|
||||
official_dataset=dataset_versions(session).get("NCM", {}), rule_version=RULE_VERSION))
|
||||
return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected, "detail": summary}
|
||||
|
||||
|
||||
def revalidate(session, is_official) -> dict:
|
||||
"""After the OFFICIAL NCM dataset changed: never delete knowledge. Rows whose NCM is no longer vigente become STALE (STALE_REVIEW_REQUIRED, audited when they were VERIFIED_KB);
|
||||
rows still vigente keep their status (VERIFIED_KB stays valid while the NCM is vigente)."""
|
||||
from .resolver import dataset_versions
|
||||
stale = still = 0
|
||||
versions = dataset_versions(session).get("NCM", {})
|
||||
for row in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm")):
|
||||
if is_official(row.value):
|
||||
still += 1
|
||||
if row.status == STALE:
|
||||
row.status = OBSERVED # became vigente again: needs a fresh evaluate() to regain its level
|
||||
continue
|
||||
if row.status != STALE:
|
||||
was = row.status
|
||||
row.status = STALE
|
||||
_mark(row, "HISTORICAL_EXPIRED")
|
||||
row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "stale_reason": "STALE_REVIEW_REQUIRED"}}
|
||||
stale += 1
|
||||
if was == VERIFIED_KB:
|
||||
product = session.get(KbProduct, row.kb_product_id)
|
||||
session.add(KbPromotionAudit(kb_product_id=row.kb_product_id, gtin=product.gtin if product else None, field="ncm", value=row.value, old_status=was, new_status=STALE,
|
||||
families=sorted((row.evidence or {}).get("families") or {}), sources={"reason": "NCM_NO_LONGER_VIGENTE"}, official_dataset=versions, rule_version=RULE_VERSION))
|
||||
session.flush()
|
||||
return {"stale": stale, "still_vigente": still}
|
||||
|
||||
@@ -39,6 +39,8 @@ def provider_sources(session, providers=None) -> list[dict]:
|
||||
"latency_ms": state.last_latency_ms if state else None,
|
||||
"cache_hit_rate": round(100 * usage.cache_hits / lookups) if usage and lookups else None,
|
||||
"errors": usage.errors if usage else 0, "rate_limited": usage.rate_limited if usage else 0,
|
||||
"found_today": usage.found if usage else 0, "ncm_found_today": usage.ncm_found if usage else 0, "cache_hits": usage.cache_hits if usage else 0,
|
||||
"next_reset": _aware(state.reset_at) if state and state.reset_at else None, "identity_only": not getattr(provider, "provides_ncm", False),
|
||||
})
|
||||
return rows
|
||||
|
||||
@@ -80,3 +82,24 @@ def coverage_metrics(session) -> dict:
|
||||
"NCM_AUTO_FILL_SAFE_RATE": rate(auto_ncm), "NAME_ONLY_RESOLUTION_RATE": rate(states.get("EXACT_KB_NAME_VERIFIED", 0)), "AMBIGUOUS_RATE": rate(states.get("AMBIGUOUS", 0)),
|
||||
"CONFLICT_RATE": rate(states.get("CONFLICT", 0)), "PROVIDER_QUOTA_BLOCK_RATE": round(100 * limited / requests, 1) if requests else None, "states": states, "by_status": by,
|
||||
"provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}
|
||||
|
||||
|
||||
def kb_summary(session) -> dict:
|
||||
"""Compact Knowledge Base panel: known products, statuses of the NCM knowledge, hits today, and the latest conflicts with their evidence (no dashboard of its own)."""
|
||||
from ..models import KbKnowledge, KbProduct
|
||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
products = session.scalar(select(func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
|
||||
by = dict(session.execute(select(KbKnowledge.status, func.count(func.distinct(KbKnowledge.kb_product_id))).where(KbKnowledge.knowledge_key == "ncm").group_by(KbKnowledge.status)).all())
|
||||
hits = session.scalar(select(func.coalesce(func.sum(ProviderUsageDaily.kb_hits), 0)).where(ProviderUsageDaily.day == today)) or 0
|
||||
conflicts = []
|
||||
rows = session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.status == "CONFLICT").order_by(KbKnowledge.id.desc()).limit(60)).all()
|
||||
grouped = {}
|
||||
for r in rows:
|
||||
grouped.setdefault(r.kb_product_id, []).append(r)
|
||||
for pid, items in list(grouped.items())[:5]:
|
||||
prod = session.get(KbProduct, pid)
|
||||
conflicts.append({"gtin": (prod.gtin or "")[-13:], "description": (prod.normalized_description or "")[:60],
|
||||
"candidates": [{"ncm": i.value, "clients": sorted({c for c in (i.evidence or {}).get("clients", [])}), "documents": sum(int(l.get("docs", 0)) for l in ((i.evidence or {}).get("lineages") or {}).values()),
|
||||
"providers": sorted(f for f in ((i.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:"))} for i in items]})
|
||||
return {"products": products, "verified": by.get("VERIFIED_KB", 0), "corroborated": by.get("CORROBORATED", 0) + by.get("EXTERNAL_CORROBORATED", 0), "observed": by.get("OBSERVED", 0),
|
||||
"conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts}
|
||||
|
||||
@@ -226,10 +226,11 @@ class Cosmos(Provider):
|
||||
|
||||
|
||||
class Oscbr(Provider):
|
||||
"""OSCBR / RSC Sistemas GTIN API (https://gtin.rscsistemas.com.br). Free plan (no card): 20 req/min; account created BY THE OPERATOR, credentials only in the Hub env
|
||||
"""OSCBR / RSC Sistemas GTIN API (https://gtin.rscsistemas.com.br). Free plan (no card): 50 calls/day, 5/min; account created BY THE OPERATOR, credentials only in the Hub env
|
||||
(OSCBR_USER / OSCBR_PASSWORD). Bearer token lasts 1 h and is renewed automatically. Optional: NOT_CONFIGURED without credentials (no call is made)."""
|
||||
name = "oscbr"
|
||||
rate_per_minute = 20
|
||||
rate_per_minute = 5 # free plan as currently documented: 50 calls/day, 5/min (conservative: the lower figure wins)
|
||||
daily_limit = 50
|
||||
provides_ncm = True
|
||||
base = "https://gtin.rscsistemas.com.br"
|
||||
|
||||
@@ -287,9 +288,41 @@ class Oscbr(Provider):
|
||||
raw_hash=hashlib.sha256(json.dumps(body, sort_keys=True, ensure_ascii=False).encode()).hexdigest(), latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
|
||||
|
||||
class Kodebar(Provider):
|
||||
"""Kodebar (Korven Sistemas) - IDENTITY source only (name, brand, thumbnail; NO NCM). Free 'Teste' plan: 50 queries/day, no card, personal API key created by the operator
|
||||
(KODEBAR_API_KEY in the Hub env). Terms (2026-09-26): integration into our own multi-user system and internal caching are allowed; redistribution/resale is not.
|
||||
429 above the quota, no overage billing. NOT_CONFIGURED without a key: no request is made."""
|
||||
name = "kodebar"
|
||||
daily_limit = 50
|
||||
rate_per_minute = 30
|
||||
base = "https://kodebar.korvensistemas.com.br"
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self._token)
|
||||
|
||||
def lookup_gtin(self, gtin: str) -> ProviderProduct:
|
||||
if not self.configured:
|
||||
raise ProviderError("AUTH_ERROR", "NOT_CONFIGURED")
|
||||
response = self._get(f"{self.base}/gtin/lookup", params={"gtin": gtin}, headers={"X-API-Key": self._token or "", "User-Agent": USER_AGENT})
|
||||
headers = {k.lower(): v for k, v in response.headers.items()}
|
||||
if response.status_code == 404:
|
||||
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
if response.status_code != 200:
|
||||
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError as error:
|
||||
raise ProviderError("BAD_RESPONSE", "not JSON") from error
|
||||
if not isinstance(body, dict) or not body.get("nome"):
|
||||
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
return ProviderProduct(self.name, gtin, True, title=body.get("nome"), brand=body.get("marca"), raw_hash=hashlib.sha256(json.dumps({k: body.get(k) for k in ("gtin", "nome", "marca")}, sort_keys=True, ensure_ascii=False).encode()).hexdigest(),
|
||||
latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
|
||||
|
||||
def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]:
|
||||
"""Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos then OSCBR (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here."""
|
||||
providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), Oscbr(client, None, getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
|
||||
providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), Oscbr(client, None, getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), Kodebar(client, getattr(settings, "kodebar_api_key", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
|
||||
if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False):
|
||||
raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)")
|
||||
return providers
|
||||
|
||||
@@ -79,7 +79,7 @@ class QuotaManager:
|
||||
return row
|
||||
|
||||
# ---- decision ----------------------------------------------------------------------------------------------------------------------------------
|
||||
def allow(self, provider: Provider) -> tuple[bool, str | None, datetime | None]:
|
||||
def allow(self, provider: Provider, purpose: str = "OPERATIONAL", reserve_percent: int = 0) -> tuple[bool, str | None, datetime | None]:
|
||||
state, now = self.state(provider), self.now()
|
||||
if not provider.configured:
|
||||
return False, "NOT_CONFIGURED", None
|
||||
@@ -91,6 +91,12 @@ class QuotaManager:
|
||||
if provider.daily_limit and usage.requests >= provider.daily_limit and state.quota_kind != "REMOTE_QUOTA":
|
||||
state.status = "RATE_LIMITED"
|
||||
return False, "RATE_LIMITED", state.reset_at and _aware(state.reset_at) or next_utc_midnight(now)
|
||||
if purpose == "BACKFILL" and provider.daily_limit: # backfill may only use what is left above the operational reserve
|
||||
used = usage.requests
|
||||
if state.quota_kind == "REMOTE_QUOTA" and state.remaining is not None:
|
||||
used = provider.daily_limit - state.remaining
|
||||
if used >= provider.daily_limit * (100 - reserve_percent) / 100:
|
||||
return False, "OPERATIONAL_RESERVE", state.reset_at and _aware(state.reset_at) or next_utc_midnight(now)
|
||||
window = (provider.burst[1] if provider.burst else 60)
|
||||
limit = (provider.burst[0] if provider.burst else provider.rate_per_minute)
|
||||
if limit:
|
||||
@@ -123,6 +129,10 @@ class QuotaManager:
|
||||
usage.success += 1
|
||||
if not result.found:
|
||||
usage.not_found += 1
|
||||
else:
|
||||
usage.found += 1
|
||||
if result.ncm:
|
||||
usage.ncm_found += 1
|
||||
state.consecutive_failures = 0
|
||||
state.circuit_open_until = None
|
||||
state.status = "ONLINE"
|
||||
@@ -147,7 +157,7 @@ class QuotaManager:
|
||||
usage.rate_limited += 1
|
||||
state.status = "RATE_LIMITED"
|
||||
wait = timedelta(seconds=error.retry_after) if error.retry_after else None
|
||||
state.circuit_open_until = now + (wait or (state.reset_at and max(timedelta(minutes=5), _aware(state.reset_at) - now) or timedelta(minutes=15)))
|
||||
state.circuit_open_until = now + (wait or (state.reset_at and max(timedelta(minutes=15), _aware(state.reset_at) - now) or timedelta(minutes=60)))
|
||||
if state.reset_at is None:
|
||||
state.reset_at = state.circuit_open_until
|
||||
elif error.kind == "AUTH_ERROR":
|
||||
@@ -165,6 +175,10 @@ class QuotaManager:
|
||||
state.status = "OFFLINE"
|
||||
self.session.commit()
|
||||
|
||||
def record_kb_hit(self):
|
||||
"""A resolution answered by the KB alone (no provider asked): counted once per day under the pseudo-provider 'kb'."""
|
||||
self.usage("kb").kb_hits += 1
|
||||
|
||||
def record_cache(self, provider_name: str, hit: bool):
|
||||
usage = self.usage(provider_name)
|
||||
if hit:
|
||||
@@ -200,7 +214,7 @@ class ProviderGateway:
|
||||
row.found, row.data, row.fetched_at, row.expires_at = product.found, data, now, now + ttl
|
||||
self.session.commit()
|
||||
|
||||
def lookup(self, provider: Provider, gtin: str) -> Outcome:
|
||||
def lookup(self, provider: Provider, gtin: str, purpose: str = "OPERATIONAL", reserve_percent: int = 0) -> Outcome:
|
||||
hit = self._cached(provider, gtin)
|
||||
if hit is not False:
|
||||
self.quota.record_cache(provider.name, True)
|
||||
@@ -216,7 +230,7 @@ class ProviderGateway:
|
||||
self.session.commit()
|
||||
return Outcome(provider.name, "CACHE_HIT", hit or None, from_cache=True) if hit else Outcome(provider.name, "NOT_FOUND", None, from_cache=True)
|
||||
self.quota.record_cache(provider.name, False)
|
||||
ok, reason, retry_at = self.quota.allow(provider)
|
||||
ok, reason, retry_at = self.quota.allow(provider, purpose, reserve_percent)
|
||||
if not ok:
|
||||
self.session.commit()
|
||||
return Outcome(provider.name, "SKIPPED", None, reason, retry_at)
|
||||
|
||||
@@ -56,6 +56,16 @@ class _ScopedDatasets(DatasetService):
|
||||
return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == self.code)).all()
|
||||
|
||||
|
||||
def ncm_exists_any(session, code: str, *, current: bool, today: date | None = None) -> bool:
|
||||
"""current=True: vigente today; current=False: present in the ACTIVE dataset with any validity window (i.e. known but not vigente)."""
|
||||
code = normalize_code(code)
|
||||
if len(code) != 8:
|
||||
return False
|
||||
if current:
|
||||
return ncm_is_official(session, code, today)
|
||||
return session.scalar(select(NcmEntry.id).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code).limit(1)) is not None
|
||||
|
||||
|
||||
def dataset_versions(session) -> dict:
|
||||
return {d.kind: {"sha256": (d.sha256 or "")[:12], "version": d.version_label} for d in session.scalars(select(Dataset).where(Dataset.status == "ACTIVE"))}
|
||||
|
||||
@@ -77,7 +87,8 @@ def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current:
|
||||
elif ncm_eval["value"] and ncm_eval["status"] == kb.VERIFIED_KB:
|
||||
out["CLASSIFICACAO_FISCAL"] = _field(AUTO, ncm_eval["value"], "NCM_KB_VERIFIED" if gtin_kb_verified or name_verified else "NCM_TWO_INDEPENDENT_SOURCES", kb.VERIFIED_KB, ncm_eval["families"])
|
||||
elif ncm_eval["value"]:
|
||||
out["CLASSIFICACAO_FISCAL"] = _field(SUGGEST, ncm_eval["value"], "NCM_SINGLE_SOURCE", kb.EVIDENCE_ONLY, ncm_eval["families"], "apenas uma fonte independente")
|
||||
rule = {"CORROBORATED": "NCM_CORROBORATED_LOCAL_HISTORY", "EXTERNAL_CORROBORATED": "NCM_EXTERNAL_CORROBORATED"}.get(ncm_eval["status"], "NCM_SINGLE_SOURCE")
|
||||
out["CLASSIFICACAO_FISCAL"] = _field(SUGGEST, ncm_eval["value"], rule, ncm_eval["status"], ncm_eval["families"], "evidência ainda insuficiente para preencher automaticamente")
|
||||
else:
|
||||
out["CLASSIFICACAO_FISCAL"] = _field(INSUFFICIENT, reason="nenhuma fonte com NCM válido no dataset oficial")
|
||||
ncm = out["CLASSIFICACAO_FISCAL"]
|
||||
@@ -124,14 +135,25 @@ def resolve(session, gateway: ProviderGateway, *, gtin_raw, description, current
|
||||
product = kb.find_by_gtin(session, key)
|
||||
ev = kb.evaluate(session, product, "ncm", is_official) if product else None
|
||||
if ev and ev["status"] == kb.VERIFIED_KB:
|
||||
gateway.quota.record_kb_hit()
|
||||
result["state"] = "EXACT_GTIN_VERIFIED"
|
||||
result["fields"] = _fiscal_decisions(session, ev, True, current, description or "")
|
||||
session.commit()
|
||||
return result
|
||||
if ev and ev["status"] == kb.CONFLICT: # local evidence already conflicts: an external answer cannot settle it, so spend no quota
|
||||
result["state"] = "CONFLICT"
|
||||
result["fields"] = _fiscal_decisions(session, ev, False, current, description or "")
|
||||
gateway.quota.record_kb_hit()
|
||||
session.commit()
|
||||
return result
|
||||
locally_known = bool(ev and ev["status"] in (kb.CORROBORATED, kb.EXTERNAL_CORROBORATED)) # identity + NCM candidate already local: only an NCM-bearing source can still add value
|
||||
query = g["gtin"] if len(g["gtin"]) in (8, 14) else key[1:] # providers/caches use the canonical EAN-13 (UPC-12 is zero-padded to it)
|
||||
found, pending = [], []
|
||||
if allow_external:
|
||||
for provider in gateway.providers:
|
||||
if locally_known and not getattr(provider, "provides_ncm", False):
|
||||
result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "LOCAL_IDENTITY_KNOWN", "from_cache": False})
|
||||
continue
|
||||
if budget_s is not None and time.monotonic() - started > budget_s:
|
||||
result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "TIME_BUDGET", "from_cache": False})
|
||||
pending.append(type("P", (), {"reason": "TIME_BUDGET", "retry_at": None})())
|
||||
|
||||
@@ -7,6 +7,7 @@ UNIT_MAP = {"LT": "L", "LTS": "L", "LITRO": "L", "LITROS": "L", "L": "L", "ML":
|
||||
ABBREVIATIONS = {"REFRIG": "REFRIGERANTE", "REFRI": "REFRIGERANTE", "CHOC": "CHOCOLATE", "BISC": "BISCOITO", "SAB": "SABONETE", "DET": "DETERGENTE", "SH": "SHAMPOO", "COND": "CONDICIONADOR",
|
||||
"MACARR": "MACARRAO", "MAC": "MACARRAO", "AMAC": "AMACIANTE", "DESOD": "DESODORANTE", "PAP": "PAPEL", "HIG": "HIGIENICO", "ORIG": "ORIGINAL", "TRAD": "TRADICIONAL"}
|
||||
STOP = {"DE", "DA", "DO", "DAS", "DOS", "E", "COM", "C", "PARA", "P", "EM", "A", "O"}
|
||||
MARKETING = {"ORIGINAL", "TRADICIONAL", "NOVO", "NOVA", "PROMO", "PROMOCAO", "OFERTA"} # ignored ONLY by the fingerprint; a different pack/flavour still differs
|
||||
_QTY = re.compile(r"^(\d+(?:[.,]\d+)?)(ML|L|KG|G|MG|UN|PCT|CX|LT|LTS|GR)$")
|
||||
|
||||
|
||||
@@ -35,7 +36,7 @@ def normalized(description) -> str:
|
||||
|
||||
|
||||
def fingerprint(description) -> str | None:
|
||||
toks = tokens(description)
|
||||
toks = [t for t in tokens(description) if t not in MARKETING]
|
||||
if not toks:
|
||||
return None
|
||||
return hashlib.sha256(" ".join(sorted(toks)).encode("utf-8")).hexdigest()
|
||||
|
||||
+21
-1
@@ -629,7 +629,7 @@ class KbKnowledge(Base):
|
||||
dataset_versions: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
validated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(16), default="EVIDENCE_ONLY", server_default="EVIDENCE_ONLY", index=True) # UNVERIFIED | EVIDENCE_ONLY | VERIFIED_KB | CONFLICT | STALE
|
||||
status: Mapped[str] = mapped_column(String(24), default="OBSERVED", server_default="OBSERVED", index=True) # OBSERVED | CORROBORATED | EXTERNAL_CORROBORATED | VERIFIED_KB | CONFLICT | STALE
|
||||
|
||||
|
||||
class KbObservation(Base):
|
||||
@@ -879,6 +879,9 @@ class ProviderUsageDaily(Base):
|
||||
timeouts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
cache_hits: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
cache_misses: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
found: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
ncm_found: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
kb_hits: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class GtinLookupCache(Base):
|
||||
@@ -907,3 +910,20 @@ class SourceRefreshRun(Base):
|
||||
version_label: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
record_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(String(300), nullable=True)
|
||||
|
||||
|
||||
class KbPromotionAudit(Base):
|
||||
"""Why a GTIN/NCM became (or stopped being) trustworthy: statuses, evidence families, source references and the official dataset hash at that moment."""
|
||||
__tablename__ = "kb_promotion_audits"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
|
||||
gtin: Mapped[str | None] = mapped_column(String(16), nullable=True, index=True)
|
||||
field: Mapped[str] = mapped_column(String(24), default="ncm", server_default="ncm")
|
||||
value: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
old_status: Mapped[str | None] = mapped_column(String(24), nullable=True)
|
||||
new_status: Mapped[str] = mapped_column(String(24))
|
||||
families: Mapped[list] = mapped_column(JSON, default=list)
|
||||
sources: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
official_dataset: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
rule_version: Mapped[str] = mapped_column(String(32))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
|
||||
once published — never a second, independent version number. Fiscal semantics
|
||||
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
|
||||
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
|
||||
GTI_FISCAL_VERSION = "1.3.1"
|
||||
GTI_FISCAL_VERSION = "1.3.2"
|
||||
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
|
||||
|
||||
+2
-2
@@ -22,7 +22,7 @@ from .panel_presentation import format_ui_datetime
|
||||
from .models import AdminUser, Agent, AgentEnrollment, AgentRelease, AgentSnapshot, BlockedProduct, FiscalContextEntry, Installation, Job, ProductEnrichment, ProductFieldAudit, RtcCandidate, RtcWriteAudit, Tenant
|
||||
from .jobs import create_job
|
||||
from .audit import record_audit
|
||||
from .enrichment.observability import coverage_metrics, official_sources, provider_sources
|
||||
from .enrichment.observability import coverage_metrics, kb_summary, official_sources, provider_sources
|
||||
|
||||
|
||||
def router(session_factory):
|
||||
@@ -161,7 +161,7 @@ def router(session_factory):
|
||||
def dashboard(request: Request):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session)})
|
||||
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session), "kb": kb_summary(session)})
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents", response_class=HTMLResponse)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "gti-fiscal-hub"
|
||||
version = "1.3.1"
|
||||
version = "1.3.2"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"fastapi>=0.115,<1",
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Import historical ERP evidence (miner output) into the central KB. Dry-run by default; --apply commits. Local data only, no provider call.
|
||||
|
||||
python -m central.scripts.import_kb_history --staging /path/to/history [--apply] [--report out.json]
|
||||
|
||||
Runs against the database in DATABASE_URL (the Hub's PostgreSQL). Idempotent. Never touches any client database.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.enrichment import history
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--staging", required=True)
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
ap.add_argument("--report", default=None)
|
||||
ap.add_argument("--only-current-vigente-gtins", action="store_true", help="skip GTINs without any NCM evidence")
|
||||
args = ap.parse_args(argv)
|
||||
files = sorted(f for f in Path(args.staging).glob("*.json") if not f.name.startswith("_"))
|
||||
stagings = [json.loads(f.read_text(encoding="utf-8")) for f in files]
|
||||
t0 = time.time()
|
||||
obs = history.build_observations(stagings)
|
||||
if args.only_current_vigente_gtins:
|
||||
obs["gtins"] = {k: v for k, v in obs["gtins"].items() if v["ncm"]}
|
||||
with SessionLocal() as session:
|
||||
current = history.official_sets(session)
|
||||
report = history.apply(session, obs, lambda v: v in current, dry_run=not args.apply)
|
||||
report.update({"mode": "APPLY" if args.apply else "DRY_RUN", "clients": [s["client"] for s in stagings], "build_stats": obs["stats"], "seconds": round(time.time() - t0, 1)})
|
||||
text = json.dumps(report, indent=1, default=str)
|
||||
if args.report:
|
||||
Path(args.report).write_text(text, encoding="utf-8")
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -19,6 +19,7 @@
|
||||
<small>{% if src.limit %}{{ src.used }} / {{ src.limit }} consultas hoje · restam {{ src.remaining if src.remaining is not none else "—" }}{% if src.reset_at %} · renova {{ src.reset_at.strftime("%d/%m %H:%M") }} UTC{% endif %}{% elif src.rate_per_minute %}{{ src.rate_per_minute }} consultas/min (limite documentado) · {{ src.used }} hoje{% else %}{{ src.used }} consultas hoje{% endif %}
|
||||
{% if src.latency_ms %} · {{ src.latency_ms }} ms{% endif %}{% if src.cache_hit_rate is not none %} · cache {{ src.cache_hit_rate }}%{% endif %}
|
||||
{% if src.last_success_at %} · último sucesso {{ src.last_success_at.strftime("%d/%m %H:%M") }}{% endif %}{% if src.last_error %} · último erro: {{ src.last_error }}{% endif %}
|
||||
· hoje: {{ src.found_today }} achados{% if not src.identity_only %}, {{ src.ncm_found_today }} com NCM{% else %} (só identidade){% endif %}, {{ src.cache_hits }} do cache, {{ src.rate_limited }} limitados{% if src.next_reset %} · próximo reset {{ src.next_reset.strftime("%d/%m %H:%M") }} UTC{% endif %}
|
||||
· configurado: {{ "sim" if src.configured else "não" }}</small></span></div>{% endfor %}
|
||||
</article>
|
||||
<article class="panel">
|
||||
@@ -27,6 +28,9 @@
|
||||
<small>{% if o.active %}{{ o.records }} registros · {{ o.version }} · {{ o.sha256 }}{% if o.last_update %} · baixada {{ o.last_update.strftime("%d/%m/%Y") }}{% endif %}{% endif %}{% if o.auto_refresh %} · atualização automática diária{% endif %}{% if o.error %} · {{ o.error }}{% endif %}</small></span></div>{% endfor %}
|
||||
</article>
|
||||
</section>
|
||||
<section class="panel" id="kb-resumo"><div class="panel-title"><div><h2>Base de conhecimento de produtos</h2><p class="muted">Evidência local, histórica e externa; nada vira verdade fiscal sem regra de promoção.</p></div></div>
|
||||
<div class="coverage"><div class="coverage-item"><strong>{{ kb.products }}</strong><span>Produtos conhecidos</span></div><div class="coverage-item"><strong>{{ kb.verified }}</strong><span>GTINs verificados</span></div><div class="coverage-item"><strong>{{ kb.corroborated }}</strong><span>Corroborados</span></div><div class="coverage-item"><strong>{{ kb.conflicts }}</strong><span>Conflitos</span></div><div class="coverage-item"><strong>{{ kb.hits_today }}</strong><span>Resolvidos pela base hoje</span></div></div>
|
||||
{% for c in kb.recent_conflicts %}<div class="list-row"><span><b>{{ c.gtin }}</b> {{ c.description }}<small>NCMs divergentes, nada é preenchido: {% for x in c.candidates %}{{ x.ncm }} ({{ x.clients|length }} cliente(s), {{ x.documents }} doc.{% if x.providers %}, {{ x.providers|join("/") }}{% endif %}){% if not loop.last %} × {% endif %}{% endfor %}</small></span></div>{% endfor %}</section>
|
||||
<section class="panel" id="enriquecimento-resumo"><div class="panel-title"><div><h2>Enriquecimento de produtos novos</h2><p class="muted">Cobertura medida sobre os produtos novos acompanhados (o catálogo histórico não entra).</p></div></div>
|
||||
{% if enrich_metrics.tracked %}<div class="coverage">{% for key,label in [("GTIN_VALID_RATE","Com GTIN válido"),("KB_HIT_RATE","Resolvidos pela base"),("PRODUCT_FOUND_RATE","Produto identificado"),("NCM_AUTO_FILL_SAFE_RATE","NCM preenchível com segurança"),("AMBIGUOUS_RATE","Ambíguos"),("CONFLICT_RATE","Em conflito")] %}<div class="coverage-item"><strong>{{ enrich_metrics[key] if enrich_metrics[key] is not none else "—" }}{% if enrich_metrics[key] is not none %}%{% endif %}</strong><span>{{ label }}</span></div>{% endfor %}</div>{% else %}<p class="empty-state">Nenhum produto novo acompanhado ainda.</p>{% endif %}</section>
|
||||
<section class="split overview-split">
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Historical evidence -> KB: temporal validity, lineage independence, conflicts, promotion audit, idempotent import. No provider is ever contacted."""
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from central.app.enrichment import history, kb
|
||||
from central.app.enrichment.resolver import resolve
|
||||
from central.app.enrichment.quota import ProviderGateway
|
||||
from central.app.models import KbKnowledge, KbProduct, KbPromotionAudit
|
||||
from central.tests.test_product_enrichment_resolver import COSMOS, Fake, G1, G2, G3, env, gtin, seed # noqa: F401
|
||||
|
||||
G = "0" + G1 # 14-digit key
|
||||
|
||||
|
||||
def staging(client, gtin13, ncm, *, master=True, years=None, nfe=0, desc="REFRIGERANTE COLA LATA 350ML", code="1"):
|
||||
return {"client": client, "master": [{"code": code, "desc": desc, "gtin": gtin13, "codbarra": "", "ncm": ncm if master else "", "cest": ""}],
|
||||
"nfce": [{"code": code, "year": y, "ncm": ncm, "cest": "", "n": n} for y, n in (years or {}).items()], "nfe": [{"code": code, "codbarra": gtin13, "ncm": ncm, "desc": desc, "n": nfe}] if nfe else [], "barcodes": []}
|
||||
|
||||
|
||||
def run(session, stagings, dry=False):
|
||||
obs = history.build_observations(stagings)
|
||||
cur = history.official_sets(session)
|
||||
return history.apply(session, obs, lambda v: v in cur, dry_run=dry)
|
||||
|
||||
|
||||
def status(session, key=G):
|
||||
p = session.scalar(select(KbProduct).where(KbProduct.gtin == key))
|
||||
cur = history.official_sets(session)
|
||||
return kb.evaluate(session, p, "ncm", lambda v: v in cur)
|
||||
|
||||
|
||||
def test_single_client_multi_year_history_is_corroborated_never_verified(env):
|
||||
r = run(env, [staging("A", G1, "22021000", years={2023: 300, 2024: 700, 2025: 900})])
|
||||
assert r["by_status"] == {"CORROBORATED": 1} and r["WOULD_VERIFY"] == 0
|
||||
assert env.scalars(select(KbPromotionAudit)).all() == []
|
||||
|
||||
|
||||
def test_master_plus_documents_of_same_erp_are_one_family(env):
|
||||
run(env, [staging("A", G1, "22021000", master=True, years={2025: 5})]) # few docs, one year: weak
|
||||
assert status(env)["status"] == kb.OBSERVED and status(env)["families"] == []
|
||||
|
||||
|
||||
def test_two_clients_corroborate_but_three_clients_alone_do_not_verify(env):
|
||||
r = run(env, [staging(c, G1, "22021000", years={2024: 2}) for c in ("A", "B", "C")])
|
||||
assert r["by_status"] == {"CORROBORATED": 1} and r["multi_client"] == 1 and r["WOULD_VERIFY"] == 0
|
||||
|
||||
|
||||
def test_history_plus_independent_provider_verifies_and_is_audited(env):
|
||||
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60})])
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
|
||||
ev = status(env)
|
||||
assert ev["status"] == kb.VERIFIED_KB and set(ev["families"]) == {"PROVIDER:oscbr", "CLIENT_DATA"}
|
||||
audit = env.scalars(select(KbPromotionAudit)).one()
|
||||
assert audit.new_status == "VERIFIED_KB" and audit.rule_version == kb.RULE_VERSION and audit.official_dataset["sha256"] and audit.sources["documents"] == 110
|
||||
status(env)
|
||||
assert len(env.scalars(select(KbPromotionAudit)).all()) == 1 # no duplicate audit
|
||||
|
||||
|
||||
def test_different_current_ncms_are_a_conflict_not_majority(env):
|
||||
r = run(env, [staging("A", G1, "22021000", years={2024: 900, 2025: 900}), staging("B", G1, "19059090", years={2024: 5, 2025: 5}), staging("C", G1, "22021000", years={2024: 900})])
|
||||
assert r["CONFLICTS"] == 1
|
||||
res = resolve(env, ProviderGateway(env, []), gtin_raw=G1, description="REFRIGERANTE COLA LATA 350ML", current={}, allow_external=False)
|
||||
assert res["state"] == "CONFLICT" and res["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "CONFLICT"
|
||||
|
||||
|
||||
def test_expired_ncm_is_history_not_a_candidate(env):
|
||||
run(env, [staging("A", G1, "11111111", years={2019: 900, 2020: 900})]) # expired in the fixture dataset
|
||||
ev = status(env)
|
||||
assert ev["value"] is None and ev["status"] == kb.OBSERVED
|
||||
row = env.scalar(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm"))
|
||||
assert row.evidence["info"]["historical_state"] == "HISTORICAL_EXPIRED" and row.status == kb.STALE
|
||||
|
||||
|
||||
def test_ncm_changed_over_time_prefers_current(env):
|
||||
run(env, [staging("A", G1, "11111111", years={2019: 900, 2020: 900}), staging("A", G1, "22021000", master=False, years={2024: 50, 2025: 60})])
|
||||
ev = status(env)
|
||||
assert ev["value"] == "22021000" and ev["status"] == kb.CORROBORATED
|
||||
|
||||
|
||||
def test_stale_ncm_with_equal_recency_does_not_count_as_stable(env):
|
||||
run(env, [staging("A", G1, "22021000", master=False, years={2024: 50, 2025: 60}), staging("B", G1, "11111111", master=False, years={2025: 500})])
|
||||
assert status(env)["status"] in (kb.OBSERVED,) # the old code is as recent: not stable, so not even corroborated
|
||||
|
||||
|
||||
def test_import_is_idempotent_and_dry_run_writes_nothing(env):
|
||||
st = [staging("A", G1, "22021000", years={2023: 300, 2024: 700}), staging("B", G3, "19059090", desc="BISCOITO RECHEADO 130G", code="9", years={2024: 30, 2025: 30})]
|
||||
dry = run(env, st, dry=True)
|
||||
assert env.scalars(select(KbProduct)).all() == []
|
||||
a, b = run(env, st), run(env, st)
|
||||
assert a["by_status"] == b["by_status"] == dry["by_status"]
|
||||
assert len(env.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm")).all()) == 2
|
||||
row = env.scalar(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.value == "22021000"))
|
||||
assert row.evidence["lineages"]["A"]["docs"] == 1000 # replaced, not doubled
|
||||
|
||||
|
||||
def test_aliases_and_no_company_tax_data_stored(env):
|
||||
run(env, [staging("A", G1, "22021000", years={2024: 30, 2025: 30}, desc="COCA COLA LT 350"), staging("B", G1, "22021000", master=False, desc="REFRIG COCA-COLA 350ML", code="2", years={2025: 5})])
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
aliases = {r.value for r in env.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == p.id, KbKnowledge.knowledge_key == "alias"))}
|
||||
assert len(aliases) >= 2
|
||||
blob = repr([r.evidence for r in env.scalars(select(KbKnowledge))]).upper()
|
||||
assert "CFOP" not in blob and "CSOSN" not in blob and "CRT" not in blob
|
||||
|
||||
|
||||
def test_kb_hit_makes_zero_external_requests(env):
|
||||
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60})])
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
|
||||
env.commit()
|
||||
spy = Fake("cosmos", COSMOS(), True)
|
||||
r = resolve(env, ProviderGateway(env, [spy]), gtin_raw=G1, description="REFRIGERANTE COLA LATA 350ML", current={})
|
||||
assert r["state"] == "EXACT_GTIN_VERIFIED" and spy.calls == 0 and r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "AUTO_FILL_SAFE"
|
||||
|
||||
|
||||
def test_revalidation_marks_stale_review_and_keeps_knowledge(env):
|
||||
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60})])
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
|
||||
assert status(env)["status"] == kb.VERIFIED_KB
|
||||
out = kb.revalidate(env, lambda v: True)
|
||||
assert out["stale"] == 0 and status(env)["status"] == kb.VERIFIED_KB # NCM still vigente: stays valid
|
||||
out = kb.revalidate(env, lambda v: False) # the official dataset dropped it
|
||||
row = env.scalar(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm"))
|
||||
assert out["stale"] == 1 and row.status == kb.STALE and row.evidence["info"]["stale_reason"] == "STALE_REVIEW_REQUIRED"
|
||||
assert [a.new_status for a in env.scalars(select(KbPromotionAudit)).all()] == ["VERIFIED_KB", "STALE"]
|
||||
|
||||
|
||||
def test_provider_disagreeing_with_verified_kb_creates_conflict_not_overwrite(env):
|
||||
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60})])
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
|
||||
assert status(env)["status"] == kb.VERIFIED_KB
|
||||
kb.add_evidence(env, p, "ncm", "19059090", "provider:cosmos", family="PROVIDER:cosmos")
|
||||
ev = status(env)
|
||||
assert ev["status"] == kb.CONFLICT and ev["value"] is None # write blocked until a human resolves it
|
||||
|
||||
|
||||
def test_aliases_from_history_enable_name_only_match_and_generic_names_stay_ambiguous(env):
|
||||
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60}, desc="COCA-COLA ORIGINAL LATA 350 ML")])
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
|
||||
env.commit()
|
||||
r = resolve(env, ProviderGateway(env, []), gtin_raw="", description="COCA COLA LATA 350ML", current={})
|
||||
assert r["state"] == "EXACT_KB_NAME_VERIFIED" and r["fields"]["CLASSIFICACAO_FISCAL"]["value"] == "22021000"
|
||||
g = resolve(env, ProviderGateway(env, []), gtin_raw="", description="REFRIGERANTE", current={})
|
||||
assert g["state"] == "AMBIGUOUS" and all(f["status"] != "AUTO_FILL_SAFE" for f in g["fields"].values())
|
||||
@@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down():
|
||||
|
||||
def test_version_endpoint_reports_release_and_fiscal_versions():
|
||||
body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
|
||||
assert body["version"] == "1.3.1" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
||||
assert body["version"] == "1.3.2" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
||||
|
||||
|
||||
def test_production_refuses_the_dev_token(monkeypatch):
|
||||
|
||||
@@ -200,3 +200,18 @@ def test_provider_panel_shows_every_state_and_column(monkeypatch):
|
||||
page = client.get("/").text
|
||||
for label in ("Limite atingido", "Instável", "Credencial recusada", "Indisponível", "Não configurado", "100 / 100", "restam 0", "renova", "último sucesso", "último erro", "140 ms", "cache 30%"):
|
||||
assert label in page, label
|
||||
|
||||
|
||||
def test_dashboard_kb_panel_and_provider_daily_columns(monkeypatch):
|
||||
from central.app.enrichment import kb as kbm
|
||||
from central.app.models import KbProduct
|
||||
client, agent_id, _ = _panel_client()
|
||||
with client._session_factory() as s:
|
||||
p = kbm.upsert_product(s, "07891000100103", "COLA LATA 350ML")
|
||||
kbm.add_evidence(s, p, "ncm", "22021000", "c", family=kbm.CLIENT_FAMILY, client_label="A", lineage={"docs": 900, "years": {"2024": 900}})
|
||||
kbm.add_evidence(s, p, "ncm", "19059090", "c", family=kbm.CLIENT_FAMILY, client_label="B", lineage={"docs": 30, "years": {"2025": 30}})
|
||||
kbm.evaluate(s, p, "ncm", lambda v: True)
|
||||
s.commit()
|
||||
page = client.get("/").text
|
||||
assert "Base de conhecimento de produtos" in page and "NCMs divergentes" in page and "22021000" in page and "19059090" in page
|
||||
assert "achados" in page and "próximo reset" in page or "hoje:" in page
|
||||
|
||||
@@ -10,7 +10,7 @@ from sqlalchemy.pool import StaticPool
|
||||
|
||||
from central.app.enrichment import kb
|
||||
from central.app.enrichment.gtin import check_digit_ok
|
||||
from central.app.enrichment.providers import Oscbr, Cosmos, OpenFoodFacts, Provider, ProviderError, ProviderProduct, UpcItemDb, build_providers
|
||||
from central.app.enrichment.providers import Kodebar, Oscbr, Cosmos, OpenFoodFacts, Provider, ProviderError, ProviderProduct, UpcItemDb, build_providers
|
||||
from central.app.enrichment.quota import ProviderGateway
|
||||
from central.app.enrichment.resolver import resolve
|
||||
from central.app.models import (Base, CestEntry, CestNcmEntry, Dataset, GtinLookupCache, KbKnowledge, NcmEntry, ProviderUsageDaily, TipiEntry)
|
||||
@@ -178,9 +178,9 @@ def test_client_consensus_needs_two_distinct_clients(env):
|
||||
p = kb.upsert_product(env, "0" + G1, "X")
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "client", family=kb.CLIENT_FAMILY, client_label="A")
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "client", family=kb.CLIENT_FAMILY, client_label="A")
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.EVIDENCE_ONLY
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.OBSERVED
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "client", family=kb.CLIENT_FAMILY, client_label="B")
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.EVIDENCE_ONLY # consensus alone is one family, never authority
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.CORROBORATED # consensus alone is one family, never authority
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider", family="PROVIDER:cosmos")
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.VERIFIED_KB
|
||||
|
||||
@@ -223,7 +223,7 @@ def test_cosmos_without_token_is_not_configured_and_costs_nothing(env):
|
||||
|
||||
def test_paid_provider_guard():
|
||||
class S: allow_paid_providers = False; cosmos_token = None
|
||||
assert [p.name for p in build_providers(S())] == ["cosmos", "oscbr", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"]
|
||||
assert [p.name for p in build_providers(S())] == ["cosmos", "oscbr", "kodebar", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"]
|
||||
|
||||
|
||||
# ---- provider contract tests against recorded fixtures (httpx.MockTransport, no network) ----------------------------------------------------------
|
||||
@@ -292,3 +292,24 @@ def test_oscbr_optional_and_contract():
|
||||
with pytest.raises(ProviderError) as e:
|
||||
bad.lookup_gtin(G1)
|
||||
assert e.value.kind == "AUTH_ERROR"
|
||||
|
||||
|
||||
def test_kodebar_identity_only_optional_and_contract():
|
||||
assert not Kodebar(_client(lambda r: pytest.fail("no call expected"))).configured
|
||||
def handler(request):
|
||||
assert request.headers["x-api-key"] == "k" and request.url.params["gtin"] == G1
|
||||
return httpx.Response(200, json={"gtin": G1, "nome": "COCA COLA 350ML", "marca": "Coca", "thumbnail": "u", "barcode_url": "b", "qr_code_url": "q"})
|
||||
p = Kodebar(_client(handler), "k").lookup_gtin(G1)
|
||||
assert p.found and p.title == "COCA COLA 350ML" and p.ncm is None and p.cest is None # identity only, never NCM
|
||||
assert not Kodebar(_client(lambda r: httpx.Response(200, json={})), "k").lookup_gtin(G1).found
|
||||
with pytest.raises(ProviderError) as e:
|
||||
Kodebar(_client(lambda r: httpx.Response(429)), "k").lookup_gtin(G1)
|
||||
assert e.value.kind == "RATE_LIMITED"
|
||||
|
||||
|
||||
def test_backfill_respects_operational_reserve(env):
|
||||
fake = Fake("cosmos", COSMOS(), True, daily_limit=10)
|
||||
gw = ProviderGateway(env, [fake])
|
||||
outs = [gw.lookup(fake, gtin(f"78970000{i:04d}"), "BACKFILL", 20).state for i in range(10)]
|
||||
assert outs.count("SKIPPED") == 2 and fake.calls == 8 # 20% of 10 kept for new products
|
||||
assert gw.lookup(fake, gtin("789700009999"), "OPERATIONAL").state in ("FOUND", "NOT_FOUND")
|
||||
|
||||
Reference in New Issue
Block a user