feat: coverage V2 - historical evidence miner, KB evidence model + promotion audit + revalidation, Kodebar/OSCBR limits, KB/provider panel, seed import; 1.3.2 (Hub only)

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-26 08:14:57 -03:00
parent 19e145ea47
commit ac1ecccc50
41 changed files with 1418 additions and 74 deletions
@@ -81,6 +81,8 @@ class Settings:
    cosmos_token: str = os.getenv("COSMOS_TOKEN", "")          # optional; without it Cosmos is NOT_CONFIGURED (not an error)
    oscbr_user: str = os.getenv("OSCBR_USER", "")              # optional free-plan account of gtin.rscsistemas.com.br (never in logs/UI)
    oscbr_password: str = os.getenv("OSCBR_PASSWORD", "")
+    kodebar_api_key: str = os.getenv("KODEBAR_API_KEY", "")     # optional free-plan key (identity only); never in logs/UI
+    provider_operational_reserve_percent: int = int(os.getenv("PROVIDER_OPERATIONAL_RESERVE_PERCENT", "20"))
    allow_paid_providers: bool = os.getenv("ALLOW_PAID_PROVIDERS", "0") == "1"
    official_refresh_enabled: bool = os.getenv("OFFICIAL_REFRESH_ENABLED", "1") == "1"
    enrichment_time_budget_seconds: float = float(os.getenv("ENRICHMENT_TIME_BUDGET_SECONDS", "4.5"))
@@ -46,6 +46,12 @@ def refresh_ncm(session, fetch=_fetch, url: str | None = None) -> SourceRefreshR
                raise ValueError(f"implausible record count {len(parsed.entries)} vs active {active.record_count}")
            DatasetService(session).import_payload(parsed, url, datetime.now(timezone.utc))      # atomic: one transaction, old ACTIVE superseded only at the end
            run.status = "UPDATED"
+            try:                                                                    # the KB is never deleted: only flagged STALE_REVIEW_REQUIRED when its NCM left the official dataset
+                from ..enrichment import history, kb
+                current = history.official_sets(session)
+                run.error = f"kb_revalidated: {kb.revalidate(session, lambda v: v in current)}"[:300]
+            except Exception as error:  # noqa: BLE001 - revalidation trouble must not undo a valid dataset promotion
+                run.error = f"kb_revalidate_failed: {type(error).__name__}"[:300]
    except Exception as error:                                                     # keep the last valid dataset
        session.rollback()
        run = session.get(SourceRefreshRun, run.id)
@@ -0,0 +1,196 @@
+"""Historical ERP evidence (product master + NFC-e/NF-e items) -> central KB evidence. Local data only: NO provider is contacted here.
+
+Evidence model
+- One 'lineage' per client per (GTIN, value): master-data flag, fiscal-document count and documents per year. Master data and the fiscal documents of the SAME ERP are one family (CLIENT_DATA):
+  the documents were issued from that very master, so they are never counted as two independent sources.
+- CLIENT_DATA alone can reach CORROBORATED (>= 20 documents over >= 2 years, or >= 2 distinct clients) but never VERIFIED_KB: that needs an independent NCM-bearing source (kb.evaluate).
+- NCM validity is judged against the ACTIVE official dataset today: CURRENT_VALID / HISTORICAL_EXPIRED / HISTORICAL_INVALID; a different NCM at least as recent -> HISTORICAL_CHANGED; two current values -> CONFLICT.
+- Company-specific tax data (CRT, CFOP, CSOSN, ...) is never read nor stored; only GTIN, descriptions/aliases, NCM, CEST and the source tenant label.
+Idempotent: a client's lineage for a (GTIN, value) is REPLACED on every import, never accumulated.
+"""
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+
+from sqlalchemy import select
+
+from ..models import KbKnowledge, KbProduct
+from . import kb, textnorm
+from .gtin import normalize_gtin
+
+
+def _digits(v) -> str:
+    return re.sub(r"\D", "", str(v or ""))
+
+
+def build_observations(stagings: list[dict]) -> dict:
+    """stagings: miner output per client. Returns {"gtins": {key: {...}}, "stats": {...}}."""
+    gtins: dict[str, dict] = {}
+    stats = defaultdict(int)
+    for st in stagings:
+        client = st["client"]
+        product_gtin: dict[str, set] = defaultdict(set)
+        master_gtin: dict[str, str] = {}
+        master_desc: dict[str, str] = {}
+        for m in st["master"]:
+            stats["master_rows"] += 1
+            master_desc[m["code"]] = m["desc"]
+            for raw in (m["gtin"], m["codbarra"]):
+                g = normalize_gtin(raw)
+                if g["status"] == "VALID_GTIN":
+                    master_gtin.setdefault(m["code"], g["key"])
+                    break
+        # product<->EAN links seen in documents/movements are used ONLY when the master has none, and only when unambiguous
+        seen: dict[str, set] = defaultdict(set)
+        for row in st.get("nfe", []):
+            g = normalize_gtin(row["codbarra"])
+            if g["status"] == "VALID_GTIN":
+                seen[row["code"]].add(g["key"])
+        for row in st.get("barcodes", []):
+            g = normalize_gtin(row["codbarra"])
+            if g["status"] == "VALID_GTIN":
+                seen[row["code"]].add(g["key"])
+        link: dict[str, str] = dict(master_gtin)
+        for code, keys in seen.items():
+            if code in link:
+                continue
+            if len(keys) == 1:
+                link[code] = next(iter(keys))
+                stats["links_from_documents"] += 1
+            else:
+                stats["ambiguous_product_links_skipped"] += 1
+
+        def slot(key):
+            return gtins.setdefault(key, {"aliases": set(), "ncm": defaultdict(lambda: defaultdict(lambda: {"master": 0, "docs": 0, "years": {}})), "cest": defaultdict(lambda: defaultdict(lambda: {"master": 0, "docs": 0, "years": {}})), "clients": set()})
+        for m in st["master"]:
+            key = link.get(m["code"])
+            if not key:
+                continue
+            s = slot(key)
+            s["clients"].add(client)
+            if m["desc"]:
+                s["aliases"].add(textnorm.normalized(m["desc"]))
+            n = _digits(m["ncm"])
+            if len(n) == 8:
+                s["ncm"][n][client]["master"] = 1
+            c = _digits(m["cest"])
+            if len(c) == 7:
+                s["cest"][c][client]["master"] = 1
+        for row in st.get("nfce", []):
+            key = link.get(row["code"])
+            stats["nfce_groups"] += 1
+            if not key:
+                stats["nfce_groups_without_gtin"] += 1
+                continue
+            s = slot(key)
+            n = _digits(row["ncm"])
+            if len(n) == 8:
+                lin = s["ncm"][n][client]
+                lin["docs"] += row["n"]
+                if row["year"]:
+                    lin["years"][str(row["year"])] = lin["years"].get(str(row["year"]), 0) + row["n"]
+                stats["nfce_documents"] += row["n"]
+            c = _digits(row["cest"])
+            if len(c) == 7:
+                lin = s["cest"][c][client]
+                lin["docs"] += row["n"]
+                if row["year"]:
+                    lin["years"][str(row["year"])] = lin["years"].get(str(row["year"]), 0) + row["n"]
+        for row in st.get("nfe", []):
+            key = link.get(row["code"])
+            if not key:
+                continue
+            s = slot(key)
+            if row["desc"]:
+                s["aliases"].add(textnorm.normalized(row["desc"]))
+            n = _digits(row["ncm"])
+            if len(n) == 8:
+                s["ncm"][n][client]["docs"] += row["n"]
+                stats["nfe_documents"] += row["n"]
+    stats["gtins"] = len(gtins)
+    return {"gtins": gtins, "stats": dict(stats)}
+
+
+def apply(session, observations: dict, is_official, *, dry_run: bool = True, chunk: int = 500) -> dict:
+    """Bulk, idempotent import into kb_products / kb_knowledge. dry_run=True computes every status and rolls back."""
+    keys = sorted(observations["gtins"])
+    report = {"GTINS_FOUND": len(keys), "OBSERVATIONS": 0, "by_status": defaultdict(int), "CONFLICTS": 0, "WOULD_VERIFY": 0, "with_current_valid_ncm": 0, "stable_across_years": 0, "changed_over_time": 0,
+              "with_historical_ncm": 0, "multi_client": 0}
+    nested = session.begin_nested()
+    try:
+        for start in range(0, len(keys), chunk):
+            part = keys[start:start + chunk]
+            existing = {p.gtin: p for p in session.scalars(select(KbProduct).where(KbProduct.gtin.in_(part)))}
+            for key in part:
+                obs = observations["gtins"][key]
+                product = existing.get(key)
+                if product is None:
+                    alias = next(iter(sorted(obs["aliases"])), "")
+                    product = kb.upsert_product(session, key, alias)
+                    existing[key] = product
+                for a in sorted(obs["aliases"])[:20]:
+                    kb.add_alias(session, product, a)
+                    fp = textnorm.fingerprint(a)
+                    if fp:
+                        kb.add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=kb.OBSERVED)
+                for field, values in (("ncm", obs["ncm"]), ("cest", obs["cest"])):
+                    for value, per_client in values.items():
+                        for client, lin in per_client.items():
+                            kb.add_evidence(session, product, field, value, "client:history", family=kb.CLIENT_FAMILY, client_label=client, lineage=None)
+                            row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == field, KbKnowledge.value == value))
+                            ev = dict(row.evidence or {})
+                            lineages = dict(ev.get("lineages") or {})
+                            lineages[client] = {"master": lin["master"], "docs": lin["docs"], "years": dict(lin["years"])}      # REPLACE: idempotent
+                            row.evidence = {**ev, "lineages": lineages}
+                            report["OBSERVATIONS"] += 1
+            session.flush()
+            for key in part:
+                product = existing[key]
+                ev = kb.evaluate(session, product, "ncm", is_official)
+                report["by_status"][ev["status"]] += 1
+                if ev["status"] == kb.CONFLICT:
+                    report["CONFLICTS"] += 1
+                if ev["status"] == kb.VERIFIED_KB:
+                    report["WOULD_VERIFY"] += 1
+                if ev["value"]:
+                    report["with_current_valid_ncm"] += 1
+                if observations["gtins"][key]["ncm"]:
+                    report["with_historical_ncm"] += 1
+                if len(observations["gtins"][key]["clients"]) >= 2:
+                    report["multi_client"] += 1
+                rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm")))
+                states = {(r.evidence.get("info") or {}).get("historical_state") for r in rows}
+                if "HISTORICAL_STABLE_CURRENT" in states and ev["value"]:
+                    years = set()
+                    for r in rows:
+                        for lin in (r.evidence.get("lineages") or {}).values():
+                            years |= set(lin.get("years") or {})
+                    if len(years) >= 2:
+                        report["stable_across_years"] += 1
+                if "HISTORICAL_CHANGED" in states or ("HISTORICAL_EXPIRED" in states and ev["value"]):
+                    report["changed_over_time"] += 1
+        if dry_run:
+            nested.rollback()
+        else:
+            nested.commit()
+            session.commit()
+    except Exception:
+        nested.rollback()
+        raise
+    report["by_status"] = dict(report["by_status"])
+    return report
+
+
+def official_sets(session):
+    """Precomputed 'vigente today' membership: one query instead of one per value."""
+    from datetime import date
+    from ..models import Dataset, NcmEntry
+    from .resolver import _parse_br
+    today = date.today()
+    current = set()
+    for row in session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")):
+        start, end = _parse_br(row.valid_from), _parse_br(row.valid_to)
+        if (start is None or start <= today) and (end is None or end >= today):
+            current.add(row.code)
+    return current
@@ -12,11 +12,16 @@ from datetime import datetime, timezone

 from sqlalchemy import select

-from ..models import KbKnowledge, KbProduct
+from sqlalchemy import func
+
+from ..models import KbKnowledge, KbProduct, KbPromotionAudit
 from . import textnorm

-UNVERIFIED, EVIDENCE_ONLY, VERIFIED_KB, CONFLICT, STALE = "UNVERIFIED", "EVIDENCE_ONLY", "VERIFIED_KB", "CONFLICT", "STALE"
-CLIENT_FAMILY = "CLIENT_OBSERVATION"
+UNVERIFIED, OBSERVED, CORROBORATED, EXTERNAL_CORROBORATED, VERIFIED_KB, CONFLICT, STALE = "UNVERIFIED", "OBSERVED", "CORROBORATED", "EXTERNAL_CORROBORATED", "VERIFIED_KB", "CONFLICT", "STALE"
+EVIDENCE_ONLY = OBSERVED                      # legacy name
+CLIENT_FAMILY = "CLIENT_DATA"                 # every client-derived lineage (master data + fiscal documents of the same ERP are NOT independent of each other)
+RULE_VERSION = "kb-promotion-v2"
+STRONG_DOCS, STRONG_YEARS = 20, 2


 def _now():
@@ -56,15 +61,29 @@ def upsert_product(session, key: str | None, description, *, brand=None, quantit
    if quantity and not product.package_quantity:
        product.package_quantity = str(quantity)[:32]
    fp = textnorm.fingerprint(description)
+    norm = textnorm.normalized(description)
+    if norm:
+        add_alias(session, product, norm)
    if fp:
-        add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=EVIDENCE_ONLY)
+        add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=OBSERVED)
    return product


-def add_evidence(session, product: KbProduct, key: str, value: str, source: str, *, family: str, client_label: str | None = None, status: str = EVIDENCE_ONLY, info: dict | None = None) -> KbKnowledge:
+def add_alias(session, product: KbProduct, normalized: str, cap: int = 20):
+    """Non-fiscal aliases of the same GTIN (descriptions seen over the years). Helps name matching; never authority for a fiscal write."""
+    if not normalized or session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "alias", KbKnowledge.value == normalized[:200])):
+        return
+    if (session.scalar(select(func.count()).select_from(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "alias")) or 0) >= cap:
+        return
+    session.add(KbKnowledge(kb_product_id=product.id, knowledge_key="alias", value=normalized[:200], source="ALIAS", status=OBSERVED, evidence={"families": {}, "clients": [], "info": {}}))
+
+
+def add_evidence(session, product: KbProduct, key: str, value: str, source: str, *, family: str, client_label: str | None = None, status: str = OBSERVED, info: dict | None = None,
+                 lineage: dict | None = None) -> KbKnowledge:
+    """lineage (client data only): {"master": 0/1, "docs": n, "years": {"2024": n}} of ONE client for this (GTIN, value); merged, never double counted."""
    row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key, KbKnowledge.value == value))
    if row is None:
-        row = KbKnowledge(kb_product_id=product.id, knowledge_key=key, value=value, source=source[:32], status=status, evidence={"families": {}, "clients": [], "info": info or {}})
+        row = KbKnowledge(kb_product_id=product.id, knowledge_key=key, value=value, source=source[:32], status=status, evidence={"families": {}, "clients": [], "lineages": {}, "info": info or {}})
        session.add(row)
        session.flush()
    evidence = dict(row.evidence or {})
@@ -73,48 +92,128 @@ def add_evidence(session, product: KbProduct, key: str, value: str, source: str,
    clients = list(evidence.get("clients") or [])
    if client_label and client_label not in clients:
        clients.append(client_label)
-    row.evidence = {**evidence, "families": families, "clients": clients, "info": {**(evidence.get("info") or {}), **(info or {})}}
+    lineages = dict(evidence.get("lineages") or {})
+    if client_label and lineage:
+        cur = dict(lineages.get(client_label) or {"master": 0, "docs": 0, "years": {}})
+        cur["master"] = max(int(cur.get("master", 0)), int(lineage.get("master", 0)))
+        cur["docs"] = int(cur.get("docs", 0)) + int(lineage.get("docs", 0))
+        years = dict(cur.get("years") or {})
+        for y, n in (lineage.get("years") or {}).items():
+            years[str(y)] = int(years.get(str(y), 0)) + int(n)
+        cur["years"] = years
+        lineages[client_label] = cur
+    row.evidence = {**evidence, "families": families, "clients": clients, "lineages": lineages, "info": {**(evidence.get("info") or {}), **(info or {})}}
    return row


+def _lineage_summary(rows):
+    clients, docs, years = set(), 0, {}
+    for r in rows:
+        ev = r.evidence or {}
+        for c, lin in (ev.get("lineages") or {}).items():
+            clients.add(c)
+            docs += int(lin.get("docs", 0))
+            for y, n in (lin.get("years") or {}).items():
+                years[y] = years.get(y, 0) + int(n)
+        for c in ev.get("clients") or []:
+            clients.add(c)
+    return {"clients": sorted(clients), "docs": docs, "years": sorted(years), "last_year": max(int(y) for y in years) if years else None}
+
+
 def independent_count(rows) -> tuple[int, list[str]]:
-    families = {}
-    clients: set[str] = set()
-    for row in rows:
-        ev = row.evidence or {}
-        for fam in (ev.get("families") or {}):
-            if fam.startswith("PROVIDER:"):
-                families[fam] = True
-            elif fam == CLIENT_FAMILY:
-                clients.update(ev.get("clients") or [])
-    names = sorted(families)
-    if len(clients) >= 2:
-        names.append(f"{CLIENT_FAMILY}(x{len(clients)})")
+    """Independent families: each NCM-bearing provider = 1; ALL client-derived data = 1 and only when STRONG (>= STRONG_DOCS documents over >= STRONG_YEARS years, or >= 2 distinct clients)."""
+    names = sorted({f for r in rows for f in ((r.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:")})
+    summary = _lineage_summary(rows)
+    has_client = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in rows)
+    if has_client and (len(summary["clients"]) >= 2 or (summary["docs"] >= STRONG_DOCS and len(summary["years"]) >= STRONG_YEARS)):
+        names.append(CLIENT_FAMILY)
    return len(names), names


+def _official_period(session, value):
+    """CURRENT_VALID / HISTORICAL_EXPIRED (exists in the dataset but not vigente today) / HISTORICAL_INVALID (never in the dataset)."""
+    from .resolver import ncm_exists_any
+    return "CURRENT_VALID" if ncm_exists_any(session, value, current=True) else ("HISTORICAL_EXPIRED" if ncm_exists_any(session, value, current=False) else "HISTORICAL_INVALID")
+
+
+def _mark(row, state):
+    row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "historical_state": state}}
+
+
 def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
-    """Recomputes and PERSISTS the status of one field. Returns {value, status, families, candidates, rejected}."""
+    """Recomputes and PERSISTS the status of one field. Never 'majority wins': two current-valid values => CONFLICT."""
    rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key)))
    if not rows:
-        return {"value": None, "status": UNVERIFIED, "families": [], "candidates": [], "rejected": []}
+        return {"value": None, "status": UNVERIFIED, "families": [], "candidates": [], "rejected": [], "detail": {}}
    valid = [r for r in rows if is_official(r.value)]
    rejected = [r.value for r in rows if not is_official(r.value)]
+    for r in rows:
+        if r not in valid:
+            state = _official_period(session, r.value) if key == "ncm" else "HISTORICAL_INVALID"
+            r.status = STALE if state == "HISTORICAL_EXPIRED" else OBSERVED
+            _mark(r, state)
    values = sorted({r.value for r in valid})
    if len(values) > 1:
-        for r in rows:
+        for r in valid:
            r.status = CONFLICT
-        return {"value": None, "status": CONFLICT, "families": [], "candidates": values, "rejected": rejected}
+            _mark(r, "HISTORICAL_CONFLICT")
+        return {"value": None, "status": CONFLICT, "families": [], "candidates": values, "rejected": rejected, "detail": {"clients": _lineage_summary(valid)["clients"]}}
    if not values:
-        for r in rows:
-            r.status = EVIDENCE_ONLY
-        return {"value": None, "status": EVIDENCE_ONLY, "families": [], "candidates": [], "rejected": rejected}
+        return {"value": None, "status": OBSERVED, "families": [], "candidates": [], "rejected": rejected, "detail": {}}
    chosen = [r for r in valid if r.value == values[0]]
+    old = chosen[0].status
    count, names = independent_count(chosen)
+    summary = _lineage_summary(chosen)
+    other_last = _lineage_summary([r for r in rows if r not in chosen])["last_year"]
+    superseded = other_last is not None and summary["last_year"] is not None and other_last >= summary["last_year"]      # a different (expired) NCM is at least as recent: not stable
+    if superseded and CLIENT_FAMILY in names:
+        names.remove(CLIENT_FAMILY)
+        count -= 1
+    providers = [n for n in names if n.startswith("PROVIDER:")]
    already = any(r.status == VERIFIED_KB and r.validated_at for r in chosen)
-    status = VERIFIED_KB if (count >= 2 or already) else EVIDENCE_ONLY
-    for r in rows:
-        r.status = status if r.value == values[0] else EVIDENCE_ONLY
-        if status == VERIFIED_KB and r.value == values[0] and not r.validated_at:
+    has_client_data = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen)
+    if count >= 2 or already:
+        status = VERIFIED_KB
+    elif providers and has_client_data:
+        status = EXTERNAL_CORROBORATED
+    elif CLIENT_FAMILY in names:
+        status = CORROBORATED
+    else:
+        status = OBSERVED
+    for r in chosen:
+        r.status = status
+        _mark(r, "HISTORICAL_CHANGED" if superseded else "HISTORICAL_STABLE_CURRENT")
+        if status == VERIFIED_KB and not r.validated_at:
            r.validated_at = _now()
-    return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected}
+    if status == VERIFIED_KB and old != VERIFIED_KB and key == "ncm":
+        from .resolver import dataset_versions
+        session.add(KbPromotionAudit(kb_product_id=product.id, gtin=product.gtin, field=key, value=values[0], old_status=old, new_status=status, families=names,
+                                     sources={"clients": summary["clients"], "documents": summary["docs"], "years": summary["years"], "providers": providers},
+                                     official_dataset=dataset_versions(session).get("NCM", {}), rule_version=RULE_VERSION))
+    return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected, "detail": summary}
+
+
+def revalidate(session, is_official) -> dict:
+    """After the OFFICIAL NCM dataset changed: never delete knowledge. Rows whose NCM is no longer vigente become STALE (STALE_REVIEW_REQUIRED, audited when they were VERIFIED_KB);
+    rows still vigente keep their status (VERIFIED_KB stays valid while the NCM is vigente)."""
+    from .resolver import dataset_versions
+    stale = still = 0
+    versions = dataset_versions(session).get("NCM", {})
+    for row in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm")):
+        if is_official(row.value):
+            still += 1
+            if row.status == STALE:
+                row.status = OBSERVED                                    # became vigente again: needs a fresh evaluate() to regain its level
+            continue
+        if row.status != STALE:
+            was = row.status
+            row.status = STALE
+            _mark(row, "HISTORICAL_EXPIRED")
+            row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "stale_reason": "STALE_REVIEW_REQUIRED"}}
+            stale += 1
+            if was == VERIFIED_KB:
+                product = session.get(KbProduct, row.kb_product_id)
+                session.add(KbPromotionAudit(kb_product_id=row.kb_product_id, gtin=product.gtin if product else None, field="ncm", value=row.value, old_status=was, new_status=STALE,
+                                             families=sorted((row.evidence or {}).get("families") or {}), sources={"reason": "NCM_NO_LONGER_VIGENTE"}, official_dataset=versions, rule_version=RULE_VERSION))
+    session.flush()
+    return {"stale": stale, "still_vigente": still}
@@ -39,6 +39,8 @@ def provider_sources(session, providers=None) -> list[dict]:
            "latency_ms": state.last_latency_ms if state else None,
            "cache_hit_rate": round(100 * usage.cache_hits / lookups) if usage and lookups else None,
            "errors": usage.errors if usage else 0, "rate_limited": usage.rate_limited if usage else 0,
+            "found_today": usage.found if usage else 0, "ncm_found_today": usage.ncm_found if usage else 0, "cache_hits": usage.cache_hits if usage else 0,
+            "next_reset": _aware(state.reset_at) if state and state.reset_at else None, "identity_only": not getattr(provider, "provides_ncm", False),
        })
    return rows

@@ -80,3 +82,24 @@ def coverage_metrics(session) -> dict:
            "NCM_AUTO_FILL_SAFE_RATE": rate(auto_ncm), "NAME_ONLY_RESOLUTION_RATE": rate(states.get("EXACT_KB_NAME_VERIFIED", 0)), "AMBIGUOUS_RATE": rate(states.get("AMBIGUOUS", 0)),
            "CONFLICT_RATE": rate(states.get("CONFLICT", 0)), "PROVIDER_QUOTA_BLOCK_RATE": round(100 * limited / requests, 1) if requests else None, "states": states, "by_status": by,
            "provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}
+
+
+def kb_summary(session) -> dict:
+    """Compact Knowledge Base panel: known products, statuses of the NCM knowledge, hits today, and the latest conflicts with their evidence (no dashboard of its own)."""
+    from ..models import KbKnowledge, KbProduct
+    today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    products = session.scalar(select(func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
+    by = dict(session.execute(select(KbKnowledge.status, func.count(func.distinct(KbKnowledge.kb_product_id))).where(KbKnowledge.knowledge_key == "ncm").group_by(KbKnowledge.status)).all())
+    hits = session.scalar(select(func.coalesce(func.sum(ProviderUsageDaily.kb_hits), 0)).where(ProviderUsageDaily.day == today)) or 0
+    conflicts = []
+    rows = session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.status == "CONFLICT").order_by(KbKnowledge.id.desc()).limit(60)).all()
+    grouped = {}
+    for r in rows:
+        grouped.setdefault(r.kb_product_id, []).append(r)
+    for pid, items in list(grouped.items())[:5]:
+        prod = session.get(KbProduct, pid)
+        conflicts.append({"gtin": (prod.gtin or "")[-13:], "description": (prod.normalized_description or "")[:60],
+                          "candidates": [{"ncm": i.value, "clients": sorted({c for c in (i.evidence or {}).get("clients", [])}), "documents": sum(int(l.get("docs", 0)) for l in ((i.evidence or {}).get("lineages") or {}).values()),
+                                          "providers": sorted(f for f in ((i.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:"))} for i in items]})
+    return {"products": products, "verified": by.get("VERIFIED_KB", 0), "corroborated": by.get("CORROBORATED", 0) + by.get("EXTERNAL_CORROBORATED", 0), "observed": by.get("OBSERVED", 0),
+            "conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts}
@@ -226,10 +226,11 @@ class Cosmos(Provider):


 class Oscbr(Provider):
-    """OSCBR / RSC Sistemas GTIN API (https://gtin.rscsistemas.com.br). Free plan (no card): 20 req/min; account created BY THE OPERATOR, credentials only in the Hub env
+    """OSCBR / RSC Sistemas GTIN API (https://gtin.rscsistemas.com.br). Free plan (no card): 50 calls/day, 5/min; account created BY THE OPERATOR, credentials only in the Hub env
    (OSCBR_USER / OSCBR_PASSWORD). Bearer token lasts 1 h and is renewed automatically. Optional: NOT_CONFIGURED without credentials (no call is made)."""
    name = "oscbr"
-    rate_per_minute = 20
+    rate_per_minute = 5          # free plan as currently documented: 50 calls/day, 5/min (conservative: the lower figure wins)
+    daily_limit = 50
    provides_ncm = True
    base = "https://gtin.rscsistemas.com.br"

@@ -287,9 +288,41 @@ class Oscbr(Provider):
                               raw_hash=hashlib.sha256(json.dumps(body, sort_keys=True, ensure_ascii=False).encode()).hexdigest(), latency_ms=getattr(response, "latency_ms", 0), headers=headers)


+class Kodebar(Provider):
+    """Kodebar (Korven Sistemas) - IDENTITY source only (name, brand, thumbnail; NO NCM). Free 'Teste' plan: 50 queries/day, no card, personal API key created by the operator
+    (KODEBAR_API_KEY in the Hub env). Terms (2026-09-26): integration into our own multi-user system and internal caching are allowed; redistribution/resale is not.
+    429 above the quota, no overage billing. NOT_CONFIGURED without a key: no request is made."""
+    name = "kodebar"
+    daily_limit = 50
+    rate_per_minute = 30
+    base = "https://kodebar.korvensistemas.com.br"
+
+    @property
+    def configured(self) -> bool:
+        return bool(self._token)
+
+    def lookup_gtin(self, gtin: str) -> ProviderProduct:
+        if not self.configured:
+            raise ProviderError("AUTH_ERROR", "NOT_CONFIGURED")
+        response = self._get(f"{self.base}/gtin/lookup", params={"gtin": gtin}, headers={"X-API-Key": self._token or "", "User-Agent": USER_AGENT})
+        headers = {k.lower(): v for k, v in response.headers.items()}
+        if response.status_code == 404:
+            return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
+        if response.status_code != 200:
+            raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
+        try:
+            body = response.json()
+        except ValueError as error:
+            raise ProviderError("BAD_RESPONSE", "not JSON") from error
+        if not isinstance(body, dict) or not body.get("nome"):
+            return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
+        return ProviderProduct(self.name, gtin, True, title=body.get("nome"), brand=body.get("marca"), raw_hash=hashlib.sha256(json.dumps({k: body.get(k) for k in ("gtin", "nome", "marca")}, sort_keys=True, ensure_ascii=False).encode()).hexdigest(),
+                               latency_ms=getattr(response, "latency_ms", 0), headers=headers)
+
+
 def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]:
    """Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos then OSCBR (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here."""
-    providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), Oscbr(client, None, getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
+    providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), Oscbr(client, None, getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), Kodebar(client, getattr(settings, "kodebar_api_key", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
    if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False):
        raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)")
    return providers
@@ -79,7 +79,7 @@ class QuotaManager:
        return row

    # ---- decision ----------------------------------------------------------------------------------------------------------------------------------
-    def allow(self, provider: Provider) -> tuple[bool, str | None, datetime | None]:
+    def allow(self, provider: Provider, purpose: str = "OPERATIONAL", reserve_percent: int = 0) -> tuple[bool, str | None, datetime | None]:
        state, now = self.state(provider), self.now()
        if not provider.configured:
            return False, "NOT_CONFIGURED", None
@@ -91,6 +91,12 @@ class QuotaManager:
        if provider.daily_limit and usage.requests >= provider.daily_limit and state.quota_kind != "REMOTE_QUOTA":
            state.status = "RATE_LIMITED"
            return False, "RATE_LIMITED", state.reset_at and _aware(state.reset_at) or next_utc_midnight(now)
+        if purpose == "BACKFILL" and provider.daily_limit:                       # backfill may only use what is left above the operational reserve
+            used = usage.requests
+            if state.quota_kind == "REMOTE_QUOTA" and state.remaining is not None:
+                used = provider.daily_limit - state.remaining
+            if used >= provider.daily_limit * (100 - reserve_percent) / 100:
+                return False, "OPERATIONAL_RESERVE", state.reset_at and _aware(state.reset_at) or next_utc_midnight(now)
        window = (provider.burst[1] if provider.burst else 60)
        limit = (provider.burst[0] if provider.burst else provider.rate_per_minute)
        if limit:
@@ -123,6 +129,10 @@ class QuotaManager:
        usage.success += 1
        if not result.found:
            usage.not_found += 1
+        else:
+            usage.found += 1
+            if result.ncm:
+                usage.ncm_found += 1
        state.consecutive_failures = 0
        state.circuit_open_until = None
        state.status = "ONLINE"
@@ -147,7 +157,7 @@ class QuotaManager:
            usage.rate_limited += 1
            state.status = "RATE_LIMITED"
            wait = timedelta(seconds=error.retry_after) if error.retry_after else None
-            state.circuit_open_until = now + (wait or (state.reset_at and max(timedelta(minutes=5), _aware(state.reset_at) - now) or timedelta(minutes=15)))
+            state.circuit_open_until = now + (wait or (state.reset_at and max(timedelta(minutes=15), _aware(state.reset_at) - now) or timedelta(minutes=60)))
            if state.reset_at is None:
                state.reset_at = state.circuit_open_until
        elif error.kind == "AUTH_ERROR":
@@ -165,6 +175,10 @@ class QuotaManager:
                state.status = "OFFLINE"
        self.session.commit()

+    def record_kb_hit(self):
+        """A resolution answered by the KB alone (no provider asked): counted once per day under the pseudo-provider 'kb'."""
+        self.usage("kb").kb_hits += 1
+
    def record_cache(self, provider_name: str, hit: bool):
        usage = self.usage(provider_name)
        if hit:
@@ -200,7 +214,7 @@ class ProviderGateway:
            row.found, row.data, row.fetched_at, row.expires_at = product.found, data, now, now + ttl
        self.session.commit()

-    def lookup(self, provider: Provider, gtin: str) -> Outcome:
+    def lookup(self, provider: Provider, gtin: str, purpose: str = "OPERATIONAL", reserve_percent: int = 0) -> Outcome:
        hit = self._cached(provider, gtin)
        if hit is not False:
            self.quota.record_cache(provider.name, True)
@@ -216,7 +230,7 @@ class ProviderGateway:
                self.session.commit()
                return Outcome(provider.name, "CACHE_HIT", hit or None, from_cache=True) if hit else Outcome(provider.name, "NOT_FOUND", None, from_cache=True)
            self.quota.record_cache(provider.name, False)
-            ok, reason, retry_at = self.quota.allow(provider)
+            ok, reason, retry_at = self.quota.allow(provider, purpose, reserve_percent)
            if not ok:
                self.session.commit()
                return Outcome(provider.name, "SKIPPED", None, reason, retry_at)
@@ -56,6 +56,16 @@ class _ScopedDatasets(DatasetService):
        return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == self.code)).all()


+def ncm_exists_any(session, code: str, *, current: bool, today: date | None = None) -> bool:
+    """current=True: vigente today; current=False: present in the ACTIVE dataset with any validity window (i.e. known but not vigente)."""
+    code = normalize_code(code)
+    if len(code) != 8:
+        return False
+    if current:
+        return ncm_is_official(session, code, today)
+    return session.scalar(select(NcmEntry.id).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code).limit(1)) is not None
+
+
 def dataset_versions(session) -> dict:
    return {d.kind: {"sha256": (d.sha256 or "")[:12], "version": d.version_label} for d in session.scalars(select(Dataset).where(Dataset.status == "ACTIVE"))}

@@ -77,7 +87,8 @@ def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current:
    elif ncm_eval["value"] and ncm_eval["status"] == kb.VERIFIED_KB:
        out["CLASSIFICACAO_FISCAL"] = _field(AUTO, ncm_eval["value"], "NCM_KB_VERIFIED" if gtin_kb_verified or name_verified else "NCM_TWO_INDEPENDENT_SOURCES", kb.VERIFIED_KB, ncm_eval["families"])
    elif ncm_eval["value"]:
-        out["CLASSIFICACAO_FISCAL"] = _field(SUGGEST, ncm_eval["value"], "NCM_SINGLE_SOURCE", kb.EVIDENCE_ONLY, ncm_eval["families"], "apenas uma fonte independente")
+        rule = {"CORROBORATED": "NCM_CORROBORATED_LOCAL_HISTORY", "EXTERNAL_CORROBORATED": "NCM_EXTERNAL_CORROBORATED"}.get(ncm_eval["status"], "NCM_SINGLE_SOURCE")
+        out["CLASSIFICACAO_FISCAL"] = _field(SUGGEST, ncm_eval["value"], rule, ncm_eval["status"], ncm_eval["families"], "evidência ainda insuficiente para preencher automaticamente")
    else:
        out["CLASSIFICACAO_FISCAL"] = _field(INSUFFICIENT, reason="nenhuma fonte com NCM válido no dataset oficial")
    ncm = out["CLASSIFICACAO_FISCAL"]
@@ -124,14 +135,25 @@ def resolve(session, gateway: ProviderGateway, *, gtin_raw, description, current
        product = kb.find_by_gtin(session, key)
        ev = kb.evaluate(session, product, "ncm", is_official) if product else None
        if ev and ev["status"] == kb.VERIFIED_KB:
+            gateway.quota.record_kb_hit()
            result["state"] = "EXACT_GTIN_VERIFIED"
            result["fields"] = _fiscal_decisions(session, ev, True, current, description or "")
            session.commit()
            return result
+        if ev and ev["status"] == kb.CONFLICT:                                     # local evidence already conflicts: an external answer cannot settle it, so spend no quota
+            result["state"] = "CONFLICT"
+            result["fields"] = _fiscal_decisions(session, ev, False, current, description or "")
+            gateway.quota.record_kb_hit()
+            session.commit()
+            return result
+        locally_known = bool(ev and ev["status"] in (kb.CORROBORATED, kb.EXTERNAL_CORROBORATED))      # identity + NCM candidate already local: only an NCM-bearing source can still add value
        query = g["gtin"] if len(g["gtin"]) in (8, 14) else key[1:]      # providers/caches use the canonical EAN-13 (UPC-12 is zero-padded to it)
        found, pending = [], []
        if allow_external:
            for provider in gateway.providers:
+                if locally_known and not getattr(provider, "provides_ncm", False):
+                    result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "LOCAL_IDENTITY_KNOWN", "from_cache": False})
+                    continue
                if budget_s is not None and time.monotonic() - started > budget_s:
                    result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "TIME_BUDGET", "from_cache": False})
                    pending.append(type("P", (), {"reason": "TIME_BUDGET", "retry_at": None})())
@@ -7,6 +7,7 @@ UNIT_MAP = {"LT": "L", "LTS": "L", "LITRO": "L", "LITROS": "L", "L": "L", "ML":
 ABBREVIATIONS = {"REFRIG": "REFRIGERANTE", "REFRI": "REFRIGERANTE", "CHOC": "CHOCOLATE", "BISC": "BISCOITO", "SAB": "SABONETE", "DET": "DETERGENTE", "SH": "SHAMPOO", "COND": "CONDICIONADOR",
                 "MACARR": "MACARRAO", "MAC": "MACARRAO", "AMAC": "AMACIANTE", "DESOD": "DESODORANTE", "PAP": "PAPEL", "HIG": "HIGIENICO", "ORIG": "ORIGINAL", "TRAD": "TRADICIONAL"}
 STOP = {"DE", "DA", "DO", "DAS", "DOS", "E", "COM", "C", "PARA", "P", "EM", "A", "O"}
+MARKETING = {"ORIGINAL", "TRADICIONAL", "NOVO", "NOVA", "PROMO", "PROMOCAO", "OFERTA"}      # ignored ONLY by the fingerprint; a different pack/flavour still differs
 _QTY = re.compile(r"^(\d+(?:[.,]\d+)?)(ML|L|KG|G|MG|UN|PCT|CX|LT|LTS|GR)$")


@@ -35,7 +36,7 @@ def normalized(description) -> str:


 def fingerprint(description) -> str | None:
-    toks = tokens(description)
+    toks = [t for t in tokens(description) if t not in MARKETING]
    if not toks:
        return None
    return hashlib.sha256(" ".join(sorted(toks)).encode("utf-8")).hexdigest()
@@ -629,7 +629,7 @@ class KbKnowledge(Base):
    dataset_versions: Mapped[dict] = mapped_column(JSON, default=dict)
    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
    validated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
-    status: Mapped[str] = mapped_column(String(16), default="EVIDENCE_ONLY", server_default="EVIDENCE_ONLY", index=True)   # UNVERIFIED | EVIDENCE_ONLY | VERIFIED_KB | CONFLICT | STALE
+    status: Mapped[str] = mapped_column(String(24), default="OBSERVED", server_default="OBSERVED", index=True)   # OBSERVED | CORROBORATED | EXTERNAL_CORROBORATED | VERIFIED_KB | CONFLICT | STALE


 class KbObservation(Base):
@@ -879,6 +879,9 @@ class ProviderUsageDaily(Base):
    timeouts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
    cache_hits: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
    cache_misses: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+    found: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+    ncm_found: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+    kb_hits: Mapped[int] = mapped_column(Integer, default=0, server_default="0")


 class GtinLookupCache(Base):
@@ -907,3 +910,20 @@ class SourceRefreshRun(Base):
    version_label: Mapped[str | None] = mapped_column(String(64), nullable=True)
    record_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
    error: Mapped[str | None] = mapped_column(String(300), nullable=True)
+
+
+class KbPromotionAudit(Base):
+    """Why a GTIN/NCM became (or stopped being) trustworthy: statuses, evidence families, source references and the official dataset hash at that moment."""
+    __tablename__ = "kb_promotion_audits"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
+    gtin: Mapped[str | None] = mapped_column(String(16), nullable=True, index=True)
+    field: Mapped[str] = mapped_column(String(24), default="ncm", server_default="ncm")
+    value: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    old_status: Mapped[str | None] = mapped_column(String(24), nullable=True)
+    new_status: Mapped[str] = mapped_column(String(24))
+    families: Mapped[list] = mapped_column(JSON, default=list)
+    sources: Mapped[dict] = mapped_column(JSON, default=dict)
+    official_dataset: Mapped[dict] = mapped_column(JSON, default=dict)
+    rule_version: Mapped[str] = mapped_column(String(32))
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
 once published — never a second, independent version number. Fiscal semantics
 are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
 DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
-GTI_FISCAL_VERSION = "1.3.1"
+GTI_FISCAL_VERSION = "1.3.2"
 API_COMPAT_MAJOR_MINOR = "1.0"  # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
@@ -22,7 +22,7 @@ from .panel_presentation import format_ui_datetime
 from .models import AdminUser, Agent, AgentEnrollment, AgentRelease, AgentSnapshot, BlockedProduct, FiscalContextEntry, Installation, Job, ProductEnrichment, ProductFieldAudit, RtcCandidate, RtcWriteAudit, Tenant
 from .jobs import create_job
 from .audit import record_audit
-from .enrichment.observability import coverage_metrics, official_sources, provider_sources
+from .enrichment.observability import coverage_metrics, kb_summary, official_sources, provider_sources


 def router(session_factory):
@@ -161,7 +161,7 @@ def router(session_factory):
    def dashboard(request: Request):
        session, auth_data, redirect = page_auth(request)
        if redirect: return redirect
-        try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session)})
+        try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session), "kb": kb_summary(session)})
        finally: session.close()

    @routes.get("/agents", response_class=HTMLResponse)