feat: OSCBR token support, Cosmos out of the active chain, quota-safe OSCBR backfill (dry-run selection), provider env passthrough; 1.3.3 (Hub only)

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

central/app/config.py

@@ -79,6 +79,9 @@ class Settings:
     ai_base_url: str = os.getenv("AI_BASE_URL", "https://api.z.ai/api/paas/v4")
     # ---- Product enrichment (docs/product_enrichment.md): free providers only, tokens live ONLY in the Hub environment ----
     cosmos_token: str = os.getenv("COSMOS_TOKEN", "")          # optional; without it Cosmos is NOT_CONFIGURED (not an error)
+    oscbr_token: str = os.getenv("OSCBR_TOKEN", "")            # preferred: a ready Bearer token instead of the login
+    cosmos_enabled: bool = os.getenv("COSMOS_ENABLED", "0") == "1"   # Cosmos stays out of the active chain unless explicitly enabled
+    oscbr_backfill_enabled: bool = os.getenv("OSCBR_BACKFILL_ENABLED", "0") == "1"   # gradual backfill of CORROBORATED GTINs; OFF until a real benchmark proves value
     oscbr_user: str = os.getenv("OSCBR_USER", "")              # optional free-plan account of gtin.rscsistemas.com.br (never in logs/UI)
     oscbr_password: str = os.getenv("OSCBR_PASSWORD", "")
     kodebar_api_key: str = os.getenv("KODEBAR_API_KEY", "")     # optional free-plan key (identity only); never in logs/UI

central/app/enrichment/backfill.py

@@ -0,0 +1,101 @@
+"""Gradual, quota-safe verification of locally CORROBORATED GTINs by an NCM-bearing free provider (OSCBR). Never spends the operational reserve.
+
+Rules: only CORROBORATED single-NCM GTINs (never VERIFIED_KB, never CONFLICT, never an invalid GTIN); each GTIN is checked once per `RECHECK_DAYS` (next_checked_at kept in the KB evidence, plus the
+provider cache); a provider answer is EVIDENCE only - the promotion/conflict decision is kb.evaluate's (local strong evidence + independent provider NCM + vigente + zero conflict).
+"""
+from __future__ import annotations
+
+import time
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import select
+
+from ..models import KbKnowledge, KbProduct
+from . import kb
+from .gtin import normalize_gtin
+
+RECHECK_DAYS = 30
+
+
+def _now():
+    return datetime.now(timezone.utc)
+
+
+def select_sample(session, is_official, limit: int, provider: str = "oscbr") -> list[dict]:
+    """Priority: multi-client, more documents, more years; current-vigente NCM; zero conflict; not checked recently."""
+    ids = list(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.status == kb.CORROBORATED).distinct()))
+    by_product: dict[int, list] = {}
+    for start in range(0, len(ids), 500):                                        # every NCM row of those products: a second (even weaker) value already disqualifies the GTIN
+        for r in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.kb_product_id.in_(ids[start:start + 500]))):
+            by_product.setdefault(r.kb_product_id, []).append(r)
+    now = _now().isoformat()
+    picks = []
+    for pid, items in by_product.items():
+        valid = [i for i in items if is_official(i.value)]
+        if len(valid) != 1 or valid[0].status != kb.CORROBORATED:
+            continue                                                         # conflicts / no vigente NCM / already verified are not asked
+        row = valid[0]
+        info = (row.evidence or {}).get("info") or {}
+        if info.get(f"{provider}_next_checked_at", "") > now:
+            continue
+        prod = session.get(KbProduct, pid)
+        if not prod or normalize_gtin(prod.gtin[1:] if prod.gtin and prod.gtin.startswith("0") else prod.gtin)["status"] != "VALID_GTIN":
+            continue
+        summary = kb._lineage_summary([row])
+        if any(f.startswith("PROVIDER:" + provider) for f in ((row.evidence or {}).get("families") or {})):
+            continue
+        picks.append({"gtin_key": prod.gtin, "ncm": row.value, "clients": len(summary["clients"]), "docs": summary["docs"], "years": len(summary["years"]), "product_id": pid})
+    picks.sort(key=lambda p: (-p["clients"], -p["docs"], -p["years"], p["gtin_key"]))
+    return picks[:limit]
+
+
+def apply_outcome(session, gtin_key: str, provider: str, outcome, is_official) -> dict:
+    """Turns one gateway Outcome into KB evidence. Returns the per-GTIN measurement record."""
+    product = kb.find_by_gtin(session, gtin_key)
+    rec = {"gtin": gtin_key[-13:] if gtin_key.startswith("0") else gtin_key, "state": outcome.state, "reason": outcome.reason, "found": False, "ncm": None, "cest": None, "description": None, "brand": None,
+           "local_ncm": None, "official_valid": None, "kb_status": None, "external_conflict": False}
+    local_rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm"))) if product else []
+    rec["local_ncm"] = local_rows[0].value if local_rows else None
+    stamp = (_now() + timedelta(days=RECHECK_DAYS)).isoformat()
+    if outcome.state in ("FOUND", "CACHE_HIT") and outcome.product:
+        p = outcome.product
+        rec.update({"found": True, "ncm": p.ncm, "cest": p.cest, "description": p.title, "brand": p.brand})
+        if p.ncm:
+            rec["official_valid"] = bool(is_official(p.ncm))
+            kb.add_evidence(session, product, "ncm", p.ncm, f"provider:{provider}"[:32], family=f"PROVIDER:{provider}", info={"raw_hash": p.raw_hash})
+        if p.cest:
+            kb.add_evidence(session, product, "cest", p.cest, f"provider:{provider}"[:32], family=f"PROVIDER:{provider}", info={"raw_hash": p.raw_hash})
+        if p.brand and not product.brand:
+            product.brand = p.brand
+        if p.title:
+            kb.add_alias(session, product, p.title.upper()[:200])
+    ev = kb.evaluate(session, product, "ncm", is_official)
+    rec["kb_status"] = ev["status"]
+    rec["external_conflict"] = bool(ev["status"] == kb.CONFLICT and rec["ncm"])
+    if outcome.state in ("FOUND", "CACHE_HIT", "NOT_FOUND"):                    # remember the check so it is not repeated (skipped/errors are retried later)
+        for r in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm")):
+            r.evidence = {**(r.evidence or {}), "info": {**((r.evidence or {}).get("info") or {}), f"{provider}_next_checked_at": stamp}}
+    return rec
+
+
+def run(session, gateway, provider, *, limit: int, reserve_percent: int, is_official, dry_run: bool = False, wait_rate_window: bool = True) -> dict:
+    """One backfill pass. dry_run only lists the sample (no request)."""
+    sample = select_sample(session, is_official, limit, provider.name)
+    report = {"provider": provider.name, "sample_size": len(sample), "requests": 0, "records": [], "stopped": None, "dry_run": dry_run}
+    if dry_run:
+        report["records"] = [{k: v for k, v in s.items() if k != "product_id"} for s in sample]
+        return report
+    for item in sample:
+        query = item["gtin_key"][1:] if item["gtin_key"].startswith("0") and len(item["gtin_key"]) == 14 else item["gtin_key"]
+        outcome = gateway.lookup(provider, query, "BACKFILL", reserve_percent)
+        while outcome.state == "SKIPPED" and outcome.reason == "RATE_WINDOW" and wait_rate_window:      # per-minute governor: wait for the window, never exceed it
+            time.sleep(max(1, min(65, int((outcome.retry_at - _now()).total_seconds()) + 1)) if outcome.retry_at else 12)
+            outcome = gateway.lookup(provider, query, "BACKFILL", reserve_percent)
+        if outcome.state == "SKIPPED":
+            report["stopped"] = outcome.reason                                 # quota / reserve / per-minute window: stop, never insist
+            break
+        if not outcome.from_cache:
+            report["requests"] += 1
+        report["records"].append(apply_outcome(session, item["gtin_key"], provider.name, outcome, is_official))
+        session.commit()
+    return report

central/app/enrichment/providers.py

@@ -235,15 +235,17 @@ class Oscbr(Provider):
     base = "https://gtin.rscsistemas.com.br"
 
     def __init__(self, client=None, token=None, user=None, password=None):
-        super().__init__(client, token)
+        super().__init__(client, token)                       # token = a ready Bearer token (OSCBR_TOKEN); preferred over storing the login
         self._user, self._password = user, password
         self._bearer, self._bearer_until = None, 0.0
 
     @property
     def configured(self) -> bool:
-        return bool(self._user and self._password)
+        return bool(self._token or (self._user and self._password))
 
     def _auth(self) -> str:
+        if self._token:
+            return self._token
         if self._bearer and time.time() < self._bearer_until:
             return self._bearer
         try:
@@ -322,7 +324,9 @@ class Kodebar(Provider):
 
 def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]:
     """Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos then OSCBR (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here."""
-    providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), Oscbr(client, None, getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), Kodebar(client, getattr(settings, "kodebar_api_key", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
+    providers: list[Provider] = [Oscbr(client, getattr(settings, "oscbr_token", None), getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), Kodebar(client, getattr(settings, "kodebar_api_key", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
+    if getattr(settings, "cosmos_enabled", False):                     # Cosmos is OFF by default (account unavailable); only an explicit COSMOS_ENABLED=1 puts it back in the chain
+        providers.insert(0, Cosmos(client, getattr(settings, "cosmos_token", None)))
     if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False):
         raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)")
     return providers

central/app/version.py

@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
 once published — never a second, independent version number. Fiscal semantics
 are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
 DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
-GTI_FISCAL_VERSION = "1.3.2"
+GTI_FISCAL_VERSION = "1.3.3"
 API_COMPAT_MAJOR_MINOR = "1.0"  # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x

central/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "gti-fiscal-hub"
-version = "1.3.2"
+version = "1.3.3"
 requires-python = ">=3.10"
 dependencies = [
   "fastapi>=0.115,<1",

central/scripts/oscbr_backfill.py

@@ -0,0 +1,41 @@
+"""Verify a small, prioritised sample of CORROBORATED GTINs with OSCBR (free plan). Credentials come ONLY from the Hub environment (OSCBR_TOKEN or OSCBR_USER/OSCBR_PASSWORD).
+
+    python -m central.scripts.oscbr_backfill --limit 20 [--dry-run] [--report out.json]
+
+Never spends the operational reserve (PROVIDER_OPERATIONAL_RESERVE_PERCENT), respects 5/min and 50/day, and never queries VERIFIED_KB, CONFLICT or invalid GTINs.
+"""
+import argparse
+import json
+import sys
+
+from central.app.config import settings
+from central.app.db import SessionLocal
+from central.app.enrichment import backfill, history
+from central.app.enrichment.providers import build_providers
+from central.app.enrichment.quota import ProviderGateway
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--limit", type=int, default=20)
+    ap.add_argument("--dry-run", action="store_true")
+    ap.add_argument("--report", default=None)
+    args = ap.parse_args(argv)
+    providers = build_providers(settings)
+    oscbr = next(p for p in providers if p.name == "oscbr")
+    if not oscbr.configured and not args.dry_run:
+        print(json.dumps({"OSCBR_STATUS": "NOT_CONFIGURED", "requests": 0}))
+        return 2
+    with SessionLocal() as session:
+        current = history.official_sets(session)
+        report = backfill.run(session, ProviderGateway(session, [oscbr]), oscbr, limit=args.limit, reserve_percent=settings.provider_operational_reserve_percent,
+                              is_official=lambda v: v in current, dry_run=args.dry_run)
+    text = json.dumps(report, indent=1, default=str)
+    if args.report:
+        open(args.report, "w", encoding="utf-8").write(text)
+    print(text)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

central/tests/test_health.py

@@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down():
 
 def test_version_endpoint_reports_release_and_fiscal_versions():
     body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
-    assert body["version"] == "1.3.2" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
+    assert body["version"] == "1.3.3" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
 
 
 def test_production_refuses_the_dev_token(monkeypatch):

central/tests/test_oscbr_backfill.py

@@ -0,0 +1,104 @@
+"""OSCBR backfill: prioritised selection, promotion only on agreement, external conflict, quota (5/min, 50/day, reserve, reset), Cosmos out of the chain."""
+from datetime import datetime, timedelta, timezone
+
+import httpx
+from sqlalchemy import select
+
+from central.app.enrichment import backfill, history, kb, quota
+from central.app.enrichment.providers import Oscbr, build_providers
+from central.app.enrichment.quota import ProviderGateway, QuotaManager
+from central.app.models import KbProduct, KbPromotionAudit, ProviderUsageDaily
+from central.tests.test_enrichment_history import run, staging
+from central.tests.test_product_enrichment_resolver import env, gtin  # noqa: F401
+
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _fresh_windows():
+    quota._windows.clear()
+    yield
+    quota._windows.clear()
+
+
+def _oscbr(ncm_by_gtin, calls):
+    def handler(request):
+        if request.url.path == "/oauth/token":
+            return httpx.Response(200, json={"token": "t"})
+        g = request.url.path.rsplit("/", 1)[1]
+        calls.append(g)
+        if g in ncm_by_gtin:
+            return httpx.Response(200, json={"ean": g, "nome": "PRODUTO X", "marca": "M", "ncm": ncm_by_gtin[g], "cest": "0300700"})
+        return httpx.Response(404, json={})
+    return Oscbr(httpx.Client(transport=httpx.MockTransport(handler)), None, "u", "p")
+
+
+def _seed(env, n=6):
+    gs = [gtin(f"78960000{i:04d}") for i in range(n)]
+    for i, g in enumerate(gs):
+        run(env, [staging("A", g, "22021000", years={2023: 30 + i, 2024: 40}, code=str(i), desc=f"PRODUTO TESTE {i} 350ML")])
+    return gs
+
+
+def test_backfill_promotes_only_on_agreement_and_keeps_corroborated_when_no_ncm(env):
+    gs = _seed(env, 4)
+    prov = _oscbr({gs[0]: "22021000", gs[1]: "19059090"}, [])
+    cur = history.official_sets(env)
+    rep = backfill.run(env, ProviderGateway(env, [prov]), prov, limit=10, reserve_percent=20, is_official=lambda v: v in cur, wait_rate_window=False)
+    by = {r["gtin"]: r for r in rep["records"]}
+    assert by[gs[0]]["kb_status"] == "VERIFIED_KB" and by[gs[1]]["kb_status"] == "CONFLICT" and by[gs[1]]["external_conflict"]
+    assert by[gs[2]]["kb_status"] == "CORROBORATED" and by[gs[2]]["found"] is False
+    assert env.scalars(select(KbPromotionAudit)).all()[0].sources["providers"] == ["PROVIDER:oscbr"]
+    cest = [r for r in by.values() if r["cest"]]
+    assert cest and cest[0]["cest"] == "0300700"                                                   # CEST kept as evidence only
+
+
+def test_backfill_never_selects_verified_or_conflict_or_rechecked(env):
+    gs = _seed(env, 3)
+    p = env.scalar(select(KbProduct).where(KbProduct.gtin == "0" + gs[0]))
+    kb.add_evidence(env, p, "ncm", "19059090", "x", family=kb.CLIENT_FAMILY, client_label="Z", lineage={"docs": 50, "years": {"2024": 25, "2025": 25}})
+    env.commit()
+    cur = history.official_sets(env)
+    sample = backfill.select_sample(env, lambda v: v in cur, 10, "oscbr")
+    assert "0" + gs[0] not in {s["gtin_key"] for s in sample} and len(sample) == 2
+    prov = _oscbr({}, [])
+    backfill.run(env, ProviderGateway(env, [prov]), prov, limit=10, reserve_percent=20, is_official=lambda v: v in cur, wait_rate_window=False)
+    assert backfill.select_sample(env, lambda v: v in cur, 10, "oscbr") == []                       # checked once, next_checked_at in the future
+
+
+def test_oscbr_quota_5_per_minute_50_per_day_reserve_and_reset(env):
+    prov = _oscbr({}, [])
+    gw = ProviderGateway(env, [prov])
+    quota._windows["oscbr"].clear()
+    outs = [gw.lookup(prov, gtin(f"78970000{i:04d}")).state for i in range(7)]
+    assert outs[:5] == ["NOT_FOUND"] * 5 and outs[5:] == ["SKIPPED", "SKIPPED"]
+    assert gw.lookup(prov, gtin("789700009991")).reason == "RATE_WINDOW"
+    usage = env.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == "oscbr"))
+    assert usage.requests == 5
+    quota._windows["oscbr"].clear()
+    usage.requests = 40
+    assert gw.lookup(prov, gtin("789700009992"), "BACKFILL", 20).reason == "OPERATIONAL_RESERVE"
+    assert gw.lookup(prov, gtin("789700009993"), "OPERATIONAL").state == "NOT_FOUND"
+    quota._windows["oscbr"].clear()
+    usage.requests = 50
+    assert gw.lookup(prov, gtin("789700009994"), "OPERATIONAL").reason == "RATE_LIMITED"
+    quota._windows["oscbr"].clear()
+    tomorrow = QuotaManager(env, now=lambda: datetime.now(timezone.utc) + timedelta(days=1, minutes=1))
+    ok, reason, _ = tomorrow.allow(prov)
+    assert ok and reason is None
+
+
+def test_cosmos_out_of_chain_and_token_preferred():
+    class S:
+        cosmos_token = "SECRET-C"
+        cosmos_enabled = False
+        oscbr_token = "SECRET-T"
+        oscbr_user = ""
+        oscbr_password = ""
+        kodebar_api_key = ""
+        allow_paid_providers = False
+    names = [p.name for p in build_providers(S())]
+    assert "cosmos" not in names and names[0] == "oscbr" and build_providers(S())[0].configured
+    S.cosmos_enabled = True
+    assert build_providers(S())[0].name == "cosmos"

central/tests/test_product_enrichment_api.py

@@ -98,10 +98,10 @@ def test_features_route_is_admin_only_csrf_and_audited():
 
 def test_dashboard_shows_sources_without_tokens_and_official_freshness(monkeypatch):
     from central.app.enrichment import observability, providers
-    monkeypatch.setattr(observability, "build_providers", lambda _s: providers.build_providers(type("S", (), {"cosmos_token": "SEGREDO-NAO-VAZAR", "allow_paid_providers": False})()))
+    monkeypatch.setattr(observability, "build_providers", lambda _s: providers.build_providers(type("S", (), {"oscbr_token": "SEGREDO-NAO-VAZAR", "allow_paid_providers": False})()))
     client, agent_id, _ = _panel_client()
     page = client.get("/").text
-    assert "Fontes de produtos" in page and "cosmos" in page and "configurado: sim" in page and "upcitemdb" in page and "openfoodfacts" in page
+    assert "Fontes de produtos" in page and "oscbr" in page and "cosmos" not in page and "configurado: sim" in page and "upcitemdb" in page and "openfoodfacts" in page
     assert "SEGREDO-NAO-VAZAR" not in page and "Fontes oficiais" in page and "NCM (Siscomex)" in page
 
 

central/tests/test_product_enrichment_resolver.py

@@ -223,7 +223,7 @@ def test_cosmos_without_token_is_not_configured_and_costs_nothing(env):
 
 def test_paid_provider_guard():
     class S: allow_paid_providers = False; cosmos_token = None
-    assert [p.name for p in build_providers(S())] == ["cosmos", "oscbr", "kodebar", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"]
+    assert [p.name for p in build_providers(S())] == ["oscbr", "kodebar", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"]
 
 
 # ---- provider contract tests against recorded fixtures (httpx.MockTransport, no network) ----------------------------------------------------------