feat: OSCBR token support, Cosmos out of the active chain, quota-safe OSCBR backfill (dry-run selection), provider env passthrough; 1.3.3 (Hub only)
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "gti-fiscal-agent"
|
name = "gti-fiscal-agent"
|
||||||
version = "1.3.2"
|
version = "1.3.3"
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = ["httpx>=0.27,<1"]
|
dependencies = ["httpx>=0.27,<1"]
|
||||||
|
|
||||||
|
|||||||
@@ -107,7 +107,7 @@ class Config:
|
|||||||
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
|
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
|
||||||
# RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib.
|
# RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib.
|
||||||
self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB)))
|
self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB)))
|
||||||
self.agent_version = env.get("AGENT_VERSION", "1.3.2")
|
self.agent_version = env.get("AGENT_VERSION", "1.3.3")
|
||||||
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||||
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
||||||
# Pilot write authorization: both must be explicitly set by the installer for
|
# Pilot write authorization: both must be explicitly set by the installer for
|
||||||
@@ -349,7 +349,7 @@ def execute_job(job, config, executor, state, env):
|
|||||||
"fiscal_context": fiscal_context,
|
"fiscal_context": fiscal_context,
|
||||||
"rtc_auto_apply": rtc_apply,
|
"rtc_auto_apply": rtc_apply,
|
||||||
"fiscal_mode": getattr(config, "fiscal_mode", None),
|
"fiscal_mode": getattr(config, "fiscal_mode", None),
|
||||||
"release_version": env.get("AGENT_VERSION") or "1.3.2",
|
"release_version": env.get("AGENT_VERSION") or "1.3.3",
|
||||||
"products_total": len(products),
|
"products_total": len(products),
|
||||||
"products_changed": len(to_process),
|
"products_changed": len(to_process),
|
||||||
"products_processed": summary["processed"],
|
"products_processed": summary["processed"],
|
||||||
@@ -520,7 +520,7 @@ def run(config, env, *, max_iterations=None, stopper=None):
|
|||||||
def enforce_startup_preflight(env):
|
def enforce_startup_preflight(env):
|
||||||
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
|
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
|
||||||
from gti_fiscal_agent.preflight import run_preflight
|
from gti_fiscal_agent.preflight import run_preflight
|
||||||
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.2")
|
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.3")
|
||||||
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
|
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
|
||||||
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
|
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
|
||||||
if report["PREFLIGHT"] != "PASS":
|
if report["PREFLIGHT"] != "PASS":
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import os
|
|||||||
import sqlite3
|
import sqlite3
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
AGENT_VERSION_DEFAULT = "1.3.2"
|
AGENT_VERSION_DEFAULT = "1.3.3"
|
||||||
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
|
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
|
||||||
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
|
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
|
||||||
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
|
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
|
||||||
|
|||||||
@@ -79,6 +79,9 @@ class Settings:
|
|||||||
ai_base_url: str = os.getenv("AI_BASE_URL", "https://api.z.ai/api/paas/v4")
|
ai_base_url: str = os.getenv("AI_BASE_URL", "https://api.z.ai/api/paas/v4")
|
||||||
# ---- Product enrichment (docs/product_enrichment.md): free providers only, tokens live ONLY in the Hub environment ----
|
# ---- Product enrichment (docs/product_enrichment.md): free providers only, tokens live ONLY in the Hub environment ----
|
||||||
cosmos_token: str = os.getenv("COSMOS_TOKEN", "") # optional; without it Cosmos is NOT_CONFIGURED (not an error)
|
cosmos_token: str = os.getenv("COSMOS_TOKEN", "") # optional; without it Cosmos is NOT_CONFIGURED (not an error)
|
||||||
|
oscbr_token: str = os.getenv("OSCBR_TOKEN", "") # preferred: a ready Bearer token instead of the login
|
||||||
|
cosmos_enabled: bool = os.getenv("COSMOS_ENABLED", "0") == "1" # Cosmos stays out of the active chain unless explicitly enabled
|
||||||
|
oscbr_backfill_enabled: bool = os.getenv("OSCBR_BACKFILL_ENABLED", "0") == "1" # gradual backfill of CORROBORATED GTINs; OFF until a real benchmark proves value
|
||||||
oscbr_user: str = os.getenv("OSCBR_USER", "") # optional free-plan account of gtin.rscsistemas.com.br (never in logs/UI)
|
oscbr_user: str = os.getenv("OSCBR_USER", "") # optional free-plan account of gtin.rscsistemas.com.br (never in logs/UI)
|
||||||
oscbr_password: str = os.getenv("OSCBR_PASSWORD", "")
|
oscbr_password: str = os.getenv("OSCBR_PASSWORD", "")
|
||||||
kodebar_api_key: str = os.getenv("KODEBAR_API_KEY", "") # optional free-plan key (identity only); never in logs/UI
|
kodebar_api_key: str = os.getenv("KODEBAR_API_KEY", "") # optional free-plan key (identity only); never in logs/UI
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
"""Gradual, quota-safe verification of locally CORROBORATED GTINs by an NCM-bearing free provider (OSCBR). Never spends the operational reserve.
|
||||||
|
|
||||||
|
Rules: only CORROBORATED single-NCM GTINs (never VERIFIED_KB, never CONFLICT, never an invalid GTIN); each GTIN is checked once per `RECHECK_DAYS` (next_checked_at kept in the KB evidence, plus the
|
||||||
|
provider cache); a provider answer is EVIDENCE only - the promotion/conflict decision is kb.evaluate's (local strong evidence + independent provider NCM + vigente + zero conflict).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from ..models import KbKnowledge, KbProduct
|
||||||
|
from . import kb
|
||||||
|
from .gtin import normalize_gtin
|
||||||
|
|
||||||
|
RECHECK_DAYS = 30
|
||||||
|
|
||||||
|
|
||||||
|
def _now():
|
||||||
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def select_sample(session, is_official, limit: int, provider: str = "oscbr") -> list[dict]:
|
||||||
|
"""Priority: multi-client, more documents, more years; current-vigente NCM; zero conflict; not checked recently."""
|
||||||
|
ids = list(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.status == kb.CORROBORATED).distinct()))
|
||||||
|
by_product: dict[int, list] = {}
|
||||||
|
for start in range(0, len(ids), 500): # every NCM row of those products: a second (even weaker) value already disqualifies the GTIN
|
||||||
|
for r in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.kb_product_id.in_(ids[start:start + 500]))):
|
||||||
|
by_product.setdefault(r.kb_product_id, []).append(r)
|
||||||
|
now = _now().isoformat()
|
||||||
|
picks = []
|
||||||
|
for pid, items in by_product.items():
|
||||||
|
valid = [i for i in items if is_official(i.value)]
|
||||||
|
if len(valid) != 1 or valid[0].status != kb.CORROBORATED:
|
||||||
|
continue # conflicts / no vigente NCM / already verified are not asked
|
||||||
|
row = valid[0]
|
||||||
|
info = (row.evidence or {}).get("info") or {}
|
||||||
|
if info.get(f"{provider}_next_checked_at", "") > now:
|
||||||
|
continue
|
||||||
|
prod = session.get(KbProduct, pid)
|
||||||
|
if not prod or normalize_gtin(prod.gtin[1:] if prod.gtin and prod.gtin.startswith("0") else prod.gtin)["status"] != "VALID_GTIN":
|
||||||
|
continue
|
||||||
|
summary = kb._lineage_summary([row])
|
||||||
|
if any(f.startswith("PROVIDER:" + provider) for f in ((row.evidence or {}).get("families") or {})):
|
||||||
|
continue
|
||||||
|
picks.append({"gtin_key": prod.gtin, "ncm": row.value, "clients": len(summary["clients"]), "docs": summary["docs"], "years": len(summary["years"]), "product_id": pid})
|
||||||
|
picks.sort(key=lambda p: (-p["clients"], -p["docs"], -p["years"], p["gtin_key"]))
|
||||||
|
return picks[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def apply_outcome(session, gtin_key: str, provider: str, outcome, is_official) -> dict:
|
||||||
|
"""Turns one gateway Outcome into KB evidence. Returns the per-GTIN measurement record."""
|
||||||
|
product = kb.find_by_gtin(session, gtin_key)
|
||||||
|
rec = {"gtin": gtin_key[-13:] if gtin_key.startswith("0") else gtin_key, "state": outcome.state, "reason": outcome.reason, "found": False, "ncm": None, "cest": None, "description": None, "brand": None,
|
||||||
|
"local_ncm": None, "official_valid": None, "kb_status": None, "external_conflict": False}
|
||||||
|
local_rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm"))) if product else []
|
||||||
|
rec["local_ncm"] = local_rows[0].value if local_rows else None
|
||||||
|
stamp = (_now() + timedelta(days=RECHECK_DAYS)).isoformat()
|
||||||
|
if outcome.state in ("FOUND", "CACHE_HIT") and outcome.product:
|
||||||
|
p = outcome.product
|
||||||
|
rec.update({"found": True, "ncm": p.ncm, "cest": p.cest, "description": p.title, "brand": p.brand})
|
||||||
|
if p.ncm:
|
||||||
|
rec["official_valid"] = bool(is_official(p.ncm))
|
||||||
|
kb.add_evidence(session, product, "ncm", p.ncm, f"provider:{provider}"[:32], family=f"PROVIDER:{provider}", info={"raw_hash": p.raw_hash})
|
||||||
|
if p.cest:
|
||||||
|
kb.add_evidence(session, product, "cest", p.cest, f"provider:{provider}"[:32], family=f"PROVIDER:{provider}", info={"raw_hash": p.raw_hash})
|
||||||
|
if p.brand and not product.brand:
|
||||||
|
product.brand = p.brand
|
||||||
|
if p.title:
|
||||||
|
kb.add_alias(session, product, p.title.upper()[:200])
|
||||||
|
ev = kb.evaluate(session, product, "ncm", is_official)
|
||||||
|
rec["kb_status"] = ev["status"]
|
||||||
|
rec["external_conflict"] = bool(ev["status"] == kb.CONFLICT and rec["ncm"])
|
||||||
|
if outcome.state in ("FOUND", "CACHE_HIT", "NOT_FOUND"): # remember the check so it is not repeated (skipped/errors are retried later)
|
||||||
|
for r in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm")):
|
||||||
|
r.evidence = {**(r.evidence or {}), "info": {**((r.evidence or {}).get("info") or {}), f"{provider}_next_checked_at": stamp}}
|
||||||
|
return rec
|
||||||
|
|
||||||
|
|
||||||
|
def run(session, gateway, provider, *, limit: int, reserve_percent: int, is_official, dry_run: bool = False, wait_rate_window: bool = True) -> dict:
|
||||||
|
"""One backfill pass. dry_run only lists the sample (no request)."""
|
||||||
|
sample = select_sample(session, is_official, limit, provider.name)
|
||||||
|
report = {"provider": provider.name, "sample_size": len(sample), "requests": 0, "records": [], "stopped": None, "dry_run": dry_run}
|
||||||
|
if dry_run:
|
||||||
|
report["records"] = [{k: v for k, v in s.items() if k != "product_id"} for s in sample]
|
||||||
|
return report
|
||||||
|
for item in sample:
|
||||||
|
query = item["gtin_key"][1:] if item["gtin_key"].startswith("0") and len(item["gtin_key"]) == 14 else item["gtin_key"]
|
||||||
|
outcome = gateway.lookup(provider, query, "BACKFILL", reserve_percent)
|
||||||
|
while outcome.state == "SKIPPED" and outcome.reason == "RATE_WINDOW" and wait_rate_window: # per-minute governor: wait for the window, never exceed it
|
||||||
|
time.sleep(max(1, min(65, int((outcome.retry_at - _now()).total_seconds()) + 1)) if outcome.retry_at else 12)
|
||||||
|
outcome = gateway.lookup(provider, query, "BACKFILL", reserve_percent)
|
||||||
|
if outcome.state == "SKIPPED":
|
||||||
|
report["stopped"] = outcome.reason # quota / reserve / per-minute window: stop, never insist
|
||||||
|
break
|
||||||
|
if not outcome.from_cache:
|
||||||
|
report["requests"] += 1
|
||||||
|
report["records"].append(apply_outcome(session, item["gtin_key"], provider.name, outcome, is_official))
|
||||||
|
session.commit()
|
||||||
|
return report
|
||||||
@@ -235,15 +235,17 @@ class Oscbr(Provider):
|
|||||||
base = "https://gtin.rscsistemas.com.br"
|
base = "https://gtin.rscsistemas.com.br"
|
||||||
|
|
||||||
def __init__(self, client=None, token=None, user=None, password=None):
|
def __init__(self, client=None, token=None, user=None, password=None):
|
||||||
super().__init__(client, token)
|
super().__init__(client, token) # token = a ready Bearer token (OSCBR_TOKEN); preferred over storing the login
|
||||||
self._user, self._password = user, password
|
self._user, self._password = user, password
|
||||||
self._bearer, self._bearer_until = None, 0.0
|
self._bearer, self._bearer_until = None, 0.0
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def configured(self) -> bool:
|
def configured(self) -> bool:
|
||||||
return bool(self._user and self._password)
|
return bool(self._token or (self._user and self._password))
|
||||||
|
|
||||||
def _auth(self) -> str:
|
def _auth(self) -> str:
|
||||||
|
if self._token:
|
||||||
|
return self._token
|
||||||
if self._bearer and time.time() < self._bearer_until:
|
if self._bearer and time.time() < self._bearer_until:
|
||||||
return self._bearer
|
return self._bearer
|
||||||
try:
|
try:
|
||||||
@@ -322,7 +324,9 @@ class Kodebar(Provider):
|
|||||||
|
|
||||||
def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]:
|
def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]:
|
||||||
"""Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos then OSCBR (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here."""
|
"""Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos then OSCBR (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here."""
|
||||||
providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), Oscbr(client, None, getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), Kodebar(client, getattr(settings, "kodebar_api_key", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
|
providers: list[Provider] = [Oscbr(client, getattr(settings, "oscbr_token", None), getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), Kodebar(client, getattr(settings, "kodebar_api_key", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
|
||||||
|
if getattr(settings, "cosmos_enabled", False): # Cosmos is OFF by default (account unavailable); only an explicit COSMOS_ENABLED=1 puts it back in the chain
|
||||||
|
providers.insert(0, Cosmos(client, getattr(settings, "cosmos_token", None)))
|
||||||
if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False):
|
if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False):
|
||||||
raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)")
|
raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)")
|
||||||
return providers
|
return providers
|
||||||
|
|||||||
@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
|
|||||||
once published — never a second, independent version number. Fiscal semantics
|
once published — never a second, independent version number. Fiscal semantics
|
||||||
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
|
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
|
||||||
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
|
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
|
||||||
GTI_FISCAL_VERSION = "1.3.2"
|
GTI_FISCAL_VERSION = "1.3.3"
|
||||||
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
|
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "gti-fiscal-hub"
|
name = "gti-fiscal-hub"
|
||||||
version = "1.3.2"
|
version = "1.3.3"
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"fastapi>=0.115,<1",
|
"fastapi>=0.115,<1",
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
"""Verify a small, prioritised sample of CORROBORATED GTINs with OSCBR (free plan). Credentials come ONLY from the Hub environment (OSCBR_TOKEN or OSCBR_USER/OSCBR_PASSWORD).
|
||||||
|
|
||||||
|
python -m central.scripts.oscbr_backfill --limit 20 [--dry-run] [--report out.json]
|
||||||
|
|
||||||
|
Never spends the operational reserve (PROVIDER_OPERATIONAL_RESERVE_PERCENT), respects 5/min and 50/day, and never queries VERIFIED_KB, CONFLICT or invalid GTINs.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from central.app.config import settings
|
||||||
|
from central.app.db import SessionLocal
|
||||||
|
from central.app.enrichment import backfill, history
|
||||||
|
from central.app.enrichment.providers import build_providers
|
||||||
|
from central.app.enrichment.quota import ProviderGateway
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--limit", type=int, default=20)
|
||||||
|
ap.add_argument("--dry-run", action="store_true")
|
||||||
|
ap.add_argument("--report", default=None)
|
||||||
|
args = ap.parse_args(argv)
|
||||||
|
providers = build_providers(settings)
|
||||||
|
oscbr = next(p for p in providers if p.name == "oscbr")
|
||||||
|
if not oscbr.configured and not args.dry_run:
|
||||||
|
print(json.dumps({"OSCBR_STATUS": "NOT_CONFIGURED", "requests": 0}))
|
||||||
|
return 2
|
||||||
|
with SessionLocal() as session:
|
||||||
|
current = history.official_sets(session)
|
||||||
|
report = backfill.run(session, ProviderGateway(session, [oscbr]), oscbr, limit=args.limit, reserve_percent=settings.provider_operational_reserve_percent,
|
||||||
|
is_official=lambda v: v in current, dry_run=args.dry_run)
|
||||||
|
text = json.dumps(report, indent=1, default=str)
|
||||||
|
if args.report:
|
||||||
|
open(args.report, "w", encoding="utf-8").write(text)
|
||||||
|
print(text)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down():
|
|||||||
|
|
||||||
def test_version_endpoint_reports_release_and_fiscal_versions():
|
def test_version_endpoint_reports_release_and_fiscal_versions():
|
||||||
body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
|
body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
|
||||||
assert body["version"] == "1.3.2" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
assert body["version"] == "1.3.3" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
||||||
|
|
||||||
|
|
||||||
def test_production_refuses_the_dev_token(monkeypatch):
|
def test_production_refuses_the_dev_token(monkeypatch):
|
||||||
|
|||||||
@@ -0,0 +1,104 @@
|
|||||||
|
"""OSCBR backfill: prioritised selection, promotion only on agreement, external conflict, quota (5/min, 50/day, reserve, reset), Cosmos out of the chain."""
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from central.app.enrichment import backfill, history, kb, quota
|
||||||
|
from central.app.enrichment.providers import Oscbr, build_providers
|
||||||
|
from central.app.enrichment.quota import ProviderGateway, QuotaManager
|
||||||
|
from central.app.models import KbProduct, KbPromotionAudit, ProviderUsageDaily
|
||||||
|
from central.tests.test_enrichment_history import run, staging
|
||||||
|
from central.tests.test_product_enrichment_resolver import env, gtin # noqa: F401
|
||||||
|
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _fresh_windows():
|
||||||
|
quota._windows.clear()
|
||||||
|
yield
|
||||||
|
quota._windows.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def _oscbr(ncm_by_gtin, calls):
|
||||||
|
def handler(request):
|
||||||
|
if request.url.path == "/oauth/token":
|
||||||
|
return httpx.Response(200, json={"token": "t"})
|
||||||
|
g = request.url.path.rsplit("/", 1)[1]
|
||||||
|
calls.append(g)
|
||||||
|
if g in ncm_by_gtin:
|
||||||
|
return httpx.Response(200, json={"ean": g, "nome": "PRODUTO X", "marca": "M", "ncm": ncm_by_gtin[g], "cest": "0300700"})
|
||||||
|
return httpx.Response(404, json={})
|
||||||
|
return Oscbr(httpx.Client(transport=httpx.MockTransport(handler)), None, "u", "p")
|
||||||
|
|
||||||
|
|
||||||
|
def _seed(env, n=6):
|
||||||
|
gs = [gtin(f"78960000{i:04d}") for i in range(n)]
|
||||||
|
for i, g in enumerate(gs):
|
||||||
|
run(env, [staging("A", g, "22021000", years={2023: 30 + i, 2024: 40}, code=str(i), desc=f"PRODUTO TESTE {i} 350ML")])
|
||||||
|
return gs
|
||||||
|
|
||||||
|
|
||||||
|
def test_backfill_promotes_only_on_agreement_and_keeps_corroborated_when_no_ncm(env):
|
||||||
|
gs = _seed(env, 4)
|
||||||
|
prov = _oscbr({gs[0]: "22021000", gs[1]: "19059090"}, [])
|
||||||
|
cur = history.official_sets(env)
|
||||||
|
rep = backfill.run(env, ProviderGateway(env, [prov]), prov, limit=10, reserve_percent=20, is_official=lambda v: v in cur, wait_rate_window=False)
|
||||||
|
by = {r["gtin"]: r for r in rep["records"]}
|
||||||
|
assert by[gs[0]]["kb_status"] == "VERIFIED_KB" and by[gs[1]]["kb_status"] == "CONFLICT" and by[gs[1]]["external_conflict"]
|
||||||
|
assert by[gs[2]]["kb_status"] == "CORROBORATED" and by[gs[2]]["found"] is False
|
||||||
|
assert env.scalars(select(KbPromotionAudit)).all()[0].sources["providers"] == ["PROVIDER:oscbr"]
|
||||||
|
cest = [r for r in by.values() if r["cest"]]
|
||||||
|
assert cest and cest[0]["cest"] == "0300700" # CEST kept as evidence only
|
||||||
|
|
||||||
|
|
||||||
|
def test_backfill_never_selects_verified_or_conflict_or_rechecked(env):
|
||||||
|
gs = _seed(env, 3)
|
||||||
|
p = env.scalar(select(KbProduct).where(KbProduct.gtin == "0" + gs[0]))
|
||||||
|
kb.add_evidence(env, p, "ncm", "19059090", "x", family=kb.CLIENT_FAMILY, client_label="Z", lineage={"docs": 50, "years": {"2024": 25, "2025": 25}})
|
||||||
|
env.commit()
|
||||||
|
cur = history.official_sets(env)
|
||||||
|
sample = backfill.select_sample(env, lambda v: v in cur, 10, "oscbr")
|
||||||
|
assert "0" + gs[0] not in {s["gtin_key"] for s in sample} and len(sample) == 2
|
||||||
|
prov = _oscbr({}, [])
|
||||||
|
backfill.run(env, ProviderGateway(env, [prov]), prov, limit=10, reserve_percent=20, is_official=lambda v: v in cur, wait_rate_window=False)
|
||||||
|
assert backfill.select_sample(env, lambda v: v in cur, 10, "oscbr") == [] # checked once, next_checked_at in the future
|
||||||
|
|
||||||
|
|
||||||
|
def test_oscbr_quota_5_per_minute_50_per_day_reserve_and_reset(env):
|
||||||
|
prov = _oscbr({}, [])
|
||||||
|
gw = ProviderGateway(env, [prov])
|
||||||
|
quota._windows["oscbr"].clear()
|
||||||
|
outs = [gw.lookup(prov, gtin(f"78970000{i:04d}")).state for i in range(7)]
|
||||||
|
assert outs[:5] == ["NOT_FOUND"] * 5 and outs[5:] == ["SKIPPED", "SKIPPED"]
|
||||||
|
assert gw.lookup(prov, gtin("789700009991")).reason == "RATE_WINDOW"
|
||||||
|
usage = env.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == "oscbr"))
|
||||||
|
assert usage.requests == 5
|
||||||
|
quota._windows["oscbr"].clear()
|
||||||
|
usage.requests = 40
|
||||||
|
assert gw.lookup(prov, gtin("789700009992"), "BACKFILL", 20).reason == "OPERATIONAL_RESERVE"
|
||||||
|
assert gw.lookup(prov, gtin("789700009993"), "OPERATIONAL").state == "NOT_FOUND"
|
||||||
|
quota._windows["oscbr"].clear()
|
||||||
|
usage.requests = 50
|
||||||
|
assert gw.lookup(prov, gtin("789700009994"), "OPERATIONAL").reason == "RATE_LIMITED"
|
||||||
|
quota._windows["oscbr"].clear()
|
||||||
|
tomorrow = QuotaManager(env, now=lambda: datetime.now(timezone.utc) + timedelta(days=1, minutes=1))
|
||||||
|
ok, reason, _ = tomorrow.allow(prov)
|
||||||
|
assert ok and reason is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_cosmos_out_of_chain_and_token_preferred():
|
||||||
|
class S:
|
||||||
|
cosmos_token = "SECRET-C"
|
||||||
|
cosmos_enabled = False
|
||||||
|
oscbr_token = "SECRET-T"
|
||||||
|
oscbr_user = ""
|
||||||
|
oscbr_password = ""
|
||||||
|
kodebar_api_key = ""
|
||||||
|
allow_paid_providers = False
|
||||||
|
names = [p.name for p in build_providers(S())]
|
||||||
|
assert "cosmos" not in names and names[0] == "oscbr" and build_providers(S())[0].configured
|
||||||
|
S.cosmos_enabled = True
|
||||||
|
assert build_providers(S())[0].name == "cosmos"
|
||||||
@@ -98,10 +98,10 @@ def test_features_route_is_admin_only_csrf_and_audited():
|
|||||||
|
|
||||||
def test_dashboard_shows_sources_without_tokens_and_official_freshness(monkeypatch):
|
def test_dashboard_shows_sources_without_tokens_and_official_freshness(monkeypatch):
|
||||||
from central.app.enrichment import observability, providers
|
from central.app.enrichment import observability, providers
|
||||||
monkeypatch.setattr(observability, "build_providers", lambda _s: providers.build_providers(type("S", (), {"cosmos_token": "SEGREDO-NAO-VAZAR", "allow_paid_providers": False})()))
|
monkeypatch.setattr(observability, "build_providers", lambda _s: providers.build_providers(type("S", (), {"oscbr_token": "SEGREDO-NAO-VAZAR", "allow_paid_providers": False})()))
|
||||||
client, agent_id, _ = _panel_client()
|
client, agent_id, _ = _panel_client()
|
||||||
page = client.get("/").text
|
page = client.get("/").text
|
||||||
assert "Fontes de produtos" in page and "cosmos" in page and "configurado: sim" in page and "upcitemdb" in page and "openfoodfacts" in page
|
assert "Fontes de produtos" in page and "oscbr" in page and "cosmos" not in page and "configurado: sim" in page and "upcitemdb" in page and "openfoodfacts" in page
|
||||||
assert "SEGREDO-NAO-VAZAR" not in page and "Fontes oficiais" in page and "NCM (Siscomex)" in page
|
assert "SEGREDO-NAO-VAZAR" not in page and "Fontes oficiais" in page and "NCM (Siscomex)" in page
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -223,7 +223,7 @@ def test_cosmos_without_token_is_not_configured_and_costs_nothing(env):
|
|||||||
|
|
||||||
def test_paid_provider_guard():
|
def test_paid_provider_guard():
|
||||||
class S: allow_paid_providers = False; cosmos_token = None
|
class S: allow_paid_providers = False; cosmos_token = None
|
||||||
assert [p.name for p in build_providers(S())] == ["cosmos", "oscbr", "kodebar", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"]
|
assert [p.name for p in build_providers(S())] == ["oscbr", "kodebar", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"]
|
||||||
|
|
||||||
|
|
||||||
# ---- provider contract tests against recorded fixtures (httpx.MockTransport, no network) ----------------------------------------------------------
|
# ---- provider contract tests against recorded fixtures (httpx.MockTransport, no network) ----------------------------------------------------------
|
||||||
|
|||||||
@@ -19,4 +19,4 @@ FB_PASSWORD=
|
|||||||
FISCAL_MODE=DRY_RUN
|
FISCAL_MODE=DRY_RUN
|
||||||
FIREBIRD_WRITES=0
|
FIREBIRD_WRITES=0
|
||||||
POLL_INTERVAL_SECONDS=5
|
POLL_INTERVAL_SECONDS=5
|
||||||
AGENT_VERSION=1.3.2
|
AGENT_VERSION=1.3.3
|
||||||
|
|||||||
@@ -164,7 +164,7 @@ switch ($Command) {
|
|||||||
"GTI_HUB_URL=$hub", "AGENT_TOKEN=$token", "AGENT_ID=$agentId",
|
"GTI_HUB_URL=$hub", "AGENT_TOKEN=$token", "AGENT_ID=$agentId",
|
||||||
"COMPANY_CODE=$code", "COMPANY_CNPJ=$cnpj", "COMPANY_CRT_SOURCE=$crtSource",
|
"COMPANY_CODE=$code", "COMPANY_CNPJ=$cnpj", "COMPANY_CRT_SOURCE=$crtSource",
|
||||||
"FIREBIRD_DATABASE_PATH=$db", "FB_USER=$fbUser", "FB_PASSWORD=$fbPass",
|
"FIREBIRD_DATABASE_PATH=$db", "FB_USER=$fbUser", "FB_PASSWORD=$fbPass",
|
||||||
'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.2'
|
'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.3'
|
||||||
)
|
)
|
||||||
Set-Content -Path $EnvFile -Value $lines -Encoding UTF8
|
Set-Content -Path $EnvFile -Value $lines -Encoding UTF8
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,4 +19,4 @@ FB_PASSWORD=
|
|||||||
FISCAL_MODE=DRY_RUN
|
FISCAL_MODE=DRY_RUN
|
||||||
FIREBIRD_WRITES=0
|
FIREBIRD_WRITES=0
|
||||||
POLL_INTERVAL_SECONDS=5
|
POLL_INTERVAL_SECONDS=5
|
||||||
AGENT_VERSION=1.3.2
|
AGENT_VERSION=1.3.3
|
||||||
|
|||||||
@@ -19,4 +19,4 @@ FB_PASSWORD=
|
|||||||
FISCAL_MODE=DRY_RUN
|
FISCAL_MODE=DRY_RUN
|
||||||
FIREBIRD_WRITES=0
|
FIREBIRD_WRITES=0
|
||||||
POLL_INTERVAL_SECONDS=5
|
POLL_INTERVAL_SECONDS=5
|
||||||
AGENT_VERSION=1.3.2
|
AGENT_VERSION=1.3.3
|
||||||
|
|||||||
@@ -19,4 +19,4 @@ FB_PASSWORD=
|
|||||||
FISCAL_MODE=DRY_RUN
|
FISCAL_MODE=DRY_RUN
|
||||||
FIREBIRD_WRITES=0
|
FIREBIRD_WRITES=0
|
||||||
POLL_INTERVAL_SECONDS=5
|
POLL_INTERVAL_SECONDS=5
|
||||||
AGENT_VERSION=1.3.2
|
AGENT_VERSION=1.3.3
|
||||||
|
|||||||
@@ -19,4 +19,4 @@ FB_PASSWORD=
|
|||||||
FISCAL_MODE=DRY_RUN
|
FISCAL_MODE=DRY_RUN
|
||||||
FIREBIRD_WRITES=0
|
FIREBIRD_WRITES=0
|
||||||
POLL_INTERVAL_SECONDS=5
|
POLL_INTERVAL_SECONDS=5
|
||||||
AGENT_VERSION=1.3.2
|
AGENT_VERSION=1.3.3
|
||||||
|
|||||||
+10
-1
@@ -1,5 +1,5 @@
|
|||||||
# GTI Fiscal Hub — production environment (copy to .env, never commit .env)
|
# GTI Fiscal Hub — production environment (copy to .env, never commit .env)
|
||||||
GTI_FISCAL_VERSION=1.3.2
|
GTI_FISCAL_VERSION=1.3.3
|
||||||
|
|
||||||
# REQUIRED. URL-safe secret (used inside a connection URL): openssl rand -hex 24
|
# REQUIRED. URL-safe secret (used inside a connection URL): openssl rand -hex 24
|
||||||
POSTGRES_PASSWORD=
|
POSTGRES_PASSWORD=
|
||||||
@@ -45,3 +45,12 @@ FISCAL_INTERNAL_API_TOKEN=
|
|||||||
BACKUP_DIR=./backups
|
BACKUP_DIR=./backups
|
||||||
# Max whole-database jobs (INITIAL_SYNC / REPROCESS_ALL) running at once across all Agents; others queue as PENDING. <=0 disables.
|
# Max whole-database jobs (INITIAL_SYNC / REPROCESS_ALL) running at once across all Agents; others queue as PENDING. <=0 disables.
|
||||||
HEAVY_JOB_CONCURRENCY_LIMIT=3
|
HEAVY_JOB_CONCURRENCY_LIMIT=3
|
||||||
|
|
||||||
|
# Product-enrichment providers (free plans) - fill on the SERVER only, never commit values
|
||||||
|
OSCBR_TOKEN=
|
||||||
|
OSCBR_USER=
|
||||||
|
OSCBR_PASSWORD=
|
||||||
|
KODEBAR_API_KEY=
|
||||||
|
COSMOS_ENABLED=0
|
||||||
|
OSCBR_BACKFILL_ENABLED=0
|
||||||
|
PROVIDER_OPERATIONAL_RESERVE_PERCENT=20
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ services:
|
|||||||
build:
|
build:
|
||||||
context: ../..
|
context: ../..
|
||||||
dockerfile: central/Dockerfile
|
dockerfile: central/Dockerfile
|
||||||
image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.2}
|
image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.3}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
@@ -45,6 +45,14 @@ services:
|
|||||||
APP_ENV: production
|
APP_ENV: production
|
||||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
|
||||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||||
|
# Product-enrichment providers (free plans). Secrets live ONLY in deploy/hub/.env on the server (never committed, never in the Agent/installer/panel/logs).
|
||||||
|
OSCBR_TOKEN: ${OSCBR_TOKEN:-}
|
||||||
|
OSCBR_USER: ${OSCBR_USER:-}
|
||||||
|
OSCBR_PASSWORD: ${OSCBR_PASSWORD:-}
|
||||||
|
KODEBAR_API_KEY: ${KODEBAR_API_KEY:-}
|
||||||
|
COSMOS_ENABLED: ${COSMOS_ENABLED:-0}
|
||||||
|
OSCBR_BACKFILL_ENABLED: ${OSCBR_BACKFILL_ENABLED:-0}
|
||||||
|
PROVIDER_OPERATIONAL_RESERVE_PERCENT: ${PROVIDER_OPERATIONAL_RESERVE_PERCENT:-20}
|
||||||
BASE_URL: ${BASE_URL:?BASE_URL is required (public HTTPS URL of the Hub)}
|
BASE_URL: ${BASE_URL:?BASE_URL is required (public HTTPS URL of the Hub)}
|
||||||
AGENT_ONLINE_THRESHOLD_SECONDS: ${AGENT_ONLINE_THRESHOLD_SECONDS:-120}
|
AGENT_ONLINE_THRESHOLD_SECONDS: ${AGENT_ONLINE_THRESHOLD_SECONDS:-120}
|
||||||
# Reverse proxy on another host: uvicorn only honours X-Forwarded-For/-Proto
|
# Reverse proxy on another host: uvicorn only honours X-Forwarded-For/-Proto
|
||||||
|
|||||||
@@ -1,14 +1,14 @@
|
|||||||
{
|
{
|
||||||
"artifact": "RELEASE_MANIFEST",
|
"artifact": "RELEASE_MANIFEST",
|
||||||
"GTI_FISCAL_VERSION": "1.3.2",
|
"GTI_FISCAL_VERSION": "1.3.3",
|
||||||
"generated_at": "2026-09-25T15:43:40+00:00",
|
"generated_at": "2026-09-25T15:43:40+00:00",
|
||||||
"hub": {
|
"hub": {
|
||||||
"version": "1.3.2",
|
"version": "1.3.3",
|
||||||
"api_compat": "1.0",
|
"api_compat": "1.0",
|
||||||
"engine_version": "0.3.0",
|
"engine_version": "0.3.0",
|
||||||
"decision_version": "mission-04c-ipi-ex-split"
|
"decision_version": "mission-04c-ipi-ex-split"
|
||||||
},
|
},
|
||||||
"agent_version": "1.3.2",
|
"agent_version": "1.3.3",
|
||||||
"fiscal_engine": {
|
"fiscal_engine": {
|
||||||
"ENGINE_VERSION": "0.3.0",
|
"ENGINE_VERSION": "0.3.0",
|
||||||
"DECISION_VERSION": "mission-04c-ipi-ex-split",
|
"DECISION_VERSION": "mission-04c-ipi-ex-split",
|
||||||
@@ -89,5 +89,5 @@
|
|||||||
"record_count": 11103
|
"record_count": 11103
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"note": "Hub-only release 1.3.2: the Windows Agent installer distributed remains GTIFiscalAgent-Setup-1.3.1.exe (no functional Agent change)."
|
"note": "Hub-only release 1.3.3: the Windows Agent installer distributed remains GTIFiscalAgent-Setup-1.3.1.exe (no functional Agent change)."
|
||||||
}
|
}
|
||||||
+1
-1
@@ -1 +1 @@
|
|||||||
1.3.2
|
1.3.3
|
||||||
|
|||||||
@@ -14,11 +14,11 @@ sys.path.insert(0, str(ROOT))
|
|||||||
def test_versions_are_coherent_across_hub_agent_and_packages():
|
def test_versions_are_coherent_across_hub_agent_and_packages():
|
||||||
from central.app.version import GTI_FISCAL_VERSION
|
from central.app.version import GTI_FISCAL_VERSION
|
||||||
v = (ROOT / "release" / "VERSION").read_text(encoding="utf-8").strip()
|
v = (ROOT / "release" / "VERSION").read_text(encoding="utf-8").strip()
|
||||||
assert v == GTI_FISCAL_VERSION == "1.3.2"
|
assert v == GTI_FISCAL_VERSION == "1.3.3"
|
||||||
pep = v.replace("-rc", "rc")
|
pep = v.replace("-rc", "rc")
|
||||||
for pp in ("central/pyproject.toml", "agent/pyproject.toml"):
|
for pp in ("central/pyproject.toml", "agent/pyproject.toml"):
|
||||||
assert f'version = "{pep}"' in (ROOT / pp).read_text(encoding="utf-8")
|
assert f'version = "{pep}"' in (ROOT / pp).read_text(encoding="utf-8")
|
||||||
assert '"AGENT_VERSION", "1.3.2"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8")
|
assert '"AGENT_VERSION", "1.3.3"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8")
|
||||||
manifest = ROOT / "release" / "RELEASE_MANIFEST.json"
|
manifest = ROOT / "release" / "RELEASE_MANIFEST.json"
|
||||||
if manifest.exists():
|
if manifest.exists():
|
||||||
m = json.loads(manifest.read_text(encoding="utf-8"))
|
m = json.loads(manifest.read_text(encoding="utf-8"))
|
||||||
|
|||||||
Reference in New Issue
Block a user