feat: OSCBR optional provider, re-baseline after inactivity, field audit, provider UI/feature matrix tests, benchmark chain; 1.3.1

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
2026-09-26 00:48:18 -03:00
co-authored by Claude Sonnet 5
parent c3c9d1b4e0
commit e6bcf12d53
27 changed files with 1015 additions and 41 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
[project] [project]
name = "gti-fiscal-agent" name = "gti-fiscal-agent"
version = "1.3.0" version = "1.3.1"
requires-python = ">=3.10" requires-python = ">=3.10"
dependencies = ["httpx>=0.27,<1"] dependencies = ["httpx>=0.27,<1"]
+3 -3
View File
@@ -107,7 +107,7 @@ class Config:
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5")) self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
# RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib. # RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib.
self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB))) self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB)))
self.agent_version = env.get("AGENT_VERSION", "1.3.0") self.agent_version = env.get("AGENT_VERSION", "1.3.1")
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1) self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite")) self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
# Pilot write authorization: both must be explicitly set by the installer for # Pilot write authorization: both must be explicitly set by the installer for
@@ -349,7 +349,7 @@ def execute_job(job, config, executor, state, env):
"fiscal_context": fiscal_context, "fiscal_context": fiscal_context,
"rtc_auto_apply": rtc_apply, "rtc_auto_apply": rtc_apply,
"fiscal_mode": getattr(config, "fiscal_mode", None), "fiscal_mode": getattr(config, "fiscal_mode", None),
"release_version": env.get("AGENT_VERSION") or "1.3.0", "release_version": env.get("AGENT_VERSION") or "1.3.1",
"products_total": len(products), "products_total": len(products),
"products_changed": len(to_process), "products_changed": len(to_process),
"products_processed": summary["processed"], "products_processed": summary["processed"],
@@ -520,7 +520,7 @@ def run(config, env, *, max_iterations=None, stopper=None):
def enforce_startup_preflight(env): def enforce_startup_preflight(env):
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed.""" """Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
from gti_fiscal_agent.preflight import run_preflight from gti_fiscal_agent.preflight import run_preflight
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.0") report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.1")
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False)) logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False)) print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
if report["PREFLIGHT"] != "PASS": if report["PREFLIGHT"] != "PASS":
+10
View File
@@ -271,6 +271,14 @@ class EnrichmentWorker:
self.store.set_meta("baseline_count", len(codes)) self.store.set_meta("baseline_count", len(codes))
self._last_count, self._last_diff = len(codes), self.clock() self._last_count, self._last_diff = len(codes), self.clock()
return [] return []
if self.store.meta("rebaseline") == "1":
# the function was inactive (DISABLED / WAITING_FEATURE_CONFIGURATION) for a while: whatever appeared meanwhile is NOT "new" for the enrichment
# (a long-disabled installation must never send a burst of products to the providers when it is switched on again)
codes = read_codes(conn)
self.store.add_known(codes)
self.store.set_meta("rebaseline", "0")
self._last_count, self._last_diff = len(codes), self.clock()
return []
started = self.clock() started = self.clock()
t0 = time.monotonic() t0 = time.monotonic()
count = read_count(conn) count = read_count(conn)
@@ -417,6 +425,8 @@ class EnrichmentWorker:
"""One iteration; returns the list of processed product codes.""" """One iteration; returns the list of processed product codes."""
mode = self.mode() mode = self.mode()
if mode not in ("DRY_RUN", "ENABLED"): if mode not in ("DRY_RUN", "ENABLED"):
if self.store.meta("baseline_done") == "1":
self.store.set_meta("rebaseline", "1")
return [] # WAITING_FEATURE_CONFIGURATION / DISABLED: zero Firebird activity return [] # WAITING_FEATURE_CONFIGURATION / DISABLED: zero Firebird activity
self.detect() self.detect()
done = [] done = []
+1 -1
View File
@@ -9,7 +9,7 @@ import os
import sqlite3 import sqlite3
from pathlib import Path from pathlib import Path
AGENT_VERSION_DEFAULT = "1.3.0" AGENT_VERSION_DEFAULT = "1.3.1"
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD") REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest") REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY") FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
+22 -1
View File
@@ -124,7 +124,6 @@ def test_existing_value_is_never_overwritten(env):
def test_dry_run_and_disabled_and_waiting_do_not_write_or_touch_firebird(env): def test_dry_run_and_disabled_and_waiting_do_not_write_or_touch_firebird(env):
db, hub, store, worker, now, _ = env db, hub, store, worker, now, _ = env
worker.step() worker.step()
add(db, "N3")
for features in ({"features_configured": False}, {"features_configured": True, "enrichment_mode": "DISABLED"}): for features in ({"features_configured": False}, {"features_configured": True, "enrichment_mode": "DISABLED"}):
hub.features = features hub.features = features
worker._features_at = -999 worker._features_at = -999
@@ -132,6 +131,9 @@ def test_dry_run_and_disabled_and_waiting_do_not_write_or_touch_firebird(env):
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"} hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
worker._features_at = -999 worker._features_at = -999
now[0] += 5 now[0] += 5
worker.step() # re-baseline after the inactive period
add(db, "N3")
now[0] += 5
worker.step() worker.step()
assert row(db, "N3") == (None, None, None, "keep") and hub.reports[0]["fields"][0]["outcome"] == "DRY_RUN" assert row(db, "N3") == (None, None, None, "keep") and hub.reports[0]["fields"][0]["outcome"] == "DRY_RUN"
@@ -209,3 +211,22 @@ def test_gtin_check_parity_with_the_hub_and_codbarra_is_used():
assert E.pick_gtin({"GTIN": "", "CODBARRA": "7891000100103"}) == "7891000100103" assert E.pick_gtin({"GTIN": "", "CODBARRA": "7891000100103"}) == "7891000100103"
assert E.pick_gtin({"GTIN": "7891000100103", "CODBARRA": "999"}) == "7891000100103" assert E.pick_gtin({"GTIN": "7891000100103", "CODBARRA": "999"}) == "7891000100103"
assert E.pick_gtin({"GTIN": "", "CODBARRA": "123"}) == "123" and E.pick_gtin({}) == "" assert E.pick_gtin({"GTIN": "", "CODBARRA": "123"}) == "123" and E.pick_gtin({}) == ""
def test_products_created_while_inactive_are_not_new_when_reactivated(env):
db, hub, store, worker, now, _ = env
worker.step() # baseline
hub.features = {"features_configured": True, "enrichment_mode": "DISABLED"}
worker._features_at = -999
now[0] += 5
worker.step()
add(db, "W1") # created while the function is off
now[0] += 5
worker.step()
hub.features = dict(ENABLED)
worker._features_at = -999
now[0] += 5
assert worker.step() == [] and hub.resolve_calls == [] # re-baselined: W1 is history now
add(db, "W2")
now[0] += 5
assert worker.step() == ["W2"]
+2
View File
@@ -79,6 +79,8 @@ class Settings:
ai_base_url: str = os.getenv("AI_BASE_URL", "https://api.z.ai/api/paas/v4") ai_base_url: str = os.getenv("AI_BASE_URL", "https://api.z.ai/api/paas/v4")
# ---- Product enrichment (docs/product_enrichment.md): free providers only, tokens live ONLY in the Hub environment ---- # ---- Product enrichment (docs/product_enrichment.md): free providers only, tokens live ONLY in the Hub environment ----
cosmos_token: str = os.getenv("COSMOS_TOKEN", "") # optional; without it Cosmos is NOT_CONFIGURED (not an error) cosmos_token: str = os.getenv("COSMOS_TOKEN", "") # optional; without it Cosmos is NOT_CONFIGURED (not an error)
oscbr_user: str = os.getenv("OSCBR_USER", "") # optional free-plan account of gtin.rscsistemas.com.br (never in logs/UI)
oscbr_password: str = os.getenv("OSCBR_PASSWORD", "")
allow_paid_providers: bool = os.getenv("ALLOW_PAID_PROVIDERS", "0") == "1" allow_paid_providers: bool = os.getenv("ALLOW_PAID_PROVIDERS", "0") == "1"
official_refresh_enabled: bool = os.getenv("OFFICIAL_REFRESH_ENABLED", "1") == "1" official_refresh_enabled: bool = os.getenv("OFFICIAL_REFRESH_ENABLED", "1") == "1"
enrichment_time_budget_seconds: float = float(os.getenv("ENRICHMENT_TIME_BUDGET_SECONDS", "4.5")) enrichment_time_budget_seconds: float = float(os.getenv("ENRICHMENT_TIME_BUDGET_SECONDS", "4.5"))
+64 -2
View File
@@ -225,9 +225,71 @@ class Cosmos(Provider):
return out return out
class Oscbr(Provider):
"""OSCBR / RSC Sistemas GTIN API (https://gtin.rscsistemas.com.br). Free plan (no card): 20 req/min; account created BY THE OPERATOR, credentials only in the Hub env
(OSCBR_USER / OSCBR_PASSWORD). Bearer token lasts 1 h and is renewed automatically. Optional: NOT_CONFIGURED without credentials (no call is made)."""
name = "oscbr"
rate_per_minute = 20
provides_ncm = True
base = "https://gtin.rscsistemas.com.br"
def __init__(self, client=None, token=None, user=None, password=None):
super().__init__(client, token)
self._user, self._password = user, password
self._bearer, self._bearer_until = None, 0.0
@property
def configured(self) -> bool:
return bool(self._user and self._password)
def _auth(self) -> str:
if self._bearer and time.time() < self._bearer_until:
return self._bearer
try:
response = self.client.post(f"{self.base}/oauth/token", auth=(self._user, self._password), headers={"User-Agent": USER_AGENT})
except httpx.TimeoutException as error:
raise ProviderError("TIMEOUT", type(error).__name__) from error
except httpx.HTTPError as error:
raise ProviderError("NETWORK", type(error).__name__) from error
if response.status_code in (400, 401, 403):
raise ProviderError("AUTH_ERROR", f"HTTP {response.status_code}")
if response.status_code == 429:
raise ProviderError("RATE_LIMITED", "HTTP 429")
if response.status_code >= 500 or response.status_code != 200:
raise ProviderError("SERVER_ERROR", f"HTTP {response.status_code}")
try:
token = response.json().get("token") or response.json().get("access_token")
except ValueError as error:
raise ProviderError("BAD_RESPONSE", "not JSON") from error
if not token:
raise ProviderError("BAD_RESPONSE", "no token")
self._bearer, self._bearer_until = token, time.time() + 55 * 60
return token
def lookup_gtin(self, gtin: str) -> ProviderProduct:
if not self.configured:
raise ProviderError("AUTH_ERROR", "NOT_CONFIGURED")
response = self._get(f"{self.base}/api/gtin/infor/{gtin}", headers={"Authorization": f"Bearer {self._auth()}", "User-Agent": USER_AGENT})
headers = {k.lower(): v for k, v in response.headers.items()}
if response.status_code == 404:
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
if response.status_code != 200:
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
try:
body = response.json()
except ValueError as error:
raise ProviderError("BAD_RESPONSE", "not JSON") from error
if isinstance(body, list):
body = body[0] if body else {}
if not body or not (body.get("nome") or body.get("ncm")):
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
return ProviderProduct(self.name, gtin, True, title=body.get("nome"), brand=body.get("marca"), category=body.get("categoria"), ncm=_digits(body.get("ncm")), cest=_digits(body.get("cest")),
raw_hash=hashlib.sha256(json.dumps(body, sort_keys=True, ensure_ascii=False).encode()).hexdigest(), latency_ms=getattr(response, "latency_ms", 0), headers=headers)
def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]: def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]:
"""Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here.""" """Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos then OSCBR (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here."""
providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)] providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), Oscbr(client, None, getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False): if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False):
raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)") raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)")
return providers return providers
+1 -1
View File
@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
once published — never a second, independent version number. Fiscal semantics once published — never a second, independent version number. Fiscal semantics
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py, are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump.""" DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
GTI_FISCAL_VERSION = "1.3.0" GTI_FISCAL_VERSION = "1.3.1"
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
+1 -1
View File
@@ -1,6 +1,6 @@
[project] [project]
name = "gti-fiscal-hub" name = "gti-fiscal-hub"
version = "1.3.0" version = "1.3.1"
requires-python = ">=3.10" requires-python = ">=3.10"
dependencies = [ dependencies = [
"fastapi>=0.115,<1", "fastapi>=0.115,<1",
+1 -1
View File
@@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down():
def test_version_endpoint_reports_release_and_fiscal_versions(): def test_version_endpoint_reports_release_and_fiscal_versions():
body = TestClient(create_app(database_probe=lambda: True)).get("/version").json() body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
assert body["version"] == "1.3.0" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"] assert body["version"] == "1.3.1" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
def test_production_refuses_the_dev_token(monkeypatch): def test_production_refuses_the_dev_token(monkeypatch):
@@ -158,3 +158,45 @@ def test_corrector_switch_off_stops_jobs_but_not_enrichment(monkeypatch):
s.commit() s.commit()
assert client.get("/v1/jobs/next", headers=H).json() is None assert client.get("/v1/jobs/next", headers=H).json() is None
assert resolve(client)["state"] != "FEATURE_DISABLED" assert resolve(client)["state"] != "FEATURE_DISABLED"
import pytest
@pytest.mark.parametrize("corrector,mode,configured,jobs,enrich", [
(True, "DISABLED", True, True, False), # Corrector ON / Enrichment OFF
(False, "DRY_RUN", True, False, True), # Corrector OFF / DRY_RUN
(False, "ENABLED", True, False, True), # Corrector OFF / ENABLED
(True, "ENABLED", True, True, True), # both ON
(False, "DISABLED", False, False, False), # no features -> WAITING_FEATURE_CONFIGURATION
])
def test_feature_mode_matrix(monkeypatch, corrector, mode, configured, jobs, enrich):
client, engine = setup(monkeypatch, mode)
with Session(engine) as s:
i = s.scalars(select(Installation)).one()
i.corrector_enabled, i.features_configured = corrector, configured
from central.app.jobs import create_job
from central.app.models import Agent
create_job(s, s.scalars(select(Agent)).one(), "SYNC_INCREMENTAL", {})
s.commit()
assert (client.get("/v1/jobs/next", headers=H).json() is not None) == jobs
assert (resolve(client)["state"] != "FEATURE_DISABLED") == enrich
feats = client.get("/v1/agents/me/authorizations", headers=H).json()["features"]
assert feats["features_configured"] == configured
def test_provider_panel_shows_every_state_and_column(monkeypatch):
from datetime import datetime, timedelta, timezone
from central.app.models import ProviderState, ProviderUsageDaily
client, agent_id, _ = _panel_client()
now = datetime.now(timezone.utc)
with client._session_factory() as s:
s.add(ProviderState(provider="upcitemdb", configured=True, status="RATE_LIMITED", quota_kind="REMOTE_QUOTA", daily_limit=100, remaining=0, reset_at=now + timedelta(hours=3), last_success_at=now, last_error="RATE_LIMITED: HTTP 429", last_latency_ms=140))
s.add(ProviderState(provider="openfoodfacts", configured=True, status="DEGRADED", last_latency_ms=655))
s.add(ProviderState(provider="openproductsfacts", configured=True, status="AUTH_ERROR"))
s.add(ProviderState(provider="openbeautyfacts", configured=True, status="OFFLINE"))
s.add(ProviderUsageDaily(provider="upcitemdb", day=now.strftime("%Y-%m-%d"), requests=100, cache_hits=30, cache_misses=70))
s.commit()
page = client.get("/").text
for label in ("Limite atingido", "Instável", "Credencial recusada", "Indisponível", "Não configurado", "100 / 100", "restam 0", "renova", "último sucesso", "último erro", "140 ms", "cache 30%"):
assert label in page, label
@@ -10,7 +10,7 @@ from sqlalchemy.pool import StaticPool
from central.app.enrichment import kb from central.app.enrichment import kb
from central.app.enrichment.gtin import check_digit_ok from central.app.enrichment.gtin import check_digit_ok
from central.app.enrichment.providers import Cosmos, OpenFoodFacts, Provider, ProviderError, ProviderProduct, UpcItemDb, build_providers from central.app.enrichment.providers import Oscbr, Cosmos, OpenFoodFacts, Provider, ProviderError, ProviderProduct, UpcItemDb, build_providers
from central.app.enrichment.quota import ProviderGateway from central.app.enrichment.quota import ProviderGateway
from central.app.enrichment.resolver import resolve from central.app.enrichment.resolver import resolve
from central.app.models import (Base, CestEntry, CestNcmEntry, Dataset, GtinLookupCache, KbKnowledge, NcmEntry, ProviderUsageDaily, TipiEntry) from central.app.models import (Base, CestEntry, CestNcmEntry, Dataset, GtinLookupCache, KbKnowledge, NcmEntry, ProviderUsageDaily, TipiEntry)
@@ -223,7 +223,7 @@ def test_cosmos_without_token_is_not_configured_and_costs_nothing(env):
def test_paid_provider_guard(): def test_paid_provider_guard():
class S: allow_paid_providers = False; cosmos_token = None class S: allow_paid_providers = False; cosmos_token = None
assert [p.name for p in build_providers(S())] == ["cosmos", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"] assert [p.name for p in build_providers(S())] == ["cosmos", "oscbr", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"]
# ---- provider contract tests against recorded fixtures (httpx.MockTransport, no network) ---------------------------------------------------------- # ---- provider contract tests against recorded fixtures (httpx.MockTransport, no network) ----------------------------------------------------------
@@ -269,3 +269,26 @@ def test_cosmos_contract_and_errors():
with pytest.raises(ProviderError) as e: with pytest.raises(ProviderError) as e:
Cosmos(_client(boom), "tok").lookup_gtin(G1) Cosmos(_client(boom), "tok").lookup_gtin(G1)
assert e.value.kind == "TIMEOUT" assert e.value.kind == "TIMEOUT"
def test_oscbr_optional_and_contract():
calls = []
assert not Oscbr(_client(lambda r: pytest.fail("no call expected"))).configured
with pytest.raises(ProviderError):
Oscbr(_client(lambda r: pytest.fail("no call expected"))).lookup_gtin(G1)
def handler(request):
calls.append(str(request.url))
if request.url.path == "/oauth/token":
assert request.headers["authorization"].startswith("Basic ")
return httpx.Response(200, json={"token": "tkn"})
assert request.headers["authorization"] == "Bearer tkn"
return httpx.Response(200, json={"ean": G1, "nome": "COLA 350ML", "marca": "X", "ncm": "2202.10.00", "cest": "03.007.00"})
p = Oscbr(_client(handler), None, "u", "p")
first, second = p.lookup_gtin(G1), p.lookup_gtin(G1)
assert first.ncm == "22021000" and first.cest == "0300700" and second.found
assert sum(1 for c in calls if "oauth" in c) == 1 # token reused for ~55 min
bad = Oscbr(_client(lambda r: httpx.Response(401)), None, "u", "p")
with pytest.raises(ProviderError) as e:
bad.lookup_gtin(G1)
assert e.value.kind == "AUTH_ERROR"
+1 -1
View File
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0 FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5 POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.0 AGENT_VERSION=1.3.1
+1 -1
View File
@@ -164,7 +164,7 @@ switch ($Command) {
"GTI_HUB_URL=$hub", "AGENT_TOKEN=$token", "AGENT_ID=$agentId", "GTI_HUB_URL=$hub", "AGENT_TOKEN=$token", "AGENT_ID=$agentId",
"COMPANY_CODE=$code", "COMPANY_CNPJ=$cnpj", "COMPANY_CRT_SOURCE=$crtSource", "COMPANY_CODE=$code", "COMPANY_CNPJ=$cnpj", "COMPANY_CRT_SOURCE=$crtSource",
"FIREBIRD_DATABASE_PATH=$db", "FB_USER=$fbUser", "FB_PASSWORD=$fbPass", "FIREBIRD_DATABASE_PATH=$db", "FB_USER=$fbUser", "FB_PASSWORD=$fbPass",
'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.0' 'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.1'
) )
Set-Content -Path $EnvFile -Value $lines -Encoding UTF8 Set-Content -Path $EnvFile -Value $lines -Encoding UTF8
} }
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0 FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5 POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.0 AGENT_VERSION=1.3.1
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0 FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5 POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.0 AGENT_VERSION=1.3.1
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0 FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5 POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.0 AGENT_VERSION=1.3.1
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0 FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5 POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.0 AGENT_VERSION=1.3.1
+1 -1
View File
@@ -1,5 +1,5 @@
# GTI Fiscal Hub — production environment (copy to .env, never commit .env) # GTI Fiscal Hub — production environment (copy to .env, never commit .env)
GTI_FISCAL_VERSION=1.3.0 GTI_FISCAL_VERSION=1.3.1
# REQUIRED. URL-safe secret (used inside a connection URL): openssl rand -hex 24 # REQUIRED. URL-safe secret (used inside a connection URL): openssl rand -hex 24
POSTGRES_PASSWORD= POSTGRES_PASSWORD=
+1 -1
View File
@@ -36,7 +36,7 @@ services:
build: build:
context: ../.. context: ../..
dockerfile: central/Dockerfile dockerfile: central/Dockerfile
image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.0} image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.1}
restart: unless-stopped restart: unless-stopped
depends_on: depends_on:
postgres: postgres:
+183
View File
@@ -0,0 +1,183 @@
{
"artifact": "PRODUCT_ENRICHMENT_FIELD_AUDIT_V1",
"source": "central/app/field_contract.py CONTRACT (all confirmed C000025 write fields)",
"fields_evaluated": 24,
"auto_fill_safe": [
"CLASSIFICACAO_FISCAL",
"IPI",
"CEST"
],
"rows": [
{
"FIELD": "CLASSIFICACAO_FISCAL",
"APPLICABILITY_SOURCE": "dataset NCM (Siscomex) ACTIVE + legacy NCM table",
"DETERMINISTIC_RULE_AVAILABLE": true,
"AUTO_FILL_SAFE": true,
"WHY_NOT": null,
"CONDITION": "only when KB-verified; single source = suggestion"
},
{
"FIELD": "IPI",
"APPLICABILITY_SOURCE": "dataset TIPI_IPI ACTIVE",
"DETERMINISTIC_RULE_AVAILABLE": true,
"AUTO_FILL_SAFE": true,
"WHY_NOT": null,
"CONDITION": "NT/blank rate or several EX rows => not written"
},
{
"FIELD": "EX_TIPI",
"APPLICABILITY_SOURCE": "dataset TIPI_IPI + product description evidence",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "needs the description to evidence the official EX text; ambiguous from name/GTIN -> manual"
},
{
"FIELD": "CEST",
"APPLICABILITY_SOURCE": "dataset CEST (CONFAZ 142/18) ACTIVE + BA_ST_MVA",
"DETERMINISTIC_RULE_AVAILABLE": true,
"AUTO_FILL_SAFE": true,
"WHY_NOT": null,
"CONDITION": "several candidates / description mismatch => not written"
},
{
"FIELD": "SITUACAO_TRIBUTARIA",
"APPLICABILITY_SOURCE": "ST engine (BA Anexo 1 + CEST corroboration) + purchase evidence / operator st_withheld",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on ST withheld / operator icms_treatment (company decision)"
},
{
"FIELD": "CFOP",
"APPLICABILITY_SOURCE": "dataset CFOP oficial + product role + ST",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on product_role, ST, service CFOP (company/operator decision)"
},
{
"FIELD": "CFOP_ENTRADA",
"APPLICABILITY_SOURCE": "dataset CFOP + product role + ST + company entry scope",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on company entry_scope and product role"
},
{
"FIELD": "CST",
"APPLICABILITY_SOURCE": "MOC CST_ICMS + ORIGEM tables + icms_treatment",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on icms_treatment and origin (operator)"
},
{
"FIELD": "CST_ENTRADA",
"APPLICABILITY_SOURCE": "MOC tables + icms_treatment",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on icms_treatment and origin (operator)"
},
{
"FIELD": "CSOSN",
"APPLICABILITY_SOURCE": "MOC CSOSN table + icms_treatment (operator or proven ST)",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on icms_treatment (operator or proven ST)"
},
{
"FIELD": "ALIQUOTA",
"APPLICABILITY_SOURCE": "regime + CSOSN/CST family + operator icms_rate",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on company/group icms_rate"
},
{
"FIELD": "COD_ORIGEM_MERCADORIA",
"APPLICABILITY_SOURCE": "operator origin only (no ERP field carries origin evidence)",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "no evidence source exists; operator only (explicitly never written)"
},
{
"FIELD": "CST_IPI",
"APPLICABILITY_SOURCE": "company C000004.CONTRIBUINTE_IPI",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "company flag decides: non-IPI-taxpayer => NOT_APPLICABLE (nothing to write); taxpayer => operator value"
},
{
"FIELD": "COD_ENTRADA_PIS",
"APPLICABILITY_SOURCE": "MOC CST_PIS + operator pis_cofins_profile",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on operator pis_cofins_profile"
},
{
"FIELD": "COD_SAIDA_PIS",
"APPLICABILITY_SOURCE": "MOC CST_PIS + operator pis_cofins_profile",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on operator pis_cofins_profile"
},
{
"FIELD": "ALIQ_PIS",
"APPLICABILITY_SOURCE": "operator pis_cofins_profile",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on operator pis_cofins_profile"
},
{
"FIELD": "COD_ENTRADA_COFINS",
"APPLICABILITY_SOURCE": "MOC CST_COFINS + operator pis_cofins_profile",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on operator pis_cofins_profile"
},
{
"FIELD": "COD_SAIDA_COFINS",
"APPLICABILITY_SOURCE": "MOC CST_COFINS + operator pis_cofins_profile",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on operator pis_cofins_profile"
},
{
"FIELD": "ALIQ_COFINS",
"APPLICABILITY_SOURCE": "operator pis_cofins_profile",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on operator pis_cofins_profile"
},
{
"FIELD": "INCIDENCIA_PISCOFINS",
"APPLICABILITY_SOURCE": "ERP declaration (NENHUMA / MONOFÁSICA / SUBST.TRIBUTÁRIA) + operator incidence",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "ERP declaration / operator incidence"
},
{
"FIELD": "PISCOFINS",
"APPLICABILITY_SOURCE": "ERP declaration + operator piscofins",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "ERP declaration / operator"
},
{
"FIELD": "IAT",
"APPLICABILITY_SOURCE": "company rounding policy (operator) or ERP declaration",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "company rounding policy (operator)"
},
{
"FIELD": "IPPT",
"APPLICABILITY_SOURCE": "product_role (own production evidence / purchase evidence / operator)",
"DETERMINISTIC_RULE_AVAILABLE": false,
"AUTO_FILL_SAFE": false,
"WHY_NOT": "depends on product_role (own production vs resale) - company decision"
},
{
"FIELD": "CCLASSTRIB",
"APPLICABILITY_SOURCE": "official RTC catalogue + CRT/date gates (existing RTC_CCLASSTRIB_WRITE_V1)",
"DETERMINISTIC_RULE_AVAILABLE": true,
"AUTO_FILL_SAFE": "only through the existing RTC gates/modes",
"WHY_NOT": "not part of PRODUCT_ENRICHMENT_WRITE_V1; the Agent only requests REPROCESS_PRODUCTS and the RTC central mode decides"
}
]
}
+618 -10
View File
@@ -1,5 +1,5 @@
{ {
"measured_at": "2026-09-25T21:53:01Z", "measured_at": "2026-09-26T03:30:10Z",
"sample_source": "lab ERP databases (read-only), valid GTINs only", "sample_source": "lab ERP databases (read-only), valid GTINs only",
"limit_per_provider": 30, "limit_per_provider": 30,
"providers": [ "providers": [
@@ -16,6 +16,23 @@
"errors": 0, "errors": 0,
"error_kinds": {}, "error_kinds": {},
"latencies_ms": [], "latencies_ms": [],
"per_gtin": {},
"status": "NOT_CONFIGURED"
},
{
"provider": "oscbr",
"configured": false,
"sample_size": 0,
"found": 0,
"not_found": 0,
"with_description": 0,
"with_brand": 0,
"with_ncm": 0,
"with_cest": 0,
"errors": 0,
"error_kinds": {},
"latencies_ms": [],
"per_gtin": {},
"status": "NOT_CONFIGURED" "status": "NOT_CONFIGURED"
}, },
{ {
@@ -32,9 +49,47 @@
"error_kinds": { "error_kinds": {
"RATE_LIMITED": 1 "RATE_LIMITED": 1
}, },
"per_gtin": {
"0000000000017": {
"found": true,
"description": true,
"ncm": false,
"cest": false
},
"0000078947269": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"3165140434904": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7509546671574": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891000062722": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891010804626": {
"found": false,
"description": false,
"ncm": false,
"cest": false
}
},
"error_rate": 0.143, "error_rate": 0.143,
"latency_p50_ms": 139.5, "latency_p50_ms": 155.5,
"latency_p95_ms": 140, "latency_p95_ms": 156,
"found_rate": 0.167, "found_rate": 0.167,
"status": "MEASURED" "status": "MEASURED"
}, },
@@ -50,9 +105,191 @@
"with_cest": 0, "with_cest": 0,
"errors": 0, "errors": 0,
"error_kinds": {}, "error_kinds": {},
"per_gtin": {
"0000000000017": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"0000078947269": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"3165140434904": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7509546671574": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891000062722": {
"found": true,
"description": true,
"ncm": false,
"cest": false
},
"7891010804626": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891060773071": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891117029595": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891150066038": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891182019842": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891515901066": {
"found": true,
"description": true,
"ncm": false,
"cest": false
},
"7892261330643": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7893946483739": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7895316729362": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896007200368": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896014183111": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896058257663": {
"found": true,
"description": true,
"ncm": false,
"cest": false
},
"7896110005805": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896229906383": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896306617669": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896451880482": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896798602020": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7897051900570": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7897417402847": {
"found": true,
"description": true,
"ncm": false,
"cest": false
},
"7897751901631": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898016380031": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898128473119": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898219120700": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898322642458": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898402390316": {
"found": false,
"description": false,
"ncm": false,
"cest": false
}
},
"error_rate": 0.0, "error_rate": 0.0,
"latency_p50_ms": 655.0, "latency_p50_ms": 718.0,
"latency_p95_ms": 827, "latency_p95_ms": 796,
"found_rate": 0.133, "found_rate": 0.133,
"status": "MEASURED" "status": "MEASURED"
}, },
@@ -68,9 +305,191 @@
"with_cest": 0, "with_cest": 0,
"errors": 0, "errors": 0,
"error_kinds": {}, "error_kinds": {},
"per_gtin": {
"0000000000017": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"0000078947269": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"3165140434904": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7509546671574": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891000062722": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891010804626": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891060773071": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891117029595": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891150066038": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891182019842": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891515901066": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7892261330643": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7893946483739": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7895316729362": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896007200368": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896014183111": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896058257663": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896110005805": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896229906383": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896306617669": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896451880482": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896798602020": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7897051900570": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7897417402847": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7897751901631": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898016380031": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898128473119": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898219120700": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898322642458": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898402390316": {
"found": false,
"description": false,
"ncm": false,
"cest": false
}
},
"error_rate": 0.0, "error_rate": 0.0,
"latency_p50_ms": 640.0, "latency_p50_ms": 703.0,
"latency_p95_ms": 969, "latency_p95_ms": 953,
"found_rate": 0.0, "found_rate": 0.0,
"status": "MEASURED" "status": "MEASURED"
}, },
@@ -86,11 +505,200 @@
"with_cest": 0, "with_cest": 0,
"errors": 0, "errors": 0,
"error_kinds": {}, "error_kinds": {},
"per_gtin": {
"0000000000017": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"0000078947269": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"3165140434904": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7509546671574": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891000062722": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891010804626": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891060773071": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891117029595": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891150066038": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891182019842": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7891515901066": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7892261330643": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7893946483739": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7895316729362": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896007200368": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896014183111": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896058257663": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896110005805": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896229906383": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896306617669": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896451880482": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7896798602020": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7897051900570": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7897417402847": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7897751901631": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898016380031": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898128473119": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898219120700": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898322642458": {
"found": false,
"description": false,
"ncm": false,
"cest": false
},
"7898402390316": {
"found": false,
"description": false,
"ncm": false,
"cest": false
}
},
"error_rate": 0.0, "error_rate": 0.0,
"latency_p50_ms": 655.5, "latency_p50_ms": 703.0,
"latency_p95_ms": 1311, "latency_p95_ms": 938,
"found_rate": 0.0, "found_rate": 0.0,
"status": "MEASURED" "status": "MEASURED"
} }
] ],
"chain": {
"gtins_tested": 30,
"ANY_PROVIDER_FOUND_RATE": 0.167,
"ANY_PROVIDER_NCM_RATE": 0.0,
"ANY_PROVIDER_CEST_RATE": 0.0,
"ANY_PROVIDER_DESCRIPTION_RATE": 0.167
}
} }
+17
View File
@@ -42,6 +42,7 @@ and `enrichment.is_human_overridden(state_path, code, field)` is the hook every
|---|---|---|---| |---|---|---|---|
| Cosmos (Bluesoft) | `COSMOS_TOKEN` in the Hub env (optional) | 25/day | description, brand, **NCM, CEST** — `NOT_CONFIGURED` without a token (not an error) | | Cosmos (Bluesoft) | `COSMOS_TOKEN` in the Hub env (optional) | 25/day | description, brand, **NCM, CEST** — `NOT_CONFIGURED` without a token (not an error) |
| UPCitemdb (trial) | no key | 100/day, burst 15 / 30 s (headers `X-RateLimit-*`) | title, brand, category | | UPCitemdb (trial) | no key | 100/day, burst 15 / 30 s (headers `X-RateLimit-*`) | title, brand, category |
| OSCBR / RSC Sistemas GTIN | free plan account made by the operator; `OSCBR_USER` / `OSCBR_PASSWORD` in the Hub env; 1 h bearer renewed automatically | 20 req/min | name, brand, category, **NCM** (CEST when present) — `NOT_CONFIGURED` without credentials |
| Open Food Facts / Open Products Facts / Open Beauty Facts | open (ODbL), identifiable `User-Agent` | 15 req/min | title, brand, quantity | | Open Food Facts / Open Products Facts / Open Beauty Facts | open (ODbL), identifiable `User-Agent` | 15 req/min | title, brand, quantity |
Quota is central (`ProviderState`, `ProviderUsageDaily`, `quota_kind = REMOTE_QUOTA | LOCAL_ESTIMATE | DOCUMENTED_RATE`), with positive (30 d) and negative (24 h) `GtinLookupCache`, single-flight per (provider, GTIN), circuit breaker Quota is central (`ProviderState`, `ProviderUsageDaily`, `quota_kind = REMOTE_QUOTA | LOCAL_ESTIMATE | DOCUMENTED_RATE`), with positive (30 d) and negative (24 h) `GtinLookupCache`, single-flight per (provider, GTIN), circuit breaker
(3 failures → 60 s doubling up to 15 min; 429 → until reset; 401/403 → 1 h). Quota exhausted → `PENDING_PROVIDER_QUOTA` + retry after reset. (3 failures → 60 s doubling up to 15 min; 429 → until reset; 401/403 → 1 h). Quota exhausted → `PENDING_PROVIDER_QUOTA` + retry after reset.
@@ -65,3 +66,19 @@ Agent detail: *Funcionalidades* (admin) and *Enriquecimento* (counters + product
second run 0 writes; DRY_RUN 0 writes; WAITING/DISABLED 0 Firebird activity and no baseline; human override incl. deleted value; Hub offline → pending → recovers. second run 0 writes; DRY_RUN 0 writes; WAITING/DISABLED 0 Firebird activity and no baseline; human override incl. deleted value; Hub offline → pending → recovers.
- Migration 0029: fresh-vs-upgrade `pg_dump -s` identical; upgrade → downgrade → upgrade clean. - Migration 0029: fresh-vs-upgrade `pg_dump -s` identical; upgrade → downgrade → upgrade clean.
- Live smoke `scripts/enrichment_live_smoke.py` (1 lookup/provider, real internet, separate from the suite). - Live smoke `scripts/enrichment_live_smoke.py` (1 lookup/provider, real internet, separate from the suite).
## Provider evaluation (2026-09-26)
- **DotCompany — NOT integrated.** The free query (25/day per IP, no signup) is the *website* demo for human use; the API (`/api/v1/catalogo/gtin/{ean}`) is prepaid credits (4 credits = R$ 0,04 per query, packages from R$ 120); the only free API access is a `dc_test_` sandbox with test data. Automated free use is not offered, so no adapter (would violate zero cost / permitted automation).
- **OSCBR / RSC Sistemas — adapter implemented, optional.** Free initial plan without card, 20 req/min, token via account registration (Basic auth on `/oauth/token`, 1 h Bearer), `GET /api/gtin/infor/{gtin}` (fields `ncm`, `nome`, `marca`, `categoria`, `pais`). Terms are shown only inside the authenticated panel, so the operator must create the account and read them; the Hub never creates accounts. Without credentials it is `NOT_CONFIGURED` and makes no request.
- Cosmos stays optional (`COSMOS_TOKEN`). Nothing else qualified (GS1 CNP is a paid membership; Verified by GS1 is not an automation API).
- `PAID_PROVIDER_CALLS=0`; `ALLOW_PAID_PROVIDERS=false`.
## Field audit (`docs/PRODUCT_ENRICHMENT_FIELD_AUDIT_V1.json`)
24 fields checked against `field_contract`; only `CLASSIFICACAO_FISCAL`, `CEST`, `IPI` have a deterministic, company-independent rule. All others need company/operator context (product role, ICMS treatment, origin, PIS/COFINS profile, entry scope, rounding) or description evidence (EX_TIPI) and are excluded on purpose; `CST_IPI` is NOT_APPLICABLE for non-IPI companies; `CCLASSTRIB` stays with the RTC gates.
## Re-baseline after inactivity (1.3.1)
While the function is DISABLED / WAITING_FEATURE_CONFIGURATION the Agent marks a re-baseline; on reactivation the current catalogue becomes history, so a long-disabled installation never sends a burst of products to the providers.
## Production KB seed strategy
Production KB starts empty (0 products, head 0028 before deploy). The lab databases are **not** imported into production: their observations are `CLIENT_OBSERVATION` evidence only and cannot verify anything alone (93/241 shared GTINs disagree on NCM). The KB grows from provider evidence on real new products; `scripts/enrichment_lab_kb.py --seed-url` exists for a dev/staging seed only.
+3 -3
View File
@@ -1,14 +1,14 @@
{ {
"artifact": "RELEASE_MANIFEST", "artifact": "RELEASE_MANIFEST",
"GTI_FISCAL_VERSION": "1.3.0", "GTI_FISCAL_VERSION": "1.3.1",
"generated_at": "2026-09-25T15:43:40+00:00", "generated_at": "2026-09-25T15:43:40+00:00",
"hub": { "hub": {
"version": "1.3.0", "version": "1.3.1",
"api_compat": "1.0", "api_compat": "1.0",
"engine_version": "0.3.0", "engine_version": "0.3.0",
"decision_version": "mission-04c-ipi-ex-split" "decision_version": "mission-04c-ipi-ex-split"
}, },
"agent_version": "1.3.0", "agent_version": "1.3.1",
"fiscal_engine": { "fiscal_engine": {
"ENGINE_VERSION": "0.3.0", "ENGINE_VERSION": "0.3.0",
"DECISION_VERSION": "mission-04c-ipi-ex-split", "DECISION_VERSION": "mission-04c-ipi-ex-split",
+1 -1
View File
@@ -1 +1 @@
1.3.0 1.3.1
+10 -4
View File
@@ -17,13 +17,13 @@ ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
import httpx # noqa: E402 import httpx # noqa: E402
from central.app.enrichment.providers import Cosmos, OpenBeautyFacts, OpenFoodFacts, OpenProductsFacts, ProviderError, UpcItemDb, USER_AGENT # noqa: E402 from central.app.enrichment.providers import Cosmos, Oscbr, OpenBeautyFacts, OpenFoodFacts, OpenProductsFacts, ProviderError, UpcItemDb, USER_AGENT # noqa: E402
PACE = {"upcitemdb": 2.6, "openfoodfacts": 5.2, "openproductsfacts": 5.2, "openbeautyfacts": 5.2, "cosmos": 1.0} # seconds between calls: comfortably under the documented burst / per-minute limits PACE = {"oscbr": 3.2, "upcitemdb": 2.6, "openfoodfacts": 5.2, "openproductsfacts": 5.2, "openbeautyfacts": 5.2, "cosmos": 1.0} # seconds between calls: comfortably under the documented burst / per-minute limits
def bench(provider, items): def bench(provider, items):
out = {"provider": provider.name, "configured": provider.configured, "sample_size": 0, "found": 0, "not_found": 0, "with_description": 0, "with_brand": 0, "with_ncm": 0, "with_cest": 0, "errors": 0, "error_kinds": {}, "latencies_ms": []} out = {"provider": provider.name, "configured": provider.configured, "sample_size": 0, "found": 0, "not_found": 0, "with_description": 0, "with_brand": 0, "with_ncm": 0, "with_cest": 0, "errors": 0, "error_kinds": {}, "latencies_ms": [], "per_gtin": {}}
if not provider.configured: if not provider.configured:
out["status"] = "NOT_CONFIGURED" out["status"] = "NOT_CONFIGURED"
return out return out
@@ -40,6 +40,7 @@ def bench(provider, items):
continue continue
out["sample_size"] += 1 out["sample_size"] += 1
out["latencies_ms"].append(int((time.monotonic() - started) * 1000)) out["latencies_ms"].append(int((time.monotonic() - started) * 1000))
out["per_gtin"][item["gtin"]] = {"found": product.found, "description": bool(product.title), "ncm": bool(product.ncm), "cest": bool(product.cest)}
if product.found: if product.found:
out["found"] += 1 out["found"] += 1
out["with_description"] += bool(product.title) out["with_description"] += bool(product.title)
@@ -65,12 +66,17 @@ def main():
args = ap.parse_args() args = ap.parse_args()
items = json.loads(Path(args.sample).read_text(encoding="utf-8"))[: args.limit] items = json.loads(Path(args.sample).read_text(encoding="utf-8"))[: args.limit]
client = httpx.Client(timeout=8.0, headers={"User-Agent": USER_AGENT}) client = httpx.Client(timeout=8.0, headers={"User-Agent": USER_AGENT})
providers = [Cosmos(client, os.environ.get("COSMOS_TOKEN")), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)] providers = [Cosmos(client, os.environ.get("COSMOS_TOKEN")), Oscbr(client, None, os.environ.get("OSCBR_USER"), os.environ.get("OSCBR_PASSWORD")), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
only = {n for n in args.only.split(",") if n} only = {n for n in args.only.split(",") if n}
out = Path(args.out) out = Path(args.out)
result = json.loads(out.read_text(encoding="utf-8")) if only and out.exists() else {"measured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "sample_source": "lab ERP databases (read-only), valid GTINs only", "limit_per_provider": args.limit, "providers": []} result = json.loads(out.read_text(encoding="utf-8")) if only and out.exists() else {"measured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "sample_source": "lab ERP databases (read-only), valid GTINs only", "limit_per_provider": args.limit, "providers": []}
measured = [bench(p, items) for p in providers if not only or p.name in only] measured = [bench(p, items) for p in providers if not only or p.name in only]
result["providers"] = [p for p in result["providers"] if p["provider"] not in {m["provider"] for m in measured}] + measured result["providers"] = [p for p in result["providers"] if p["provider"] not in {m["provider"] for m in measured}] + measured
per = [p.get("per_gtin", {}) for p in result["providers"]]
tested = sorted({g for d in per for g in d})
def rate(key):
return round(sum(1 for g in tested if any(d.get(g, {}).get(key) for d in per)) / len(tested), 3) if tested else None
result["chain"] = {"gtins_tested": len(tested), "ANY_PROVIDER_FOUND_RATE": rate("found"), "ANY_PROVIDER_NCM_RATE": rate("ncm"), "ANY_PROVIDER_CEST_RATE": rate("cest"), "ANY_PROVIDER_DESCRIPTION_RATE": rate("description")}
Path(args.out).write_text(json.dumps(result, ensure_ascii=False, indent=1), encoding="utf-8") Path(args.out).write_text(json.dumps(result, ensure_ascii=False, indent=1), encoding="utf-8")
print(json.dumps(result, ensure_ascii=False, indent=1)) print(json.dumps(result, ensure_ascii=False, indent=1))
+2 -2
View File
@@ -14,11 +14,11 @@ sys.path.insert(0, str(ROOT))
def test_versions_are_coherent_across_hub_agent_and_packages(): def test_versions_are_coherent_across_hub_agent_and_packages():
from central.app.version import GTI_FISCAL_VERSION from central.app.version import GTI_FISCAL_VERSION
v = (ROOT / "release" / "VERSION").read_text(encoding="utf-8").strip() v = (ROOT / "release" / "VERSION").read_text(encoding="utf-8").strip()
assert v == GTI_FISCAL_VERSION == "1.3.0" assert v == GTI_FISCAL_VERSION == "1.3.1"
pep = v.replace("-rc", "rc") pep = v.replace("-rc", "rc")
for pp in ("central/pyproject.toml", "agent/pyproject.toml"): for pp in ("central/pyproject.toml", "agent/pyproject.toml"):
assert f'version = "{pep}"' in (ROOT / pp).read_text(encoding="utf-8") assert f'version = "{pep}"' in (ROOT / pp).read_text(encoding="utf-8")
assert '"AGENT_VERSION", "1.3.0"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8") assert '"AGENT_VERSION", "1.3.1"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8")
manifest = ROOT / "release" / "RELEASE_MANIFEST.json" manifest = ROOT / "release" / "RELEASE_MANIFEST.json"
if manifest.exists(): if manifest.exists():
m = json.loads(manifest.read_text(encoding="utf-8")) m = json.loads(manifest.read_text(encoding="utf-8"))