From e6bcf12d5362aedfc8b9e17455f9f62e5fea7db2 Mon Sep 17 00:00:00 2001 From: gti Date: Sat, 26 Sep 2026 00:48:18 -0300 Subject: [PATCH] feat: OSCBR optional provider, re-baseline after inactivity, field audit, provider UI/feature matrix tests, benchmark chain; 1.3.1 Co-Authored-By: Claude Sonnet 5 --- agent/pyproject.toml | 2 +- agent/scripts/run_agent_service.py | 6 +- agent/src/gti_fiscal_agent/enrichment.py | 10 + agent/src/gti_fiscal_agent/preflight.py | 2 +- agent/tests/test_enrichment.py | 23 +- central/app/config.py | 2 + central/app/enrichment/providers.py | 66 +- central/app/version.py | 2 +- central/pyproject.toml | 2 +- central/tests/test_health.py | 2 +- central/tests/test_product_enrichment_api.py | 42 ++ .../tests/test_product_enrichment_resolver.py | 27 +- deploy/agent-windows/agent.env.template | 2 +- deploy/agent-windows/gti-agent.ps1 | 2 +- .../pilot-construcao.env.example | 2 +- .../profiles/mmmodas.env.example | 2 +- .../agent-windows/profiles/rincao.env.example | 2 +- .../agent-windows/profiles/uniao.env.example | 2 +- deploy/hub/.env.example | 2 +- deploy/hub/docker-compose.prod.yml | 2 +- docs/PRODUCT_ENRICHMENT_FIELD_AUDIT_V1.json | 183 +++++ docs/PRODUCT_PROVIDER_BENCHMARK_V1.json | 628 +++++++++++++++++- docs/product_enrichment.md | 17 + release/RELEASE_MANIFEST.json | 6 +- release/VERSION | 2 +- scripts/enrichment_benchmark.py | 14 +- tests/test_release_candidate.py | 4 +- 27 files changed, 1015 insertions(+), 41 deletions(-) create mode 100644 docs/PRODUCT_ENRICHMENT_FIELD_AUDIT_V1.json diff --git a/agent/pyproject.toml b/agent/pyproject.toml index 89a595c..be78fe0 100644 --- a/agent/pyproject.toml +++ b/agent/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "gti-fiscal-agent" -version = "1.3.0" +version = "1.3.1" requires-python = ">=3.10" dependencies = ["httpx>=0.27,<1"] diff --git a/agent/scripts/run_agent_service.py b/agent/scripts/run_agent_service.py index 8d0ec21..fa23456 100644 --- a/agent/scripts/run_agent_service.py +++ b/agent/scripts/run_agent_service.py @@ -107,7 +107,7 @@ class Config: self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5")) # RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib. self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB))) - self.agent_version = env.get("AGENT_VERSION", "1.3.0") + self.agent_version = env.get("AGENT_VERSION", "1.3.1") self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1) self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite")) # Pilot write authorization: both must be explicitly set by the installer for @@ -349,7 +349,7 @@ def execute_job(job, config, executor, state, env): "fiscal_context": fiscal_context, "rtc_auto_apply": rtc_apply, "fiscal_mode": getattr(config, "fiscal_mode", None), - "release_version": env.get("AGENT_VERSION") or "1.3.0", + "release_version": env.get("AGENT_VERSION") or "1.3.1", "products_total": len(products), "products_changed": len(to_process), "products_processed": summary["processed"], @@ -520,7 +520,7 @@ def run(config, env, *, max_iterations=None, stopper=None): def enforce_startup_preflight(env): """Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed.""" from gti_fiscal_agent.preflight import run_preflight - report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.0") + report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.1") logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False)) print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False)) if report["PREFLIGHT"] != "PASS": diff --git a/agent/src/gti_fiscal_agent/enrichment.py b/agent/src/gti_fiscal_agent/enrichment.py index e635ff1..a124e8d 100644 --- a/agent/src/gti_fiscal_agent/enrichment.py +++ b/agent/src/gti_fiscal_agent/enrichment.py @@ -271,6 +271,14 @@ class EnrichmentWorker: self.store.set_meta("baseline_count", len(codes)) self._last_count, self._last_diff = len(codes), self.clock() return [] + if self.store.meta("rebaseline") == "1": + # the function was inactive (DISABLED / WAITING_FEATURE_CONFIGURATION) for a while: whatever appeared meanwhile is NOT "new" for the enrichment + # (a long-disabled installation must never send a burst of products to the providers when it is switched on again) + codes = read_codes(conn) + self.store.add_known(codes) + self.store.set_meta("rebaseline", "0") + self._last_count, self._last_diff = len(codes), self.clock() + return [] started = self.clock() t0 = time.monotonic() count = read_count(conn) @@ -417,6 +425,8 @@ class EnrichmentWorker: """One iteration; returns the list of processed product codes.""" mode = self.mode() if mode not in ("DRY_RUN", "ENABLED"): + if self.store.meta("baseline_done") == "1": + self.store.set_meta("rebaseline", "1") return [] # WAITING_FEATURE_CONFIGURATION / DISABLED: zero Firebird activity self.detect() done = [] diff --git a/agent/src/gti_fiscal_agent/preflight.py b/agent/src/gti_fiscal_agent/preflight.py index e065b42..cfc30bd 100644 --- a/agent/src/gti_fiscal_agent/preflight.py +++ b/agent/src/gti_fiscal_agent/preflight.py @@ -9,7 +9,7 @@ import os import sqlite3 from pathlib import Path -AGENT_VERSION_DEFAULT = "1.3.0" +AGENT_VERSION_DEFAULT = "1.3.1" REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD") REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest") FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY") diff --git a/agent/tests/test_enrichment.py b/agent/tests/test_enrichment.py index 1df92e3..aa12d21 100644 --- a/agent/tests/test_enrichment.py +++ b/agent/tests/test_enrichment.py @@ -124,7 +124,6 @@ def test_existing_value_is_never_overwritten(env): def test_dry_run_and_disabled_and_waiting_do_not_write_or_touch_firebird(env): db, hub, store, worker, now, _ = env worker.step() - add(db, "N3") for features in ({"features_configured": False}, {"features_configured": True, "enrichment_mode": "DISABLED"}): hub.features = features worker._features_at = -999 @@ -132,6 +131,9 @@ def test_dry_run_and_disabled_and_waiting_do_not_write_or_touch_firebird(env): hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"} worker._features_at = -999 now[0] += 5 + worker.step() # re-baseline after the inactive period + add(db, "N3") + now[0] += 5 worker.step() assert row(db, "N3") == (None, None, None, "keep") and hub.reports[0]["fields"][0]["outcome"] == "DRY_RUN" @@ -209,3 +211,22 @@ def test_gtin_check_parity_with_the_hub_and_codbarra_is_used(): assert E.pick_gtin({"GTIN": "", "CODBARRA": "7891000100103"}) == "7891000100103" assert E.pick_gtin({"GTIN": "7891000100103", "CODBARRA": "999"}) == "7891000100103" assert E.pick_gtin({"GTIN": "", "CODBARRA": "123"}) == "123" and E.pick_gtin({}) == "" + + +def test_products_created_while_inactive_are_not_new_when_reactivated(env): + db, hub, store, worker, now, _ = env + worker.step() # baseline + hub.features = {"features_configured": True, "enrichment_mode": "DISABLED"} + worker._features_at = -999 + now[0] += 5 + worker.step() + add(db, "W1") # created while the function is off + now[0] += 5 + worker.step() + hub.features = dict(ENABLED) + worker._features_at = -999 + now[0] += 5 + assert worker.step() == [] and hub.resolve_calls == [] # re-baselined: W1 is history now + add(db, "W2") + now[0] += 5 + assert worker.step() == ["W2"] diff --git a/central/app/config.py b/central/app/config.py index 04111cc..b66f15d 100644 --- a/central/app/config.py +++ b/central/app/config.py @@ -79,6 +79,8 @@ class Settings: ai_base_url: str = os.getenv("AI_BASE_URL", "https://api.z.ai/api/paas/v4") # ---- Product enrichment (docs/product_enrichment.md): free providers only, tokens live ONLY in the Hub environment ---- cosmos_token: str = os.getenv("COSMOS_TOKEN", "") # optional; without it Cosmos is NOT_CONFIGURED (not an error) + oscbr_user: str = os.getenv("OSCBR_USER", "") # optional free-plan account of gtin.rscsistemas.com.br (never in logs/UI) + oscbr_password: str = os.getenv("OSCBR_PASSWORD", "") allow_paid_providers: bool = os.getenv("ALLOW_PAID_PROVIDERS", "0") == "1" official_refresh_enabled: bool = os.getenv("OFFICIAL_REFRESH_ENABLED", "1") == "1" enrichment_time_budget_seconds: float = float(os.getenv("ENRICHMENT_TIME_BUDGET_SECONDS", "4.5")) diff --git a/central/app/enrichment/providers.py b/central/app/enrichment/providers.py index 9f9c3cd..8524b35 100644 --- a/central/app/enrichment/providers.py +++ b/central/app/enrichment/providers.py @@ -225,9 +225,71 @@ class Cosmos(Provider): return out +class Oscbr(Provider): + """OSCBR / RSC Sistemas GTIN API (https://gtin.rscsistemas.com.br). Free plan (no card): 20 req/min; account created BY THE OPERATOR, credentials only in the Hub env + (OSCBR_USER / OSCBR_PASSWORD). Bearer token lasts 1 h and is renewed automatically. Optional: NOT_CONFIGURED without credentials (no call is made).""" + name = "oscbr" + rate_per_minute = 20 + provides_ncm = True + base = "https://gtin.rscsistemas.com.br" + + def __init__(self, client=None, token=None, user=None, password=None): + super().__init__(client, token) + self._user, self._password = user, password + self._bearer, self._bearer_until = None, 0.0 + + @property + def configured(self) -> bool: + return bool(self._user and self._password) + + def _auth(self) -> str: + if self._bearer and time.time() < self._bearer_until: + return self._bearer + try: + response = self.client.post(f"{self.base}/oauth/token", auth=(self._user, self._password), headers={"User-Agent": USER_AGENT}) + except httpx.TimeoutException as error: + raise ProviderError("TIMEOUT", type(error).__name__) from error + except httpx.HTTPError as error: + raise ProviderError("NETWORK", type(error).__name__) from error + if response.status_code in (400, 401, 403): + raise ProviderError("AUTH_ERROR", f"HTTP {response.status_code}") + if response.status_code == 429: + raise ProviderError("RATE_LIMITED", "HTTP 429") + if response.status_code >= 500 or response.status_code != 200: + raise ProviderError("SERVER_ERROR", f"HTTP {response.status_code}") + try: + token = response.json().get("token") or response.json().get("access_token") + except ValueError as error: + raise ProviderError("BAD_RESPONSE", "not JSON") from error + if not token: + raise ProviderError("BAD_RESPONSE", "no token") + self._bearer, self._bearer_until = token, time.time() + 55 * 60 + return token + + def lookup_gtin(self, gtin: str) -> ProviderProduct: + if not self.configured: + raise ProviderError("AUTH_ERROR", "NOT_CONFIGURED") + response = self._get(f"{self.base}/api/gtin/infor/{gtin}", headers={"Authorization": f"Bearer {self._auth()}", "User-Agent": USER_AGENT}) + headers = {k.lower(): v for k, v in response.headers.items()} + if response.status_code == 404: + return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers) + if response.status_code != 200: + raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}") + try: + body = response.json() + except ValueError as error: + raise ProviderError("BAD_RESPONSE", "not JSON") from error + if isinstance(body, list): + body = body[0] if body else {} + if not body or not (body.get("nome") or body.get("ncm")): + return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers) + return ProviderProduct(self.name, gtin, True, title=body.get("nome"), brand=body.get("marca"), category=body.get("categoria"), ncm=_digits(body.get("ncm")), cest=_digits(body.get("cest")), + raw_hash=hashlib.sha256(json.dumps(body, sort_keys=True, ensure_ascii=False).encode()).hexdigest(), latency_ms=getattr(response, "latency_ms", 0), headers=headers) + + def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]: - """Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here.""" - providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)] + """Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos then OSCBR (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here.""" + providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), Oscbr(client, None, getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)] if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False): raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)") return providers diff --git a/central/app/version.py b/central/app/version.py index 40155f6..30b9747 100644 --- a/central/app/version.py +++ b/central/app/version.py @@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds once published — never a second, independent version number. Fiscal semantics are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py, DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump.""" -GTI_FISCAL_VERSION = "1.3.0" +GTI_FISCAL_VERSION = "1.3.1" API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x diff --git a/central/pyproject.toml b/central/pyproject.toml index aa31279..8b7eed0 100644 --- a/central/pyproject.toml +++ b/central/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "gti-fiscal-hub" -version = "1.3.0" +version = "1.3.1" requires-python = ">=3.10" dependencies = [ "fastapi>=0.115,<1", diff --git a/central/tests/test_health.py b/central/tests/test_health.py index e5b6d79..2281c2c 100644 --- a/central/tests/test_health.py +++ b/central/tests/test_health.py @@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down(): def test_version_endpoint_reports_release_and_fiscal_versions(): body = TestClient(create_app(database_probe=lambda: True)).get("/version").json() - assert body["version"] == "1.3.0" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"] + assert body["version"] == "1.3.1" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"] def test_production_refuses_the_dev_token(monkeypatch): diff --git a/central/tests/test_product_enrichment_api.py b/central/tests/test_product_enrichment_api.py index 074c58a..4bd56ae 100644 --- a/central/tests/test_product_enrichment_api.py +++ b/central/tests/test_product_enrichment_api.py @@ -158,3 +158,45 @@ def test_corrector_switch_off_stops_jobs_but_not_enrichment(monkeypatch): s.commit() assert client.get("/v1/jobs/next", headers=H).json() is None assert resolve(client)["state"] != "FEATURE_DISABLED" + + +import pytest + + +@pytest.mark.parametrize("corrector,mode,configured,jobs,enrich", [ + (True, "DISABLED", True, True, False), # Corrector ON / Enrichment OFF + (False, "DRY_RUN", True, False, True), # Corrector OFF / DRY_RUN + (False, "ENABLED", True, False, True), # Corrector OFF / ENABLED + (True, "ENABLED", True, True, True), # both ON + (False, "DISABLED", False, False, False), # no features -> WAITING_FEATURE_CONFIGURATION +]) +def test_feature_mode_matrix(monkeypatch, corrector, mode, configured, jobs, enrich): + client, engine = setup(monkeypatch, mode) + with Session(engine) as s: + i = s.scalars(select(Installation)).one() + i.corrector_enabled, i.features_configured = corrector, configured + from central.app.jobs import create_job + from central.app.models import Agent + create_job(s, s.scalars(select(Agent)).one(), "SYNC_INCREMENTAL", {}) + s.commit() + assert (client.get("/v1/jobs/next", headers=H).json() is not None) == jobs + assert (resolve(client)["state"] != "FEATURE_DISABLED") == enrich + feats = client.get("/v1/agents/me/authorizations", headers=H).json()["features"] + assert feats["features_configured"] == configured + + +def test_provider_panel_shows_every_state_and_column(monkeypatch): + from datetime import datetime, timedelta, timezone + from central.app.models import ProviderState, ProviderUsageDaily + client, agent_id, _ = _panel_client() + now = datetime.now(timezone.utc) + with client._session_factory() as s: + s.add(ProviderState(provider="upcitemdb", configured=True, status="RATE_LIMITED", quota_kind="REMOTE_QUOTA", daily_limit=100, remaining=0, reset_at=now + timedelta(hours=3), last_success_at=now, last_error="RATE_LIMITED: HTTP 429", last_latency_ms=140)) + s.add(ProviderState(provider="openfoodfacts", configured=True, status="DEGRADED", last_latency_ms=655)) + s.add(ProviderState(provider="openproductsfacts", configured=True, status="AUTH_ERROR")) + s.add(ProviderState(provider="openbeautyfacts", configured=True, status="OFFLINE")) + s.add(ProviderUsageDaily(provider="upcitemdb", day=now.strftime("%Y-%m-%d"), requests=100, cache_hits=30, cache_misses=70)) + s.commit() + page = client.get("/").text + for label in ("Limite atingido", "Instável", "Credencial recusada", "Indisponível", "Não configurado", "100 / 100", "restam 0", "renova", "último sucesso", "último erro", "140 ms", "cache 30%"): + assert label in page, label diff --git a/central/tests/test_product_enrichment_resolver.py b/central/tests/test_product_enrichment_resolver.py index 33d5ee8..7c66f99 100644 --- a/central/tests/test_product_enrichment_resolver.py +++ b/central/tests/test_product_enrichment_resolver.py @@ -10,7 +10,7 @@ from sqlalchemy.pool import StaticPool from central.app.enrichment import kb from central.app.enrichment.gtin import check_digit_ok -from central.app.enrichment.providers import Cosmos, OpenFoodFacts, Provider, ProviderError, ProviderProduct, UpcItemDb, build_providers +from central.app.enrichment.providers import Oscbr, Cosmos, OpenFoodFacts, Provider, ProviderError, ProviderProduct, UpcItemDb, build_providers from central.app.enrichment.quota import ProviderGateway from central.app.enrichment.resolver import resolve from central.app.models import (Base, CestEntry, CestNcmEntry, Dataset, GtinLookupCache, KbKnowledge, NcmEntry, ProviderUsageDaily, TipiEntry) @@ -223,7 +223,7 @@ def test_cosmos_without_token_is_not_configured_and_costs_nothing(env): def test_paid_provider_guard(): class S: allow_paid_providers = False; cosmos_token = None - assert [p.name for p in build_providers(S())] == ["cosmos", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"] + assert [p.name for p in build_providers(S())] == ["cosmos", "oscbr", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"] # ---- provider contract tests against recorded fixtures (httpx.MockTransport, no network) ---------------------------------------------------------- @@ -269,3 +269,26 @@ def test_cosmos_contract_and_errors(): with pytest.raises(ProviderError) as e: Cosmos(_client(boom), "tok").lookup_gtin(G1) assert e.value.kind == "TIMEOUT" + + +def test_oscbr_optional_and_contract(): + calls = [] + assert not Oscbr(_client(lambda r: pytest.fail("no call expected"))).configured + with pytest.raises(ProviderError): + Oscbr(_client(lambda r: pytest.fail("no call expected"))).lookup_gtin(G1) + + def handler(request): + calls.append(str(request.url)) + if request.url.path == "/oauth/token": + assert request.headers["authorization"].startswith("Basic ") + return httpx.Response(200, json={"token": "tkn"}) + assert request.headers["authorization"] == "Bearer tkn" + return httpx.Response(200, json={"ean": G1, "nome": "COLA 350ML", "marca": "X", "ncm": "2202.10.00", "cest": "03.007.00"}) + p = Oscbr(_client(handler), None, "u", "p") + first, second = p.lookup_gtin(G1), p.lookup_gtin(G1) + assert first.ncm == "22021000" and first.cest == "0300700" and second.found + assert sum(1 for c in calls if "oauth" in c) == 1 # token reused for ~55 min + bad = Oscbr(_client(lambda r: httpx.Response(401)), None, "u", "p") + with pytest.raises(ProviderError) as e: + bad.lookup_gtin(G1) + assert e.value.kind == "AUTH_ERROR" diff --git a/deploy/agent-windows/agent.env.template b/deploy/agent-windows/agent.env.template index 620f416..23d77ab 100644 --- a/deploy/agent-windows/agent.env.template +++ b/deploy/agent-windows/agent.env.template @@ -19,4 +19,4 @@ FB_PASSWORD= FISCAL_MODE=DRY_RUN FIREBIRD_WRITES=0 POLL_INTERVAL_SECONDS=5 -AGENT_VERSION=1.3.0 +AGENT_VERSION=1.3.1 diff --git a/deploy/agent-windows/gti-agent.ps1 b/deploy/agent-windows/gti-agent.ps1 index fb9546a..afce1c1 100644 --- a/deploy/agent-windows/gti-agent.ps1 +++ b/deploy/agent-windows/gti-agent.ps1 @@ -164,7 +164,7 @@ switch ($Command) { "GTI_HUB_URL=$hub", "AGENT_TOKEN=$token", "AGENT_ID=$agentId", "COMPANY_CODE=$code", "COMPANY_CNPJ=$cnpj", "COMPANY_CRT_SOURCE=$crtSource", "FIREBIRD_DATABASE_PATH=$db", "FB_USER=$fbUser", "FB_PASSWORD=$fbPass", - 'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.0' + 'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.1' ) Set-Content -Path $EnvFile -Value $lines -Encoding UTF8 } diff --git a/deploy/agent-windows/pilot-construcao.env.example b/deploy/agent-windows/pilot-construcao.env.example index 55f8b42..d67c8cf 100644 --- a/deploy/agent-windows/pilot-construcao.env.example +++ b/deploy/agent-windows/pilot-construcao.env.example @@ -19,4 +19,4 @@ FB_PASSWORD= FISCAL_MODE=DRY_RUN FIREBIRD_WRITES=0 POLL_INTERVAL_SECONDS=5 -AGENT_VERSION=1.3.0 +AGENT_VERSION=1.3.1 diff --git a/deploy/agent-windows/profiles/mmmodas.env.example b/deploy/agent-windows/profiles/mmmodas.env.example index 2989cd6..d45f14f 100644 --- a/deploy/agent-windows/profiles/mmmodas.env.example +++ b/deploy/agent-windows/profiles/mmmodas.env.example @@ -19,4 +19,4 @@ FB_PASSWORD= FISCAL_MODE=DRY_RUN FIREBIRD_WRITES=0 POLL_INTERVAL_SECONDS=5 -AGENT_VERSION=1.3.0 +AGENT_VERSION=1.3.1 diff --git a/deploy/agent-windows/profiles/rincao.env.example b/deploy/agent-windows/profiles/rincao.env.example index 4309ad1..80d7a32 100644 --- a/deploy/agent-windows/profiles/rincao.env.example +++ b/deploy/agent-windows/profiles/rincao.env.example @@ -19,4 +19,4 @@ FB_PASSWORD= FISCAL_MODE=DRY_RUN FIREBIRD_WRITES=0 POLL_INTERVAL_SECONDS=5 -AGENT_VERSION=1.3.0 +AGENT_VERSION=1.3.1 diff --git a/deploy/agent-windows/profiles/uniao.env.example b/deploy/agent-windows/profiles/uniao.env.example index 32f99fa..a2c1080 100644 --- a/deploy/agent-windows/profiles/uniao.env.example +++ b/deploy/agent-windows/profiles/uniao.env.example @@ -19,4 +19,4 @@ FB_PASSWORD= FISCAL_MODE=DRY_RUN FIREBIRD_WRITES=0 POLL_INTERVAL_SECONDS=5 -AGENT_VERSION=1.3.0 +AGENT_VERSION=1.3.1 diff --git a/deploy/hub/.env.example b/deploy/hub/.env.example index 9550e08..2137dce 100644 --- a/deploy/hub/.env.example +++ b/deploy/hub/.env.example @@ -1,5 +1,5 @@ # GTI Fiscal Hub — production environment (copy to .env, never commit .env) -GTI_FISCAL_VERSION=1.3.0 +GTI_FISCAL_VERSION=1.3.1 # REQUIRED. URL-safe secret (used inside a connection URL): openssl rand -hex 24 POSTGRES_PASSWORD= diff --git a/deploy/hub/docker-compose.prod.yml b/deploy/hub/docker-compose.prod.yml index bf386dc..88929ab 100644 --- a/deploy/hub/docker-compose.prod.yml +++ b/deploy/hub/docker-compose.prod.yml @@ -36,7 +36,7 @@ services: build: context: ../.. dockerfile: central/Dockerfile - image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.0} + image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.1} restart: unless-stopped depends_on: postgres: diff --git a/docs/PRODUCT_ENRICHMENT_FIELD_AUDIT_V1.json b/docs/PRODUCT_ENRICHMENT_FIELD_AUDIT_V1.json new file mode 100644 index 0000000..cba053f --- /dev/null +++ b/docs/PRODUCT_ENRICHMENT_FIELD_AUDIT_V1.json @@ -0,0 +1,183 @@ +{ + "artifact": "PRODUCT_ENRICHMENT_FIELD_AUDIT_V1", + "source": "central/app/field_contract.py CONTRACT (all confirmed C000025 write fields)", + "fields_evaluated": 24, + "auto_fill_safe": [ + "CLASSIFICACAO_FISCAL", + "IPI", + "CEST" + ], + "rows": [ + { + "FIELD": "CLASSIFICACAO_FISCAL", + "APPLICABILITY_SOURCE": "dataset NCM (Siscomex) ACTIVE + legacy NCM table", + "DETERMINISTIC_RULE_AVAILABLE": true, + "AUTO_FILL_SAFE": true, + "WHY_NOT": null, + "CONDITION": "only when KB-verified; single source = suggestion" + }, + { + "FIELD": "IPI", + "APPLICABILITY_SOURCE": "dataset TIPI_IPI ACTIVE", + "DETERMINISTIC_RULE_AVAILABLE": true, + "AUTO_FILL_SAFE": true, + "WHY_NOT": null, + "CONDITION": "NT/blank rate or several EX rows => not written" + }, + { + "FIELD": "EX_TIPI", + "APPLICABILITY_SOURCE": "dataset TIPI_IPI + product description evidence", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "needs the description to evidence the official EX text; ambiguous from name/GTIN -> manual" + }, + { + "FIELD": "CEST", + "APPLICABILITY_SOURCE": "dataset CEST (CONFAZ 142/18) ACTIVE + BA_ST_MVA", + "DETERMINISTIC_RULE_AVAILABLE": true, + "AUTO_FILL_SAFE": true, + "WHY_NOT": null, + "CONDITION": "several candidates / description mismatch => not written" + }, + { + "FIELD": "SITUACAO_TRIBUTARIA", + "APPLICABILITY_SOURCE": "ST engine (BA Anexo 1 + CEST corroboration) + purchase evidence / operator st_withheld", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on ST withheld / operator icms_treatment (company decision)" + }, + { + "FIELD": "CFOP", + "APPLICABILITY_SOURCE": "dataset CFOP oficial + product role + ST", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on product_role, ST, service CFOP (company/operator decision)" + }, + { + "FIELD": "CFOP_ENTRADA", + "APPLICABILITY_SOURCE": "dataset CFOP + product role + ST + company entry scope", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on company entry_scope and product role" + }, + { + "FIELD": "CST", + "APPLICABILITY_SOURCE": "MOC CST_ICMS + ORIGEM tables + icms_treatment", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on icms_treatment and origin (operator)" + }, + { + "FIELD": "CST_ENTRADA", + "APPLICABILITY_SOURCE": "MOC tables + icms_treatment", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on icms_treatment and origin (operator)" + }, + { + "FIELD": "CSOSN", + "APPLICABILITY_SOURCE": "MOC CSOSN table + icms_treatment (operator or proven ST)", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on icms_treatment (operator or proven ST)" + }, + { + "FIELD": "ALIQUOTA", + "APPLICABILITY_SOURCE": "regime + CSOSN/CST family + operator icms_rate", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on company/group icms_rate" + }, + { + "FIELD": "COD_ORIGEM_MERCADORIA", + "APPLICABILITY_SOURCE": "operator origin only (no ERP field carries origin evidence)", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "no evidence source exists; operator only (explicitly never written)" + }, + { + "FIELD": "CST_IPI", + "APPLICABILITY_SOURCE": "company C000004.CONTRIBUINTE_IPI", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "company flag decides: non-IPI-taxpayer => NOT_APPLICABLE (nothing to write); taxpayer => operator value" + }, + { + "FIELD": "COD_ENTRADA_PIS", + "APPLICABILITY_SOURCE": "MOC CST_PIS + operator pis_cofins_profile", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on operator pis_cofins_profile" + }, + { + "FIELD": "COD_SAIDA_PIS", + "APPLICABILITY_SOURCE": "MOC CST_PIS + operator pis_cofins_profile", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on operator pis_cofins_profile" + }, + { + "FIELD": "ALIQ_PIS", + "APPLICABILITY_SOURCE": "operator pis_cofins_profile", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on operator pis_cofins_profile" + }, + { + "FIELD": "COD_ENTRADA_COFINS", + "APPLICABILITY_SOURCE": "MOC CST_COFINS + operator pis_cofins_profile", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on operator pis_cofins_profile" + }, + { + "FIELD": "COD_SAIDA_COFINS", + "APPLICABILITY_SOURCE": "MOC CST_COFINS + operator pis_cofins_profile", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on operator pis_cofins_profile" + }, + { + "FIELD": "ALIQ_COFINS", + "APPLICABILITY_SOURCE": "operator pis_cofins_profile", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on operator pis_cofins_profile" + }, + { + "FIELD": "INCIDENCIA_PISCOFINS", + "APPLICABILITY_SOURCE": "ERP declaration (NENHUMA / MONOFÁSICA / SUBST.TRIBUTÁRIA) + operator incidence", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "ERP declaration / operator incidence" + }, + { + "FIELD": "PISCOFINS", + "APPLICABILITY_SOURCE": "ERP declaration + operator piscofins", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "ERP declaration / operator" + }, + { + "FIELD": "IAT", + "APPLICABILITY_SOURCE": "company rounding policy (operator) or ERP declaration", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "company rounding policy (operator)" + }, + { + "FIELD": "IPPT", + "APPLICABILITY_SOURCE": "product_role (own production evidence / purchase evidence / operator)", + "DETERMINISTIC_RULE_AVAILABLE": false, + "AUTO_FILL_SAFE": false, + "WHY_NOT": "depends on product_role (own production vs resale) - company decision" + }, + { + "FIELD": "CCLASSTRIB", + "APPLICABILITY_SOURCE": "official RTC catalogue + CRT/date gates (existing RTC_CCLASSTRIB_WRITE_V1)", + "DETERMINISTIC_RULE_AVAILABLE": true, + "AUTO_FILL_SAFE": "only through the existing RTC gates/modes", + "WHY_NOT": "not part of PRODUCT_ENRICHMENT_WRITE_V1; the Agent only requests REPROCESS_PRODUCTS and the RTC central mode decides" + } + ] +} \ No newline at end of file diff --git a/docs/PRODUCT_PROVIDER_BENCHMARK_V1.json b/docs/PRODUCT_PROVIDER_BENCHMARK_V1.json index b41c995..73e39ab 100644 --- a/docs/PRODUCT_PROVIDER_BENCHMARK_V1.json +++ b/docs/PRODUCT_PROVIDER_BENCHMARK_V1.json @@ -1,5 +1,5 @@ { - "measured_at": "2026-09-25T21:53:01Z", + "measured_at": "2026-09-26T03:30:10Z", "sample_source": "lab ERP databases (read-only), valid GTINs only", "limit_per_provider": 30, "providers": [ @@ -16,6 +16,23 @@ "errors": 0, "error_kinds": {}, "latencies_ms": [], + "per_gtin": {}, + "status": "NOT_CONFIGURED" + }, + { + "provider": "oscbr", + "configured": false, + "sample_size": 0, + "found": 0, + "not_found": 0, + "with_description": 0, + "with_brand": 0, + "with_ncm": 0, + "with_cest": 0, + "errors": 0, + "error_kinds": {}, + "latencies_ms": [], + "per_gtin": {}, "status": "NOT_CONFIGURED" }, { @@ -32,9 +49,47 @@ "error_kinds": { "RATE_LIMITED": 1 }, + "per_gtin": { + "0000000000017": { + "found": true, + "description": true, + "ncm": false, + "cest": false + }, + "0000078947269": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "3165140434904": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7509546671574": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891000062722": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891010804626": { + "found": false, + "description": false, + "ncm": false, + "cest": false + } + }, "error_rate": 0.143, - "latency_p50_ms": 139.5, - "latency_p95_ms": 140, + "latency_p50_ms": 155.5, + "latency_p95_ms": 156, "found_rate": 0.167, "status": "MEASURED" }, @@ -50,9 +105,191 @@ "with_cest": 0, "errors": 0, "error_kinds": {}, + "per_gtin": { + "0000000000017": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "0000078947269": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "3165140434904": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7509546671574": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891000062722": { + "found": true, + "description": true, + "ncm": false, + "cest": false + }, + "7891010804626": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891060773071": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891117029595": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891150066038": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891182019842": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891515901066": { + "found": true, + "description": true, + "ncm": false, + "cest": false + }, + "7892261330643": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7893946483739": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7895316729362": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896007200368": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896014183111": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896058257663": { + "found": true, + "description": true, + "ncm": false, + "cest": false + }, + "7896110005805": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896229906383": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896306617669": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896451880482": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896798602020": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7897051900570": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7897417402847": { + "found": true, + "description": true, + "ncm": false, + "cest": false + }, + "7897751901631": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898016380031": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898128473119": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898219120700": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898322642458": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898402390316": { + "found": false, + "description": false, + "ncm": false, + "cest": false + } + }, "error_rate": 0.0, - "latency_p50_ms": 655.0, - "latency_p95_ms": 827, + "latency_p50_ms": 718.0, + "latency_p95_ms": 796, "found_rate": 0.133, "status": "MEASURED" }, @@ -68,9 +305,191 @@ "with_cest": 0, "errors": 0, "error_kinds": {}, + "per_gtin": { + "0000000000017": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "0000078947269": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "3165140434904": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7509546671574": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891000062722": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891010804626": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891060773071": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891117029595": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891150066038": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891182019842": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891515901066": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7892261330643": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7893946483739": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7895316729362": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896007200368": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896014183111": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896058257663": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896110005805": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896229906383": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896306617669": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896451880482": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896798602020": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7897051900570": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7897417402847": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7897751901631": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898016380031": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898128473119": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898219120700": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898322642458": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898402390316": { + "found": false, + "description": false, + "ncm": false, + "cest": false + } + }, "error_rate": 0.0, - "latency_p50_ms": 640.0, - "latency_p95_ms": 969, + "latency_p50_ms": 703.0, + "latency_p95_ms": 953, "found_rate": 0.0, "status": "MEASURED" }, @@ -86,11 +505,200 @@ "with_cest": 0, "errors": 0, "error_kinds": {}, + "per_gtin": { + "0000000000017": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "0000078947269": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "3165140434904": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7509546671574": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891000062722": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891010804626": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891060773071": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891117029595": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891150066038": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891182019842": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7891515901066": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7892261330643": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7893946483739": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7895316729362": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896007200368": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896014183111": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896058257663": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896110005805": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896229906383": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896306617669": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896451880482": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7896798602020": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7897051900570": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7897417402847": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7897751901631": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898016380031": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898128473119": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898219120700": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898322642458": { + "found": false, + "description": false, + "ncm": false, + "cest": false + }, + "7898402390316": { + "found": false, + "description": false, + "ncm": false, + "cest": false + } + }, "error_rate": 0.0, - "latency_p50_ms": 655.5, - "latency_p95_ms": 1311, + "latency_p50_ms": 703.0, + "latency_p95_ms": 938, "found_rate": 0.0, "status": "MEASURED" } - ] + ], + "chain": { + "gtins_tested": 30, + "ANY_PROVIDER_FOUND_RATE": 0.167, + "ANY_PROVIDER_NCM_RATE": 0.0, + "ANY_PROVIDER_CEST_RATE": 0.0, + "ANY_PROVIDER_DESCRIPTION_RATE": 0.167 + } } \ No newline at end of file diff --git a/docs/product_enrichment.md b/docs/product_enrichment.md index 2f81fc6..16bb859 100644 --- a/docs/product_enrichment.md +++ b/docs/product_enrichment.md @@ -42,6 +42,7 @@ and `enrichment.is_human_overridden(state_path, code, field)` is the hook every |---|---|---|---| | Cosmos (Bluesoft) | `COSMOS_TOKEN` in the Hub env (optional) | 25/day | description, brand, **NCM, CEST** — `NOT_CONFIGURED` without a token (not an error) | | UPCitemdb (trial) | no key | 100/day, burst 15 / 30 s (headers `X-RateLimit-*`) | title, brand, category | +| OSCBR / RSC Sistemas GTIN | free plan account made by the operator; `OSCBR_USER` / `OSCBR_PASSWORD` in the Hub env; 1 h bearer renewed automatically | 20 req/min | name, brand, category, **NCM** (CEST when present) — `NOT_CONFIGURED` without credentials | | Open Food Facts / Open Products Facts / Open Beauty Facts | open (ODbL), identifiable `User-Agent` | 15 req/min | title, brand, quantity | Quota is central (`ProviderState`, `ProviderUsageDaily`, `quota_kind = REMOTE_QUOTA | LOCAL_ESTIMATE | DOCUMENTED_RATE`), with positive (30 d) and negative (24 h) `GtinLookupCache`, single-flight per (provider, GTIN), circuit breaker (3 failures → 60 s doubling up to 15 min; 429 → until reset; 401/403 → 1 h). Quota exhausted → `PENDING_PROVIDER_QUOTA` + retry after reset. @@ -65,3 +66,19 @@ Agent detail: *Funcionalidades* (admin) and *Enriquecimento* (counters + product second run 0 writes; DRY_RUN 0 writes; WAITING/DISABLED 0 Firebird activity and no baseline; human override incl. deleted value; Hub offline → pending → recovers. - Migration 0029: fresh-vs-upgrade `pg_dump -s` identical; upgrade → downgrade → upgrade clean. - Live smoke `scripts/enrichment_live_smoke.py` (1 lookup/provider, real internet, separate from the suite). + + +## Provider evaluation (2026-09-26) +- **DotCompany — NOT integrated.** The free query (25/day per IP, no signup) is the *website* demo for human use; the API (`/api/v1/catalogo/gtin/{ean}`) is prepaid credits (4 credits = R$ 0,04 per query, packages from R$ 120); the only free API access is a `dc_test_` sandbox with test data. Automated free use is not offered, so no adapter (would violate zero cost / permitted automation). +- **OSCBR / RSC Sistemas — adapter implemented, optional.** Free initial plan without card, 20 req/min, token via account registration (Basic auth on `/oauth/token`, 1 h Bearer), `GET /api/gtin/infor/{gtin}` (fields `ncm`, `nome`, `marca`, `categoria`, `pais`). Terms are shown only inside the authenticated panel, so the operator must create the account and read them; the Hub never creates accounts. Without credentials it is `NOT_CONFIGURED` and makes no request. +- Cosmos stays optional (`COSMOS_TOKEN`). Nothing else qualified (GS1 CNP is a paid membership; Verified by GS1 is not an automation API). +- `PAID_PROVIDER_CALLS=0`; `ALLOW_PAID_PROVIDERS=false`. + +## Field audit (`docs/PRODUCT_ENRICHMENT_FIELD_AUDIT_V1.json`) +24 fields checked against `field_contract`; only `CLASSIFICACAO_FISCAL`, `CEST`, `IPI` have a deterministic, company-independent rule. All others need company/operator context (product role, ICMS treatment, origin, PIS/COFINS profile, entry scope, rounding) or description evidence (EX_TIPI) and are excluded on purpose; `CST_IPI` is NOT_APPLICABLE for non-IPI companies; `CCLASSTRIB` stays with the RTC gates. + +## Re-baseline after inactivity (1.3.1) +While the function is DISABLED / WAITING_FEATURE_CONFIGURATION the Agent marks a re-baseline; on reactivation the current catalogue becomes history, so a long-disabled installation never sends a burst of products to the providers. + +## Production KB seed strategy +Production KB starts empty (0 products, head 0028 before deploy). The lab databases are **not** imported into production: their observations are `CLIENT_OBSERVATION` evidence only and cannot verify anything alone (93/241 shared GTINs disagree on NCM). The KB grows from provider evidence on real new products; `scripts/enrichment_lab_kb.py --seed-url` exists for a dev/staging seed only. diff --git a/release/RELEASE_MANIFEST.json b/release/RELEASE_MANIFEST.json index 6a34e7e..e0c5e04 100644 --- a/release/RELEASE_MANIFEST.json +++ b/release/RELEASE_MANIFEST.json @@ -1,14 +1,14 @@ { "artifact": "RELEASE_MANIFEST", - "GTI_FISCAL_VERSION": "1.3.0", + "GTI_FISCAL_VERSION": "1.3.1", "generated_at": "2026-09-25T15:43:40+00:00", "hub": { - "version": "1.3.0", + "version": "1.3.1", "api_compat": "1.0", "engine_version": "0.3.0", "decision_version": "mission-04c-ipi-ex-split" }, - "agent_version": "1.3.0", + "agent_version": "1.3.1", "fiscal_engine": { "ENGINE_VERSION": "0.3.0", "DECISION_VERSION": "mission-04c-ipi-ex-split", diff --git a/release/VERSION b/release/VERSION index f0bb29e..3a3cd8c 100644 --- a/release/VERSION +++ b/release/VERSION @@ -1 +1 @@ -1.3.0 +1.3.1 diff --git a/scripts/enrichment_benchmark.py b/scripts/enrichment_benchmark.py index bdeb484..4538682 100644 --- a/scripts/enrichment_benchmark.py +++ b/scripts/enrichment_benchmark.py @@ -17,13 +17,13 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) import httpx # noqa: E402 -from central.app.enrichment.providers import Cosmos, OpenBeautyFacts, OpenFoodFacts, OpenProductsFacts, ProviderError, UpcItemDb, USER_AGENT # noqa: E402 +from central.app.enrichment.providers import Cosmos, Oscbr, OpenBeautyFacts, OpenFoodFacts, OpenProductsFacts, ProviderError, UpcItemDb, USER_AGENT # noqa: E402 -PACE = {"upcitemdb": 2.6, "openfoodfacts": 5.2, "openproductsfacts": 5.2, "openbeautyfacts": 5.2, "cosmos": 1.0} # seconds between calls: comfortably under the documented burst / per-minute limits +PACE = {"oscbr": 3.2, "upcitemdb": 2.6, "openfoodfacts": 5.2, "openproductsfacts": 5.2, "openbeautyfacts": 5.2, "cosmos": 1.0} # seconds between calls: comfortably under the documented burst / per-minute limits def bench(provider, items): - out = {"provider": provider.name, "configured": provider.configured, "sample_size": 0, "found": 0, "not_found": 0, "with_description": 0, "with_brand": 0, "with_ncm": 0, "with_cest": 0, "errors": 0, "error_kinds": {}, "latencies_ms": []} + out = {"provider": provider.name, "configured": provider.configured, "sample_size": 0, "found": 0, "not_found": 0, "with_description": 0, "with_brand": 0, "with_ncm": 0, "with_cest": 0, "errors": 0, "error_kinds": {}, "latencies_ms": [], "per_gtin": {}} if not provider.configured: out["status"] = "NOT_CONFIGURED" return out @@ -40,6 +40,7 @@ def bench(provider, items): continue out["sample_size"] += 1 out["latencies_ms"].append(int((time.monotonic() - started) * 1000)) + out["per_gtin"][item["gtin"]] = {"found": product.found, "description": bool(product.title), "ncm": bool(product.ncm), "cest": bool(product.cest)} if product.found: out["found"] += 1 out["with_description"] += bool(product.title) @@ -65,12 +66,17 @@ def main(): args = ap.parse_args() items = json.loads(Path(args.sample).read_text(encoding="utf-8"))[: args.limit] client = httpx.Client(timeout=8.0, headers={"User-Agent": USER_AGENT}) - providers = [Cosmos(client, os.environ.get("COSMOS_TOKEN")), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)] + providers = [Cosmos(client, os.environ.get("COSMOS_TOKEN")), Oscbr(client, None, os.environ.get("OSCBR_USER"), os.environ.get("OSCBR_PASSWORD")), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)] only = {n for n in args.only.split(",") if n} out = Path(args.out) result = json.loads(out.read_text(encoding="utf-8")) if only and out.exists() else {"measured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "sample_source": "lab ERP databases (read-only), valid GTINs only", "limit_per_provider": args.limit, "providers": []} measured = [bench(p, items) for p in providers if not only or p.name in only] result["providers"] = [p for p in result["providers"] if p["provider"] not in {m["provider"] for m in measured}] + measured + per = [p.get("per_gtin", {}) for p in result["providers"]] + tested = sorted({g for d in per for g in d}) + def rate(key): + return round(sum(1 for g in tested if any(d.get(g, {}).get(key) for d in per)) / len(tested), 3) if tested else None + result["chain"] = {"gtins_tested": len(tested), "ANY_PROVIDER_FOUND_RATE": rate("found"), "ANY_PROVIDER_NCM_RATE": rate("ncm"), "ANY_PROVIDER_CEST_RATE": rate("cest"), "ANY_PROVIDER_DESCRIPTION_RATE": rate("description")} Path(args.out).write_text(json.dumps(result, ensure_ascii=False, indent=1), encoding="utf-8") print(json.dumps(result, ensure_ascii=False, indent=1)) diff --git a/tests/test_release_candidate.py b/tests/test_release_candidate.py index 98c293b..3703efd 100644 --- a/tests/test_release_candidate.py +++ b/tests/test_release_candidate.py @@ -14,11 +14,11 @@ sys.path.insert(0, str(ROOT)) def test_versions_are_coherent_across_hub_agent_and_packages(): from central.app.version import GTI_FISCAL_VERSION v = (ROOT / "release" / "VERSION").read_text(encoding="utf-8").strip() - assert v == GTI_FISCAL_VERSION == "1.3.0" + assert v == GTI_FISCAL_VERSION == "1.3.1" pep = v.replace("-rc", "rc") for pp in ("central/pyproject.toml", "agent/pyproject.toml"): assert f'version = "{pep}"' in (ROOT / pp).read_text(encoding="utf-8") - assert '"AGENT_VERSION", "1.3.0"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8") + assert '"AGENT_VERSION", "1.3.1"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8") manifest = ROOT / "release" / "RELEASE_MANIFEST.json" if manifest.exists(): m = json.loads(manifest.read_text(encoding="utf-8"))