feat: product enrichment observability, lab KB study, benchmark, copy + local-Hub E2E, docs; 1.3.0

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-25 19:26:13 -03:00
parent 5deebea14e
commit 4fc83a33d5
38 changed files with 1282 additions and 60 deletions
@@ -0,0 +1,119 @@
+{
+ "artifact": "PRODUCT_NEW_DETECTION_DISCOVERY_V1",
+ "table": "C000025",
+ "decision": "SELECT COUNT(*) poll (adaptive interval) + CODIGO-set diff only when the count moves (+ 60 s safety diff). No trigger/table/procedure is created in the SGV database.",
+ "why": "DATA_CADASTRO/ULTIMA_ALTERACAO are NOT indexed (a filter would scan the table like COUNT does); CODIGO is the PK; the ERP already owns triggers C000025_INS/UPD/DEL that must not be touched.",
+ "databases": {
+  "CONSTRUCAO": {
+   "rows": 3712,
+   "count_ms_p50": 2.2,
+   "code_list_ms": 62.9,
+   "max_codigo_ms": 5.5,
+   "pk": "PK_C000025(CODIGO)",
+   "indexed_columns": [
+    "CCLASSTRIB",
+    "CODIGO",
+    "PRODUTO"
+   ],
+   "sgv_triggers_present": [
+    "C000025_DEL",
+    "C000025_INS",
+    "C000025_UPD",
+    "CODBARRA_CODPRO"
+   ],
+   "date_columns_unindexed": [
+    "ULTIMA_ALTERACAO",
+    "DATA_CADASTRO"
+   ]
+  },
+  "MMMODAS": {
+   "rows": 20997,
+   "count_ms_p50": 9.0,
+   "code_list_ms": 241.1,
+   "max_codigo_ms": 24.3,
+   "pk": "PK_C000025(CODIGO)",
+   "indexed_columns": [
+    "CCLASSTRIB",
+    "CODIGO",
+    "PRODUTO"
+   ],
+   "sgv_triggers_present": [
+    "C000025_DEL",
+    "C000025_INS",
+    "C000025_UPD",
+    "CODBARRA_CODPRO"
+   ],
+   "date_columns_unindexed": [
+    "ULTIMA_ALTERACAO",
+    "DATA_CADASTRO"
+   ]
+  },
+  "RINCAO": {
+   "rows": 894,
+   "count_ms_p50": 1.0,
+   "code_list_ms": 15.6,
+   "max_codigo_ms": 4.0,
+   "pk": "PK_C000025(CODIGO)",
+   "indexed_columns": [
+    "CCLASSTRIB",
+    "CODIGO",
+    "PRODUTO"
+   ],
+   "sgv_triggers_present": [
+    "C000025_DEL",
+    "C000025_INS",
+    "C000025_UPD",
+    "CODBARRA_CODPRO"
+   ],
+   "date_columns_unindexed": [
+    "ULTIMA_ALTERACAO",
+    "DATA_CADASTRO"
+   ]
+  },
+  "UNIAO": {
+   "rows": 8942,
+   "count_ms_p50": 4.3,
+   "code_list_ms": 119.2,
+   "max_codigo_ms": 11.4,
+   "pk": "PK_C000025(CODIGO)",
+   "indexed_columns": [
+    "CCLASSTRIB",
+    "CODBARRA",
+    "CODIGO",
+    "PRODUTO"
+   ],
+   "sgv_triggers_present": [
+    "C000025_DEL",
+    "C000025_INS",
+    "C000025_UPD",
+    "CODBARRA_CODPRO"
+   ],
+   "date_columns_unindexed": [
+    "ULTIMA_ALTERACAO",
+    "DATA_CADASTRO"
+   ]
+  },
+  "TESTESOFICIAIS": {
+   "rows": 521,
+   "count_ms_p50": 0.9,
+   "code_list_ms": 8.0,
+   "max_codigo_ms": 2.2,
+   "pk": "PK_C000025(CODIGO)",
+   "indexed_columns": [
+    "CCLASSTRIB",
+    "CODIGO",
+    "PRODUTO"
+   ],
+   "sgv_triggers_present": [
+    "C000025_DEL",
+    "C000025_INS",
+    "C000025_UPD",
+    "CODBARRA_CODPRO"
+   ],
+   "date_columns_unindexed": [
+    "ULTIMA_ALTERACAO",
+    "DATA_CADASTRO"
+   ]
+  }
+ }
+}
@@ -0,0 +1,96 @@
+{
+ "measured_at": "2026-09-25T21:53:01Z",
+ "sample_source": "lab ERP databases (read-only), valid GTINs only",
+ "limit_per_provider": 30,
+ "providers": [
+  {
+   "provider": "cosmos",
+   "configured": false,
+   "sample_size": 0,
+   "found": 0,
+   "not_found": 0,
+   "with_description": 0,
+   "with_brand": 0,
+   "with_ncm": 0,
+   "with_cest": 0,
+   "errors": 0,
+   "error_kinds": {},
+   "latencies_ms": [],
+   "status": "NOT_CONFIGURED"
+  },
+  {
+   "provider": "upcitemdb",
+   "configured": true,
+   "sample_size": 6,
+   "found": 1,
+   "not_found": 5,
+   "with_description": 1,
+   "with_brand": 1,
+   "with_ncm": 0,
+   "with_cest": 0,
+   "errors": 1,
+   "error_kinds": {
+    "RATE_LIMITED": 1
+   },
+   "error_rate": 0.143,
+   "latency_p50_ms": 139.5,
+   "latency_p95_ms": 140,
+   "found_rate": 0.167,
+   "status": "MEASURED"
+  },
+  {
+   "provider": "openfoodfacts",
+   "configured": true,
+   "sample_size": 30,
+   "found": 4,
+   "not_found": 26,
+   "with_description": 4,
+   "with_brand": 3,
+   "with_ncm": 0,
+   "with_cest": 0,
+   "errors": 0,
+   "error_kinds": {},
+   "error_rate": 0.0,
+   "latency_p50_ms": 655.0,
+   "latency_p95_ms": 827,
+   "found_rate": 0.133,
+   "status": "MEASURED"
+  },
+  {
+   "provider": "openproductsfacts",
+   "configured": true,
+   "sample_size": 30,
+   "found": 0,
+   "not_found": 30,
+   "with_description": 0,
+   "with_brand": 0,
+   "with_ncm": 0,
+   "with_cest": 0,
+   "errors": 0,
+   "error_kinds": {},
+   "error_rate": 0.0,
+   "latency_p50_ms": 640.0,
+   "latency_p95_ms": 969,
+   "found_rate": 0.0,
+   "status": "MEASURED"
+  },
+  {
+   "provider": "openbeautyfacts",
+   "configured": true,
+   "sample_size": 30,
+   "found": 0,
+   "not_found": 30,
+   "with_description": 0,
+   "with_brand": 0,
+   "with_ncm": 0,
+   "with_cest": 0,
+   "errors": 0,
+   "error_kinds": {},
+   "error_rate": 0.0,
+   "latency_p50_ms": 655.5,
+   "latency_p95_ms": 1311,
+   "found_rate": 0.0,
+   "status": "MEASURED"
+  }
+ ]
+}
@@ -0,0 +1,67 @@
+# GTI Fiscal — Product Enrichment V1 (produto novo → EAN/nome → perfil fiscal → preenchimento seguro)
+
+Living document. Same Hub / FastAPI / PostgreSQL / Agent as the rest of GTI Fiscal: **no new server, service, panel or product**, no change in the SGV, no Firebird trigger/table/procedure.
+
+## Hard rules (verified by tests)
+- `AI_ENABLED=false`, `LLM_ENABLED=false`, `EMBEDDINGS_ENABLED=false`. Everything here is deterministic.
+- External cost R$ 0: only free providers; `ALLOW_PAID_PROVIDERS=false` (a paid provider makes `build_providers` raise). **Agents never call providers – only the Hub.**
+- Per-installation functions live in the Hub (Agent detail → *Funcionalidades*): **Corretor fiscal** on/off and **Enriquecimento** `DISABLED | DRY_RUN | ENABLED`. ADMIN only, CSRF, audited (`INSTALLATION_FEATURES`),
+  typed confirmation `PREENCHER` for ENABLED. A **new** installation is `WAITING_FEATURE_CONFIGURATION` (`features_configured=false`): it heartbeats but `claim_next_job`/`schedule_automatic_sync`
+  hand it nothing and the enrichment worker does zero Firebird work, until features are chosen (at creation in *Nova instalação*, or later). Existing installations keep the previous behaviour (migration 0029).
+- Scope = **new products only**. When the function is first active the Agent stores a baseline of the existing `C000025.CODIGO` set (local SQLite); historical products are never sent anywhere.
+
+## Flow
+1. **Detection (Agent, ≤ 5 s)** – `SELECT COUNT(*) FROM C000025` every 2 s (adaptive up to 30 s if the count is slow); code-set diff only when the count moved (+ a 60 s safety diff). No trigger. Discovery on the five
+   lab databases (`docs/PRODUCT_NEW_DETECTION_DISCOVERY_V1.json`): PK `CODIGO` char(6), `DATA_CADASTRO/ULTIMA_ALTERACAO` **not indexed**, count 1–9 ms, full code list 8–241 ms, the ERP already owns triggers on the table.
+   The EAN typed by the operator is in **`CODBARRA`** (fiscal `GTIN` is mostly empty): the first VALID of `GTIN`/`CODBARRA` is sent.
+2. **Resolve (Hub)** `POST /v1/products/resolve {product_code, gtin, description, current}` → per-field decisions. Hub time budget 4.5 s; beyond it (or on quota/offline) the state is `PENDING_*` and the Agent retries with backoff (30 s … 30 min); results are cached, so the retry is usually free.
+3. **Write (Agent)** typed capability `PRODUCT_ENRICHMENT_WRITE_V1`, allowlist `CLASSIFICACAO_FISCAL, CEST, IPI`; fixed statement `UPDATE C000025 SET f=? WHERE CODIGO=? AND (f IS NULL OR TRIM(f)='')`, one short transaction per field,
+   guarded re-select inside the transaction, post-write verification. Before every field: fresh central authorization (fail closed). `COD_ORIGEM_MERCADORIA` is never written; cClassTrib stays with the existing RTC gates
+   (the Agent only requests `REPROCESS_PRODUCTS` for the product; nothing is decided here).
+4. **Audit** `POST /v1/products/enrichment/applied` → `product_field_audits` (tenant, agent, installation, code, GTIN, description snapshot, field, old/new, rule, KB status, evidence, dataset versions, authorization origin, job, result). No secrets.
+
+## Identity and states
+- GTIN: 8/12/13/14 digits + check digit → `VALID_GTIN | INVALID_GTIN | MISSING_GTIN` (+ `RESTRICTED_GTIN` for store-internal prefixes 02/04/20–29). **GTIN has absolute priority over the description.** An invalid GTIN never reaches a provider.
+- Resolution states (discrete, no scores): `EXACT_GTIN_VERIFIED, EXACT_KB_NAME_VERIFIED, MULTI_SOURCE_MATCH, SINGLE_WEAK_SOURCE, AMBIGUOUS, CONFLICT, NOT_FOUND` (+ operational `INVALID_GTIN, PENDING_PROVIDER_QUOTA, PENDING_PROVIDERS_OFFLINE, FEATURE_DISABLED`).
+- Field decisions: `AUTO_FILL_SAFE | SUGGESTION_ONLY | CONFLICT | INSUFFICIENT_EVIDENCE | ALREADY_SET | NOT_APPLICABLE | HUMAN_OVERRIDDEN`. A field that is not resolved never blocks the others.
+
+## Knowledge base (same PostgreSQL: `kb_products`, `kb_knowledge` with per-field `status`)
+`UNVERIFIED / EVIDENCE_ONLY / VERIFIED_KB / CONFLICT / STALE`. NCM becomes `VERIFIED_KB` only when it is **valid and vigente in the official NCM dataset**, has **≥ 2 independent evidence families** (each provider = 1 family; client observations = 1 family and only when ≥ 2 distinct clients)
+and **no conflicting value**. Client consensus alone is never authority (measured on the lab databases: 241 GTINs shared by ≥ 2 clients, **93 with different NCMs**). Company-specific decisions (CSOSN, CFOP, …) never enter the KB.
+**Human edits never train the KB.** `NCM` auto-fill: only `VERIFIED_KB`; a single non-official source is `SUGGESTION_ONLY`; conflicts write nothing.
+CEST and IPI are deterministic functions of an official NCM through the existing engine (`Classifier`): they are filled when the NCM is being written (AUTO) **or is already present and official**, only for a unique official CEST mapping / a numeric TIPI rate.
+Name-only: only an exact fingerprint (sorted normalized tokens, quantity/unit normalized) of a **specific** description (≥ 2 informative tokens + quantity, or ≥ 4) that maps to a unique `VERIFIED_KB` product may fill; generic names never do.
+ERP observations (`scripts/enrichment_lab_kb.py`, read-only) enter as `CLIENT_OBSERVATION` evidence only.
+
+## Human override protection
+After writing, the Agent watches the value. Any value different from what it wrote — **including a deleted value** — marks the field `HUMAN_OVERRIDDEN` (local SQLite + reported to the Hub). The Hub then answers `HUMAN_OVERRIDDEN` for that field, the enrichment never rewrites it,
+and `enrichment.is_human_overridden(state_path, code, field)` is the hook every automatic writer must call. No write loops: second run = 0 writes.
+
+## Providers (Hub only; `central/app/enrichment/providers.py`, single interface `lookup_gtin / search_description / health`)
+| Provider | Access | Limit used | Gives |
+|---|---|---|---|
+| Cosmos (Bluesoft) | `COSMOS_TOKEN` in the Hub env (optional) | 25/day | description, brand, **NCM, CEST** — `NOT_CONFIGURED` without a token (not an error) |
+| UPCitemdb (trial) | no key | 100/day, burst 15 / 30 s (headers `X-RateLimit-*`) | title, brand, category |
+| Open Food Facts / Open Products Facts / Open Beauty Facts | open (ODbL), identifiable `User-Agent` | 15 req/min | title, brand, quantity |
+Quota is central (`ProviderState`, `ProviderUsageDaily`, `quota_kind = REMOTE_QUOTA | LOCAL_ESTIMATE | DOCUMENTED_RATE`), with positive (30 d) and negative (24 h) `GtinLookupCache`, single-flight per (provider, GTIN), circuit breaker
+(3 failures → 60 s doubling up to 15 min; 429 → until reset; 401/403 → 1 h). Quota exhausted → `PENDING_PROVIDER_QUOTA` + retry after reset.
+**Benchmark on real lab GTINs** (`docs/PRODUCT_PROVIDER_BENCHMARK_V1.json`): identity found rate is low for Brazilian goods (Open Food Facts 4/30, UPCitemdb rate-limited after 6 calls) and **only Cosmos returns NCM**. Consequently
+NCM auto-fill depends on the KB (Cosmos + client observations) and the product delivers CEST/IPI immediately whenever the operator already typed a valid NCM. Cosmos without a token is not a blocker.
+
+## Official data
+`NCM` (Siscomex JSON) is refreshed daily by `central/app/datasets/updater.py` (staging in memory → schema validation → plausibility ≥ 90 % of the active count → atomic promotion by `DatasetService.import_payload`; on any failure the last valid dataset stays).
+Every attempt is a `source_refresh_runs` row (url, time, sha256, version, count, status, error). TIPI/CEST/CFOP/CST/CSOSN/PIS-COFINS/RTC datasets are reused as they are. The dashboard shows *Fontes oficiais* (records, version, sha, last update, status).
+
+## Panel
+Dashboard: *Fontes de produtos* (status, used/limit/remaining/reset, last success/error, latency, cache-hit rate, `configurado: sim/não` — never a token), *Fontes oficiais*, small *Enriquecimento* coverage.
+Agent detail: *Funcionalidades* (admin) and *Enriquecimento* (counters + product lookup with per-field provenance). Problems appear through the existing blockers (`field=ENRICHMENT`, PT-BR reasons).
+
+## Coverage metrics (`observability.coverage_metrics`)
+`GTIN_VALID_RATE, KB_HIT_RATE, EXTERNAL_LOOKUP_RATE, PRODUCT_FOUND_RATE, NCM_AUTO_FILL_SAFE_RATE, NAME_ONLY_RESOLUTION_RATE, AMBIGUOUS_RATE, CONFLICT_RATE, PROVIDER_QUOTA_BLOCK_RATE` (+ cache stats).
+
+## Evidence
+- Offline tests: `central/tests/test_product_enrichment_resolver.py` (12 resolution scenarios, quota/circuit/cache, provider contract fixtures), `test_product_enrichment_api.py` (feature gates, audit, override, blockers, multi-installation), `test_official_refresh.py`, `agent/tests/test_enrichment.py`.
+- Real Firebird **copy-only** E2E `scripts/enrichment_copy_e2e.py` → `docs/runs/enrichment_copy_e2e.json`: detect→written P50 2.8 s / P95 3.9 s / max 4.1 s; only the 3 allowed columns; other tables changed only by the SGV's own triggers for the test rows;
+  second run 0 writes; DRY_RUN 0 writes; WAITING/DISABLED 0 Firebird activity and no baseline; human override incl. deleted value; Hub offline → pending → recovers.
+- Migration 0029: fresh-vs-upgrade `pg_dump -s` identical; upgrade → downgrade → upgrade clean.
+- Live smoke `scripts/enrichment_live_smoke.py` (1 lookup/provider, real internet, separate from the suite).
@@ -0,0 +1,47 @@
+{
+ "copy_only": true,
+ "products_enabled": 12,
+ "detect_to_written_seconds": {
+  "p50": 2.835,
+  "p95": 3.89,
+  "max": 4.14,
+  "all": [
+   1.66,
+   2.08,
+   3.89,
+   2.83,
+   2.52,
+   2.84,
+   3.05,
+   2.58,
+   3.31,
+   2.45,
+   3.89,
+   4.14
+  ]
+ },
+ "hub_resolve_calls": 15,
+ "tables_changed": [
+  "C000025",
+  "C000058",
+  "C000100"
+ ],
+ "checks": {
+  "WAITING_NO_BASELINE_NO_CALLS": true,
+  "DISABLED_NO_BASELINE_NO_CALLS": true,
+  "BASELINE_TAKEN_ON_ACTIVATION": true,
+  "DRY_RUN_ZERO_WRITES": true,
+  "FIELDS_WRITTEN": true,
+  "SECOND_RUN_ZERO_WRITES": true,
+  "HISTORICAL_KNOWN_COUNT": true,
+  "OVERRIDE_DELETED_VALUE": true,
+  "OVERRIDE_REPORTED_TO_HUB": true,
+  "OVERRIDE_NEVER_REWRITTEN": true,
+  "HUB_OFFLINE_PENDING_NO_WRITE": true,
+  "HUB_BACK_RECOVERS": true,
+  "OTHER_TABLES_ONLY_ERP_TRIGGER_SIDE_EFFECTS_OF_TEST_ROWS": true,
+  "ORIGIN_NEVER_WRITTEN": true
+ },
+ "ALL_PASS": true,
+ "poll_seconds": 2.0
+}
@@ -0,0 +1,21 @@
+{
+ "copy_only": true,
+ "hub": "http://127.0.0.1:8000",
+ "detect_to_written_seconds_enabled": 4.22,
+ "checks": {
+  "REGISTERED_DEFAULT_DISABLED": true,
+  "DISABLED_NO_ACTIVITY": true,
+  "BASELINE_ON_ACTIVATION": true,
+  "DRY_RUN_ZERO_WRITES": true,
+  "DRY_RUN_AUDITED_IN_HUB": true,
+  "ENABLED_NCM_WRITTEN": true,
+  "WRITTEN_FIELDS_MATCH_HUB_AUDIT": true,
+  "EARLIER_DRY_RUN_PRODUCT_STILL_UNTOUCHED": true,
+  "UNKNOWN_GTIN_NO_WRITE": true,
+  "UNKNOWN_GTIN_BLOCKER_PT": true,
+  "INVALID_GTIN_NO_PROVIDER_CALL": true,
+  "DISABLED_AGAIN_ZERO_WRITES": true,
+  "CORRECTOR_OFF_NO_JOBS": true
+ },
+ "ALL_PASS": true
+}