chore: capture semantic pipeline baseline

2026-09-21 15:52:17 -03:00
parent 9dcc263217
commit 6731eeca08
47 changed files with 3724 additions and 15 deletions
@@ -0,0 +1,40 @@
+{
+  "artifact": "AI_COST_PLAN_V1",
+  "unique_ai_work_items": 586,
+  "per_task_type": {
+    "INVALID_NCM": {
+      "items": 223,
+      "input_tokens": 156100,
+      "output_tokens": 26760,
+      "cells_covered": 1787
+    },
+    "PRODUCT_ROLE": {
+      "items": 41,
+      "input_tokens": 16400,
+      "output_tokens": 2460,
+      "cells_covered": 482
+    },
+    "EX_TIPI": {
+      "items": 20,
+      "input_tokens": 10000,
+      "output_tokens": 1600,
+      "cells_covered": 635
+    },
+    "CEST": {
+      "items": 302,
+      "input_tokens": 151000,
+      "output_tokens": 24160,
+      "cells_covered": 1793
+    }
+  },
+  "estimated_input_tokens": 333500,
+  "estimated_output_tokens": 54980,
+  "estimated_cost_usd": 0.44346,
+  "ai_max_run_cost_usd": 5.0,
+  "budget_authorized": true,
+  "provider_configured": "baseline",
+  "model_configured": "baseline-deterministic-v1",
+  "ai_enabled": false,
+  "max_batch_size": 20,
+  "min_confidence": 0.6
+}
@@ -0,0 +1,66 @@
+{
+  "artifact": "AI_EVALUATION_V1",
+  "task": "INVALID_NCM candidate selection",
+  "pipeline": "candidate_generation -> provider_selection -> validator",
+  "provider": "baseline",
+  "model": "baseline-deterministic-v1",
+  "prompt_version": "ai-semantic-v1",
+  "dataset_versions": {
+    "ncm": "3abc0d2eb2501bbd0cc3f5e0ee3774d33f091ea305e8127ee020e4009878987e"
+  },
+  "k": 50,
+  "eval_total": 200,
+  "metrics": {
+    "EXACT_MATCH": 2,
+    "WRONG_SELECTION": 164,
+    "NONE_WHEN_KNOWN": 34,
+    "INVALID_OUTPUT": 0,
+    "RECALL_AT_K": 15
+  },
+  "accuracy_exact_match": 0.01,
+  "recall_at_k": 0.075,
+  "by_segment": {
+    "base_testes": {
+      "EXACT_MATCH": 0,
+      "WRONG_SELECTION": 27,
+      "NONE_WHEN_KNOWN": 13,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 2,
+      "TOTAL": 40
+    },
+    "construcao": {
+      "EXACT_MATCH": 1,
+      "WRONG_SELECTION": 39,
+      "NONE_WHEN_KNOWN": 0,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 5,
+      "TOTAL": 40
+    },
+    "mercado": {
+      "EXACT_MATCH": 1,
+      "WRONG_SELECTION": 35,
+      "NONE_WHEN_KNOWN": 4,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 4,
+      "TOTAL": 40
+    },
+    "moda": {
+      "EXACT_MATCH": 0,
+      "WRONG_SELECTION": 35,
+      "NONE_WHEN_KNOWN": 5,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 1,
+      "TOTAL": 40
+    },
+    "restaurante": {
+      "EXACT_MATCH": 0,
+      "WRONG_SELECTION": 28,
+      "NONE_WHEN_KNOWN": 12,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 3,
+      "TOTAL": 40
+    }
+  },
+  "cost_usd": 0.0,
+  "note": "baseline provider (deterministic, zero-cost): measures the pipeline floor; rerun with a real provider once credentials exist"
+}
@@ -0,0 +1,13 @@
+{
+  "products": 30801,
+  "observations": 35065,
+  "validated_ncm": 30833,
+  "erp_ncm_observations": 212,
+  "erp_cest_observations": 11719,
+  "tenant_contexts": 0,
+  "firebird_writes": 0,
+  "gtin_products": 8559,
+  "fingerprint_products": 22242,
+  "conflicts": 163,
+  "loaded_at": "2026-09-21T15:22:11.333546+00:00"
+}