chore: capture semantic pipeline baseline

2026-09-21 15:52:17 -03:00
parent 9dcc263217
commit 6731eeca08
47 changed files with 3724 additions and 15 deletions
@@ -0,0 +1,66 @@
+{
+  "artifact": "AI_EVALUATION_V1",
+  "task": "INVALID_NCM candidate selection",
+  "pipeline": "candidate_generation -> provider_selection -> validator",
+  "provider": "baseline",
+  "model": "baseline-deterministic-v1",
+  "prompt_version": "ai-semantic-v1",
+  "dataset_versions": {
+    "ncm": "3abc0d2eb2501bbd0cc3f5e0ee3774d33f091ea305e8127ee020e4009878987e"
+  },
+  "k": 10,
+  "eval_total": 200,
+  "metrics": {
+    "EXACT_MATCH": 2,
+    "WRONG_SELECTION": 164,
+    "NONE_WHEN_KNOWN": 34,
+    "INVALID_OUTPUT": 0,
+    "RECALL_AT_K": 11
+  },
+  "accuracy_exact_match": 0.01,
+  "recall_at_k": 0.055,
+  "by_segment": {
+    "base_testes": {
+      "EXACT_MATCH": 0,
+      "WRONG_SELECTION": 27,
+      "NONE_WHEN_KNOWN": 13,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 2,
+      "TOTAL": 40
+    },
+    "construcao": {
+      "EXACT_MATCH": 1,
+      "WRONG_SELECTION": 39,
+      "NONE_WHEN_KNOWN": 0,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 4,
+      "TOTAL": 40
+    },
+    "mercado": {
+      "EXACT_MATCH": 1,
+      "WRONG_SELECTION": 35,
+      "NONE_WHEN_KNOWN": 4,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 2,
+      "TOTAL": 40
+    },
+    "moda": {
+      "EXACT_MATCH": 0,
+      "WRONG_SELECTION": 35,
+      "NONE_WHEN_KNOWN": 5,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 0,
+      "TOTAL": 40
+    },
+    "restaurante": {
+      "EXACT_MATCH": 0,
+      "WRONG_SELECTION": 28,
+      "NONE_WHEN_KNOWN": 12,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 3,
+      "TOTAL": 40
+    }
+  },
+  "cost_usd": 0.0,
+  "note": "baseline provider (deterministic, zero-cost): measures the pipeline floor; rerun with a real provider once credentials exist"
+}