gti
d21a9d1140
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
66 lines
1.6 KiB
JSON
66 lines
1.6 KiB
JSON
{
|
|
"artifact": "AI_EVALUATION_V1",
|
|
"task": "INVALID_NCM candidate selection",
|
|
"pipeline": "candidate_generation -> provider_selection -> validator",
|
|
"provider": "baseline",
|
|
"model": "baseline-deterministic-v1",
|
|
"prompt_version": "ai-semantic-v1",
|
|
"dataset_versions": {
|
|
"ncm": "3abc0d2eb2501bbd0cc3f5e0ee3774d33f091ea305e8127ee020e4009878987e"
|
|
},
|
|
"k": 50,
|
|
"eval_total": 200,
|
|
"metrics": {
|
|
"EXACT_MATCH": 2,
|
|
"WRONG_SELECTION": 164,
|
|
"NONE_WHEN_KNOWN": 34,
|
|
"INVALID_OUTPUT": 0,
|
|
"RECALL_AT_K": 15
|
|
},
|
|
"accuracy_exact_match": 0.01,
|
|
"recall_at_k": 0.075,
|
|
"by_segment": {
|
|
"base_testes": {
|
|
"EXACT_MATCH": 0,
|
|
"WRONG_SELECTION": 27,
|
|
"NONE_WHEN_KNOWN": 13,
|
|
"INVALID_OUTPUT": 0,
|
|
"RECALL_AT_K": 2,
|
|
"TOTAL": 40
|
|
},
|
|
"construcao": {
|
|
"EXACT_MATCH": 1,
|
|
"WRONG_SELECTION": 39,
|
|
"NONE_WHEN_KNOWN": 0,
|
|
"INVALID_OUTPUT": 0,
|
|
"RECALL_AT_K": 5,
|
|
"TOTAL": 40
|
|
},
|
|
"mercado": {
|
|
"EXACT_MATCH": 1,
|
|
"WRONG_SELECTION": 35,
|
|
"NONE_WHEN_KNOWN": 4,
|
|
"INVALID_OUTPUT": 0,
|
|
"RECALL_AT_K": 4,
|
|
"TOTAL": 40
|
|
},
|
|
"moda": {
|
|
"EXACT_MATCH": 0,
|
|
"WRONG_SELECTION": 35,
|
|
"NONE_WHEN_KNOWN": 5,
|
|
"INVALID_OUTPUT": 0,
|
|
"RECALL_AT_K": 1,
|
|
"TOTAL": 40
|
|
},
|
|
"restaurante": {
|
|
"EXACT_MATCH": 0,
|
|
"WRONG_SELECTION": 28,
|
|
"NONE_WHEN_KNOWN": 12,
|
|
"INVALID_OUTPUT": 0,
|
|
"RECALL_AT_K": 3,
|
|
"TOTAL": 40
|
|
}
|
|
},
|
|
"cost_usd": 0.0,
|
|
"note": "baseline provider (deterministic, zero-cost): measures the pipeline floor; rerun with a real provider once credentials exist"
|
|
} |