{ "artifact": "NCM_RETRIEVAL_EVALUATION_V2", "eval_total": 200, "quality_counts": { "GOLD": 65, "SILVER": 118, "NOISY": 17 }, "lexicon_size": 644, "lexicon_sample": { "100M": "100MT", "10A": "10AM", "10M": "10MT", "20A": "20AM", "20M": "20MT", "220V": "220VX4600W", "30M": "30MT", "32M": "32MM", "32X25": "32X25MM", "40M": "40MM", "4E0V": "4E0V4", "4PC": "4PCA", "50M": "50MT", "5X100": "5X100MM", "60M": "60MT", "ACES": "ACESSORIO", "ACHOC": "ACHOCOLATADO", "ACO": "ACOPLADO", "ACOPL": "ACOPLADA", "ACQUA": "ACQUAMARINE", "ACR": "ACRILICA", "ACRIL": "ACRILICA", "ADM": "ADMISSAO", "ADVAN": "ADVANC", "AERO": "AEROSOL", "AFRIC": "AFRICANO", "ALAV": "ALAVANCA", "ALAVA": "ALAVANCA", "ALC": "ALCOOL", "ALE": "ALEJANDRA", "ALF": "ALFAIATARIA", "ALFA": "ALFAIATARIA", "ALI": "ALINE", "ALIC": "ALICATE", "ALL": "ALLELA", "ALMOF": "ALMOFADADO", "ALUM": "ALUMASA", "ALUMI": "ALUMINIO", "AMEND": "AMENDOA", "AMOR": "AMORI" }, "overall": { "OFFICIAL_ONLY": { "total": 200, "recall_at_1": 0.015, "recall_at_5": 0.08, "recall_at_10": 0.11, "recall_at_20": 0.14, "recall_at_50": 0.195, "mrr": 0.0456 }, "KB_ONLY": { "total": 200, "recall_at_1": 0.665, "recall_at_5": 0.85, "recall_at_10": 0.895, "recall_at_20": 0.925, "recall_at_50": 0.935, "mrr": 0.7499 }, "HYBRID": { "total": 200, "recall_at_1": 0.635, "recall_at_5": 0.855, "recall_at_10": 0.9, "recall_at_20": 0.92, "recall_at_50": 0.935, "mrr": 0.7325 } }, "by_quality": { "GOLD": { "OFFICIAL_ONLY": { "total": 65, "recall_at_1": 0.0308, "recall_at_5": 0.1077, "recall_at_10": 0.1231, "recall_at_20": 0.1538, "recall_at_50": 0.1692, "mrr": 0.0681 }, "KB_ONLY": { "total": 65, "recall_at_1": 0.7692, "recall_at_5": 0.9077, "recall_at_10": 0.9385, "recall_at_20": 0.9538, "recall_at_50": 0.9538, "mrr": 0.8276 }, "HYBRID": { "total": 65, "recall_at_1": 0.7846, "recall_at_5": 0.9231, "recall_at_10": 0.9538, "recall_at_20": 0.9538, "recall_at_50": 0.9538, "mrr": 0.8392 } }, "SILVER": { "OFFICIAL_ONLY": { "total": 118, "recall_at_1": 0.0085, "recall_at_5": 0.0763, "recall_at_10": 0.1186, "recall_at_20": 0.1525, "recall_at_50": 0.2373, "mrr": 0.0398 }, "KB_ONLY": { "total": 118, "recall_at_1": 0.661, "recall_at_5": 0.8814, "recall_at_10": 0.9237, "recall_at_20": 0.9661, "recall_at_50": 0.9831, "mrr": 0.7656 }, "HYBRID": { "total": 118, "recall_at_1": 0.6271, "recall_at_5": 0.8898, "recall_at_10": 0.9322, "recall_at_20": 0.9576, "recall_at_50": 0.9831, "mrr": 0.7471 } }, "NOISY": { "OFFICIAL_ONLY": { "total": 17, "recall_at_1": 0.0, "recall_at_5": 0.0, "recall_at_10": 0.0, "recall_at_20": 0.0, "recall_at_50": 0.0, "mrr": 0.0 }, "KB_ONLY": { "total": 17, "recall_at_1": 0.2941, "recall_at_5": 0.4118, "recall_at_10": 0.5294, "recall_at_20": 0.5294, "recall_at_50": 0.5294, "mrr": 0.3442 }, "HYBRID": { "total": 17, "recall_at_1": 0.1176, "recall_at_5": 0.3529, "recall_at_10": 0.4706, "recall_at_20": 0.5294, "recall_at_50": 0.5294, "mrr": 0.2235 } } }, "by_segment": { "base_testes": { "OFFICIAL_ONLY": { "total": 40, "recall_at_1": 0.0, "recall_at_5": 0.05, "recall_at_10": 0.075, "recall_at_20": 0.075, "recall_at_50": 0.1, "mrr": 0.0254 }, "KB_ONLY": { "total": 40, "recall_at_1": 0.5, "recall_at_5": 0.75, "recall_at_10": 0.825, "recall_at_20": 0.85, "recall_at_50": 0.85, "mrr": 0.6171 }, "HYBRID": { "total": 40, "recall_at_1": 0.375, "recall_at_5": 0.7, "recall_at_10": 0.775, "recall_at_20": 0.85, "recall_at_50": 0.85, "mrr": 0.5297 } }, "construcao": { "OFFICIAL_ONLY": { "total": 40, "recall_at_1": 0.025, "recall_at_5": 0.075, "recall_at_10": 0.1, "recall_at_20": 0.175, "recall_at_50": 0.2, "mrr": 0.0578 }, "KB_ONLY": { "total": 40, "recall_at_1": 0.875, "recall_at_5": 0.95, "recall_at_10": 0.95, "recall_at_20": 0.975, "recall_at_50": 0.975, "mrr": 0.9106 }, "HYBRID": { "total": 40, "recall_at_1": 0.85, "recall_at_5": 0.975, "recall_at_10": 0.975, "recall_at_20": 0.975, "recall_at_50": 0.975, "mrr": 0.9008 } }, "mercado": { "OFFICIAL_ONLY": { "total": 40, "recall_at_1": 0.025, "recall_at_5": 0.125, "recall_at_10": 0.15, "recall_at_20": 0.15, "recall_at_50": 0.175, "mrr": 0.0637 }, "KB_ONLY": { "total": 40, "recall_at_1": 0.825, "recall_at_5": 0.95, "recall_at_10": 0.975, "recall_at_20": 0.975, "recall_at_50": 0.975, "mrr": 0.8807 }, "HYBRID": { "total": 40, "recall_at_1": 0.85, "recall_at_5": 0.95, "recall_at_10": 0.975, "recall_at_20": 0.975, "recall_at_50": 0.975, "mrr": 0.8932 } }, "moda": { "OFFICIAL_ONLY": { "total": 40, "recall_at_1": 0.025, "recall_at_5": 0.15, "recall_at_10": 0.225, "recall_at_20": 0.25, "recall_at_50": 0.425, "mrr": 0.0773 }, "KB_ONLY": { "total": 40, "recall_at_1": 0.425, "recall_at_5": 0.725, "recall_at_10": 0.85, "recall_at_20": 0.95, "recall_at_50": 1.0, "mrr": 0.5758 }, "HYBRID": { "total": 40, "recall_at_1": 0.475, "recall_at_5": 0.775, "recall_at_10": 0.9, "recall_at_20": 0.925, "recall_at_50": 1.0, "mrr": 0.6194 } }, "restaurante": { "OFFICIAL_ONLY": { "total": 40, "recall_at_1": 0.0, "recall_at_5": 0.0, "recall_at_10": 0.0, "recall_at_20": 0.05, "recall_at_50": 0.075, "mrr": 0.0039 }, "KB_ONLY": { "total": 40, "recall_at_1": 0.7, "recall_at_5": 0.875, "recall_at_10": 0.875, "recall_at_20": 0.875, "recall_at_50": 0.875, "mrr": 0.7654 }, "HYBRID": { "total": 40, "recall_at_1": 0.625, "recall_at_5": 0.875, "recall_at_10": 0.875, "recall_at_20": 0.875, "recall_at_50": 0.875, "mrr": 0.7196 } } }, "llm_liberation_gate": { "gold_recall_at_20_min": 0.8, "gold_recall_at_50_preferred": 0.9 }, "cost_usd": 0.0 }