54 lines
2.7 KiB
Python
54 lines
2.7 KiB
Python
"""Metrics and error attribution for the real LLM NCM evaluation (sections 8-9)."""
|
|
|
|
|
|
def metrics(rows) -> dict:
|
|
"""rows: [{gold_in_candidates, selected, gold, confidence, invalid}]"""
|
|
total = len(rows)
|
|
correct = sum(1 for row in rows if not row["invalid"] and row["selected"] == row["gold"])
|
|
wrong = sum(1 for row in rows if not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"])
|
|
abstain = sum(1 for row in rows if not row["invalid"] and row["selected"] is None)
|
|
invalid = sum(1 for row in rows if row["invalid"])
|
|
gold_in = sum(1 for row in rows if row["gold_in_candidates"])
|
|
answered = correct + wrong
|
|
high_conf = [row for row in rows if not row["invalid"] and row["confidence"] >= 0.95 and row["selected"] is not None]
|
|
high_conf_wrong = sum(1 for row in high_conf if row["selected"] != row["gold"])
|
|
return {
|
|
"TOTAL": total,
|
|
"CORRECT": correct,
|
|
"WRONG": wrong,
|
|
"ABSTAIN": abstain,
|
|
"INVALID": invalid,
|
|
"GOLD_IN_CANDIDATES": gold_in,
|
|
"ACCURACY_ALL": round(correct / total, 4) if total else None,
|
|
"SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE": round(correct / answered, 4) if answered else None,
|
|
"PRECISION_WHEN_HIGH_CONFIDENCE": round(sum(1 for row in high_conf if row["selected"] == row["gold"]) / len(high_conf), 4) if high_conf else None,
|
|
"HIGH_CONFIDENCE_WRONG_RATE": round(high_conf_wrong / len(high_conf), 4) if high_conf else None,
|
|
"ABSTENTION_RATE": round(abstain / total, 4) if total else None,
|
|
}
|
|
|
|
|
|
def attribution(rows) -> dict:
|
|
return {
|
|
"GOLD_NOT_IN_CANDIDATES": sum(1 for row in rows if not row["gold_in_candidates"]),
|
|
"GOLD_IN_CANDIDATES_BUT_WRONG": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"]),
|
|
"GOLD_IN_CANDIDATES_BUT_ABSTAIN": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is None),
|
|
"INVALID_OUTPUT": sum(1 for row in rows if row["invalid"]),
|
|
}
|
|
|
|
|
|
_BUCKETS = ((">=0.99", 0.99, 1.01), ("0.95-0.9899", 0.95, 0.99), ("0.90-0.9499", 0.90, 0.95), ("0.80-0.8999", 0.80, 0.90), ("<0.80", -0.01, 0.80))
|
|
|
|
|
|
def confidence_buckets(rows) -> dict:
|
|
out = {name: {"total": 0, "correct": 0} for name, _, _ in _BUCKETS}
|
|
for row in rows:
|
|
if row["invalid"] or row["selected"] is None:
|
|
continue
|
|
for name, low, high in _BUCKETS:
|
|
if low <= row["confidence"] < high:
|
|
out[name]["total"] += 1
|
|
if row["selected"] == row["gold"]:
|
|
out[name]["correct"] += 1
|
|
break
|
|
return out
|