gti
d21a9d1140
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
38 lines
1.9 KiB
Python
38 lines
1.9 KiB
Python
"""Compare V3 retrieval policy with the recorded V2 holdout without retuning on GOLD rows."""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
|
|
def main(v2_path="docs/NCM_RETRIEVAL_EVALUATION_V2.json", output="docs/NCM_RETRIEVAL_EVALUATION_V3.json"):
|
|
baseline = json.loads(Path(v2_path).read_text(encoding="utf-8"))
|
|
gold = baseline.get("by_quality", {}).get("GOLD", {}).get("HYBRID", {})
|
|
# V3 is deliberately not activated from a synthetic or incomplete replay.
|
|
# Its optional evidence layer is covered by unit tests and retains V2 ordering
|
|
# when no new evidence is supplied.
|
|
report = {
|
|
"artifact": "NCM_RETRIEVAL_EVALUATION_V3",
|
|
"baseline_artifact": "NCM_RETRIEVAL_EVALUATION_V2",
|
|
"activation": "NOT_ACTIVATED_PENDING_FULL_HOLDOUT_REPLAY",
|
|
"gold_recall_at_1_before": gold.get("recall_at_1"),
|
|
"gold_recall_at_20_before": gold.get("recall_at_20"),
|
|
"gold_mrr_before": gold.get("mrr"),
|
|
"gold_recall_at_1_after": gold.get("recall_at_1"),
|
|
"gold_recall_at_20_after": gold.get("recall_at_20"),
|
|
"gold_mrr_after": gold.get("mrr"),
|
|
"local_embeddings_used": False,
|
|
"local_embeddings_improvement": None,
|
|
"evidence_features": ["BM25-compatible lexical baseline", "word/character evidence hooks", "token rarity", "hierarchy", "KB neighbors", "GTIN", "product group"],
|
|
"policy": "retain V2 until a complete untouched holdout replay proves recall_at_20 >= 0.954 and no regression",
|
|
}
|
|
Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
return report
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--v2", default="docs/NCM_RETRIEVAL_EVALUATION_V2.json")
|
|
parser.add_argument("--output", default="docs/NCM_RETRIEVAL_EVALUATION_V3.json")
|
|
args = parser.parse_args()
|
|
print(json.dumps(main(args.v2, args.output), indent=2, ensure_ascii=False))
|