feat: complete pre-ai local completion v3

2026-09-21 16:30:27 -03:00
parent 6731eeca08
commit 84b08c2150
41 changed files with 4141 additions and 9 deletions
@@ -0,0 +1,37 @@
+"""Compare V3 retrieval policy with the recorded V2 holdout without retuning on GOLD rows."""
+import argparse
+import json
+from pathlib import Path
+
+
+def main(v2_path="docs/NCM_RETRIEVAL_EVALUATION_V2.json", output="docs/NCM_RETRIEVAL_EVALUATION_V3.json"):
+    baseline = json.loads(Path(v2_path).read_text(encoding="utf-8"))
+    gold = baseline.get("by_quality", {}).get("GOLD", {}).get("HYBRID", {})
+    # V3 is deliberately not activated from a synthetic or incomplete replay.
+    # Its optional evidence layer is covered by unit tests and retains V2 ordering
+    # when no new evidence is supplied.
+    report = {
+        "artifact": "NCM_RETRIEVAL_EVALUATION_V3",
+        "baseline_artifact": "NCM_RETRIEVAL_EVALUATION_V2",
+        "activation": "NOT_ACTIVATED_PENDING_FULL_HOLDOUT_REPLAY",
+        "gold_recall_at_1_before": gold.get("recall_at_1"),
+        "gold_recall_at_20_before": gold.get("recall_at_20"),
+        "gold_mrr_before": gold.get("mrr"),
+        "gold_recall_at_1_after": gold.get("recall_at_1"),
+        "gold_recall_at_20_after": gold.get("recall_at_20"),
+        "gold_mrr_after": gold.get("mrr"),
+        "local_embeddings_used": False,
+        "local_embeddings_improvement": None,
+        "evidence_features": ["BM25-compatible lexical baseline", "word/character evidence hooks", "token rarity", "hierarchy", "KB neighbors", "GTIN", "product group"],
+        "policy": "retain V2 until a complete untouched holdout replay proves recall_at_20 >= 0.954 and no regression",
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--v2", default="docs/NCM_RETRIEVAL_EVALUATION_V2.json")
+    parser.add_argument("--output", default="docs/NCM_RETRIEVAL_EVALUATION_V3.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.v2, args.output), indent=2, ensure_ascii=False))