feat: complete pre-ai local completion v3

2026-09-21 16:30:27 -03:00
parent 6731eeca08
commit 84b08c2150
41 changed files with 4141 additions and 9 deletions
@@ -0,0 +1,21 @@
+"""Create the conservative KB conflict audit from the read-only seed payload."""
+import argparse
+import json
+from pathlib import Path
+
+from central.app.ai.kb_consolidation import audit_payload
+
+
+def main(payload_path="docs/runs/ai_kb_seed_payload.json", output="docs/KB_CONFLICT_AUDIT_V1.json"):
+    payload = json.loads(Path(payload_path).read_text(encoding="utf-8"))
+    report = audit_payload(payload)
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--payload-path", default="docs/runs/ai_kb_seed_payload.json")
+    parser.add_argument("--output", default="docs/KB_CONFLICT_AUDIT_V1.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.payload_path, args.output), indent=2, ensure_ascii=False))
@@ -0,0 +1,31 @@
+"""Prepare versioned candidate evidence from already computed deterministic candidates."""
+import argparse
+import json
+from pathlib import Path
+
+from central.app.ai.retrieval_v3 import prepare_candidate_set
+
+
+def main(input_path, output_path):
+    source = json.loads(Path(input_path).read_text(encoding="utf-8"))
+    prepared = []
+    for item in source.get("items", []):
+        prepared.append(prepare_candidate_set(
+            type("WorkItem", (), {"task_type": item.get("task_type", "INVALID_NCM"), "dedup_key": item["dedup_key"], "payload": item.get("payload", {})})(),
+            [tuple(row) for row in item.get("candidates", [])],
+            dataset_versions=source.get("dataset_versions"),
+            gtin_evidence=item.get("gtin_evidence"),
+            kb_neighbors=item.get("kb_neighbors"),
+            hierarchical_context=item.get("hierarchical_context"),
+        ))
+    report = {"artifact": "AI_CANDIDATE_PREPARATION_V3", "items": prepared, "semantic_decisions": 0}
+    Path(output_path).write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input_path")
+    parser.add_argument("output_path")
+    args = parser.parse_args()
+    print(json.dumps(main(args.input_path, args.output_path), indent=2, ensure_ascii=False))
@@ -0,0 +1,60 @@
+"""Offline reprocessing of residual questions against the final canonical identity map."""
+import argparse
+import json
+from collections import defaultdict
+from pathlib import Path
+
+from central.app.ai.kb_consolidation import canonical_product_id
+from central.app.ai.workitems import build_work_items
+
+
+def _erp_code(question):
+    import re
+    match = re.search(r"PRODUCT\s+(\d+)", question.get("QUESTION", ""))
+    return match.group(1) if match else str(question.get("DATABASE_CODE") or "")
+
+
+def main(questions_path, payload_path, output_path):
+    questions_doc = json.loads(Path(questions_path).read_text(encoding="utf-8"))
+    payload = json.loads(Path(payload_path).read_text(encoding="utf-8"))
+    canonical = {}
+    fingerprints = {}
+    by_observation = defaultdict(list)
+    for product in payload.get("products", []):
+        cid = canonical_product_id(product)
+        for observation in product.get("observations", []):
+            key = f"{observation.get('database')}:{observation.get('erp_code')}"
+            canonical[key] = cid
+            fingerprints[key] = product.get("fingerprint") or cid
+    questions = [dict(question, DATABASE_CODE=_erp_code(question)) for question in questions_doc.get("questions", [])]
+    items = build_work_items(questions, fingerprints, canonical)
+    by_task = {}
+    for item in items:
+        bucket = by_task.setdefault(item.task_type, {"items": 0, "cells": 0})
+        bucket["items"] += 1
+        bucket["cells"] += item.cells_covered
+    report = {
+        "artifact": "AI_RESIDUAL_PLAN_V3",
+        "work_items_before": 750,
+        "work_items_after": len(items),
+        "llm_items_before": 511,
+        # Candidate preparation is evidence-only; without an official validated
+        # answer it must not reclassify a selection item as resolved.
+        "llm_items_after": 511,
+        "llm_reclassification": "UNCHANGED_PENDING_FINAL_PROVIDER_SELECTION",
+        "by_task_type": by_task,
+        "canonical_observation_keys": len(canonical),
+        "ai_write_allowed": False,
+        "semantic_decisions": 0,
+    }
+    Path(output_path).write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--questions", default="docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json")
+    parser.add_argument("--payload", default="docs/runs/ai_kb_seed_payload.json")
+    parser.add_argument("--output", default="docs/AI_RESIDUAL_PLAN_V3.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.questions, args.payload, args.output), indent=2, ensure_ascii=False))
@@ -0,0 +1,95 @@
+"""Run the real evaluation plumbing without network, credentials, or Firebird writes."""
+
+import argparse
+import json
+from pathlib import Path
+
+from central.app.ai.batch_runner import run_batches
+from central.app.ai.fake_provider import DeterministicFakeSemanticProvider
+from central.app.ai.llm_provider import LlmEvalItem
+
+
+def _items(prefix, count):
+    return [
+        LlmEvalItem(
+            work_item_id=f"{prefix}-{index:03d}",
+            description=f"Produto de teste {index}",
+            normalized_description=f"PRODUTO TESTE {index}",
+            brand="TESTE",
+            unit="UN",
+            segment="offline",
+            candidates=[(f"{index:08d}", "CANDIDATO OFICIAL PRINCIPAL", 1.0, 1.0, 0.0, 0),
+                        (f"{index + 1:08d}", "CANDIDATO OFICIAL ALTERNATIVO", 0.5, 0.5, 0.0, 0)],
+        )
+        for index in range(count)
+    ]
+
+
+def _run_phase(name, items, batch_sizes):
+    phase = {"items": len(items), "batch_sizes": {}, "provider_calls": 0, "retries": 0, "answered": 0, "errors": 0}
+    for batch_size in batch_sizes:
+        provider = DeterministicFakeSemanticProvider()
+        result = run_batches(items, provider, batch_size=batch_size)
+        phase["batch_sizes"][str(batch_size)] = {
+            "provider_calls": result.provider_calls,
+            "retries": result.retries,
+            "answered": len(result.results_by_work_item_id),
+            "errors": len(result.errors),
+        }
+        phase["provider_calls"] += result.provider_calls
+        phase["retries"] += result.retries
+        phase["answered"] = max(phase["answered"], len(result.results_by_work_item_id))
+        phase["errors"] = max(phase["errors"], len(result.errors))
+    phase["phase"] = name
+    return phase
+
+
+def run_offline_ai_simulation(output_dir="docs/runs/offline_ai", scenario="CORRECT", batch_sizes=(5, 10, 20), final_work_items_path=None, final_candidates_path=None):
+    """Exercise GOLD -> gate -> SILVER/NOISY -> residual plumbing only."""
+    if scenario != "CORRECT":
+        raise ValueError("E2E simulation uses CORRECT only; failure scenarios belong to focused tests")
+    gold = _items("gold", 65)
+    silver = _items("silver", 118)
+    noisy = _items("noisy", 17)
+    gold_phase = _run_phase("GOLD", gold, batch_sizes)
+    correct = gold_phase["answered"]
+    selection_accuracy = correct / len(gold) if gold else 0.0
+    gate = selection_accuracy >= 0.95 and gold_phase["errors"] == 0
+    phases = [gold_phase]
+    if gate:
+        phases.extend((_run_phase("SILVER", silver, batch_sizes), _run_phase("NOISY", noisy, batch_sizes)))
+    final_inputs = {}
+    for label, path in (("work_items", final_work_items_path), ("candidates", final_candidates_path)):
+        if path:
+            source = Path(path)
+            if source.exists():
+                final_inputs[label] = {"path": str(source), "bytes": source.stat().st_size, "loaded": True}
+    report = {
+        "status": "PASS_PLUMBING" if gate else "FAIL_PLUMBING",
+        "scenario": scenario,
+        "gold": {"count": len(gold), "selection_accuracy": selection_accuracy, "gate_opened": gate},
+        "phases": phases,
+        "residual_plan": {"executed": gate, "semantic_results_accepted": False, "llm_selection_required": 511},
+        "external_calls": 0,
+        "firebird_ai_writes": 0,
+        "ai_write_allowed": False,
+        "fake_results_are_authoritative": False,
+        "final_inputs": final_inputs,
+    }
+    path = Path(output_dir) / "OFFLINE_AI_SIMULATION_V1.json"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    return report
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output-dir", default="docs/runs/offline_ai")
+    parser.add_argument("--final-work-items")
+    parser.add_argument("--final-candidates")
+    args = parser.parse_args()
+    print(json.dumps(run_offline_ai_simulation(args.output_dir, final_work_items_path=args.final_work_items, final_candidates_path=args.final_candidates), indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,37 @@
+"""Compare V3 retrieval policy with the recorded V2 holdout without retuning on GOLD rows."""
+import argparse
+import json
+from pathlib import Path
+
+
+def main(v2_path="docs/NCM_RETRIEVAL_EVALUATION_V2.json", output="docs/NCM_RETRIEVAL_EVALUATION_V3.json"):
+    baseline = json.loads(Path(v2_path).read_text(encoding="utf-8"))
+    gold = baseline.get("by_quality", {}).get("GOLD", {}).get("HYBRID", {})
+    # V3 is deliberately not activated from a synthetic or incomplete replay.
+    # Its optional evidence layer is covered by unit tests and retains V2 ordering
+    # when no new evidence is supplied.
+    report = {
+        "artifact": "NCM_RETRIEVAL_EVALUATION_V3",
+        "baseline_artifact": "NCM_RETRIEVAL_EVALUATION_V2",
+        "activation": "NOT_ACTIVATED_PENDING_FULL_HOLDOUT_REPLAY",
+        "gold_recall_at_1_before": gold.get("recall_at_1"),
+        "gold_recall_at_20_before": gold.get("recall_at_20"),
+        "gold_mrr_before": gold.get("mrr"),
+        "gold_recall_at_1_after": gold.get("recall_at_1"),
+        "gold_recall_at_20_after": gold.get("recall_at_20"),
+        "gold_mrr_after": gold.get("mrr"),
+        "local_embeddings_used": False,
+        "local_embeddings_improvement": None,
+        "evidence_features": ["BM25-compatible lexical baseline", "word/character evidence hooks", "token rarity", "hierarchy", "KB neighbors", "GTIN", "product group"],
+        "policy": "retain V2 until a complete untouched holdout replay proves recall_at_20 >= 0.954 and no regression",
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--v2", default="docs/NCM_RETRIEVAL_EVALUATION_V2.json")
+    parser.add_argument("--output", default="docs/NCM_RETRIEVAL_EVALUATION_V3.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.v2, args.output), indent=2, ensure_ascii=False))