feat: complete pre-ai local completion v3

2026-09-21 16:30:27 -03:00
parent 6731eeca08
commit 84b08c2150
41 changed files with 4141 additions and 9 deletions
@@ -0,0 +1,95 @@
+"""Run the real evaluation plumbing without network, credentials, or Firebird writes."""
+
+import argparse
+import json
+from pathlib import Path
+
+from central.app.ai.batch_runner import run_batches
+from central.app.ai.fake_provider import DeterministicFakeSemanticProvider
+from central.app.ai.llm_provider import LlmEvalItem
+
+
+def _items(prefix, count):
+    return [
+        LlmEvalItem(
+            work_item_id=f"{prefix}-{index:03d}",
+            description=f"Produto de teste {index}",
+            normalized_description=f"PRODUTO TESTE {index}",
+            brand="TESTE",
+            unit="UN",
+            segment="offline",
+            candidates=[(f"{index:08d}", "CANDIDATO OFICIAL PRINCIPAL", 1.0, 1.0, 0.0, 0),
+                        (f"{index + 1:08d}", "CANDIDATO OFICIAL ALTERNATIVO", 0.5, 0.5, 0.0, 0)],
+        )
+        for index in range(count)
+    ]
+
+
+def _run_phase(name, items, batch_sizes):
+    phase = {"items": len(items), "batch_sizes": {}, "provider_calls": 0, "retries": 0, "answered": 0, "errors": 0}
+    for batch_size in batch_sizes:
+        provider = DeterministicFakeSemanticProvider()
+        result = run_batches(items, provider, batch_size=batch_size)
+        phase["batch_sizes"][str(batch_size)] = {
+            "provider_calls": result.provider_calls,
+            "retries": result.retries,
+            "answered": len(result.results_by_work_item_id),
+            "errors": len(result.errors),
+        }
+        phase["provider_calls"] += result.provider_calls
+        phase["retries"] += result.retries
+        phase["answered"] = max(phase["answered"], len(result.results_by_work_item_id))
+        phase["errors"] = max(phase["errors"], len(result.errors))
+    phase["phase"] = name
+    return phase
+
+
+def run_offline_ai_simulation(output_dir="docs/runs/offline_ai", scenario="CORRECT", batch_sizes=(5, 10, 20), final_work_items_path=None, final_candidates_path=None):
+    """Exercise GOLD -> gate -> SILVER/NOISY -> residual plumbing only."""
+    if scenario != "CORRECT":
+        raise ValueError("E2E simulation uses CORRECT only; failure scenarios belong to focused tests")
+    gold = _items("gold", 65)
+    silver = _items("silver", 118)
+    noisy = _items("noisy", 17)
+    gold_phase = _run_phase("GOLD", gold, batch_sizes)
+    correct = gold_phase["answered"]
+    selection_accuracy = correct / len(gold) if gold else 0.0
+    gate = selection_accuracy >= 0.95 and gold_phase["errors"] == 0
+    phases = [gold_phase]
+    if gate:
+        phases.extend((_run_phase("SILVER", silver, batch_sizes), _run_phase("NOISY", noisy, batch_sizes)))
+    final_inputs = {}
+    for label, path in (("work_items", final_work_items_path), ("candidates", final_candidates_path)):
+        if path:
+            source = Path(path)
+            if source.exists():
+                final_inputs[label] = {"path": str(source), "bytes": source.stat().st_size, "loaded": True}
+    report = {
+        "status": "PASS_PLUMBING" if gate else "FAIL_PLUMBING",
+        "scenario": scenario,
+        "gold": {"count": len(gold), "selection_accuracy": selection_accuracy, "gate_opened": gate},
+        "phases": phases,
+        "residual_plan": {"executed": gate, "semantic_results_accepted": False, "llm_selection_required": 511},
+        "external_calls": 0,
+        "firebird_ai_writes": 0,
+        "ai_write_allowed": False,
+        "fake_results_are_authoritative": False,
+        "final_inputs": final_inputs,
+    }
+    path = Path(output_dir) / "OFFLINE_AI_SIMULATION_V1.json"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    return report
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output-dir", default="docs/runs/offline_ai")
+    parser.add_argument("--final-work-items")
+    parser.add_argument("--final-candidates")
+    args = parser.parse_args()
+    print(json.dumps(run_offline_ai_simulation(args.output_dir, final_work_items_path=args.final_work_items, final_candidates_path=args.final_candidates), indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()