gti
d21a9d1140
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
96 lines
4.0 KiB
Python
96 lines
4.0 KiB
Python
"""Run the real evaluation plumbing without network, credentials, or Firebird writes."""
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from central.app.ai.batch_runner import run_batches
|
|
from central.app.ai.fake_provider import DeterministicFakeSemanticProvider
|
|
from central.app.ai.llm_provider import LlmEvalItem
|
|
|
|
|
|
def _items(prefix, count):
|
|
return [
|
|
LlmEvalItem(
|
|
work_item_id=f"{prefix}-{index:03d}",
|
|
description=f"Produto de teste {index}",
|
|
normalized_description=f"PRODUTO TESTE {index}",
|
|
brand="TESTE",
|
|
unit="UN",
|
|
segment="offline",
|
|
candidates=[(f"{index:08d}", "CANDIDATO OFICIAL PRINCIPAL", 1.0, 1.0, 0.0, 0),
|
|
(f"{index + 1:08d}", "CANDIDATO OFICIAL ALTERNATIVO", 0.5, 0.5, 0.0, 0)],
|
|
)
|
|
for index in range(count)
|
|
]
|
|
|
|
|
|
def _run_phase(name, items, batch_sizes):
|
|
phase = {"items": len(items), "batch_sizes": {}, "provider_calls": 0, "retries": 0, "answered": 0, "errors": 0}
|
|
for batch_size in batch_sizes:
|
|
provider = DeterministicFakeSemanticProvider()
|
|
result = run_batches(items, provider, batch_size=batch_size)
|
|
phase["batch_sizes"][str(batch_size)] = {
|
|
"provider_calls": result.provider_calls,
|
|
"retries": result.retries,
|
|
"answered": len(result.results_by_work_item_id),
|
|
"errors": len(result.errors),
|
|
}
|
|
phase["provider_calls"] += result.provider_calls
|
|
phase["retries"] += result.retries
|
|
phase["answered"] = max(phase["answered"], len(result.results_by_work_item_id))
|
|
phase["errors"] = max(phase["errors"], len(result.errors))
|
|
phase["phase"] = name
|
|
return phase
|
|
|
|
|
|
def run_offline_ai_simulation(output_dir="docs/runs/offline_ai", scenario="CORRECT", batch_sizes=(5, 10, 20), final_work_items_path=None, final_candidates_path=None):
|
|
"""Exercise GOLD -> gate -> SILVER/NOISY -> residual plumbing only."""
|
|
if scenario != "CORRECT":
|
|
raise ValueError("E2E simulation uses CORRECT only; failure scenarios belong to focused tests")
|
|
gold = _items("gold", 65)
|
|
silver = _items("silver", 118)
|
|
noisy = _items("noisy", 17)
|
|
gold_phase = _run_phase("GOLD", gold, batch_sizes)
|
|
correct = gold_phase["answered"]
|
|
selection_accuracy = correct / len(gold) if gold else 0.0
|
|
gate = selection_accuracy >= 0.95 and gold_phase["errors"] == 0
|
|
phases = [gold_phase]
|
|
if gate:
|
|
phases.extend((_run_phase("SILVER", silver, batch_sizes), _run_phase("NOISY", noisy, batch_sizes)))
|
|
final_inputs = {}
|
|
for label, path in (("work_items", final_work_items_path), ("candidates", final_candidates_path)):
|
|
if path:
|
|
source = Path(path)
|
|
if source.exists():
|
|
final_inputs[label] = {"path": str(source), "bytes": source.stat().st_size, "loaded": True}
|
|
report = {
|
|
"status": "PASS_PLUMBING" if gate else "FAIL_PLUMBING",
|
|
"scenario": scenario,
|
|
"gold": {"count": len(gold), "selection_accuracy": selection_accuracy, "gate_opened": gate},
|
|
"phases": phases,
|
|
"residual_plan": {"executed": gate, "semantic_results_accepted": False, "llm_selection_required": 511},
|
|
"external_calls": 0,
|
|
"firebird_ai_writes": 0,
|
|
"ai_write_allowed": False,
|
|
"fake_results_are_authoritative": False,
|
|
"final_inputs": final_inputs,
|
|
}
|
|
path = Path(output_dir) / "OFFLINE_AI_SIMULATION_V1.json"
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
return report
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--output-dir", default="docs/runs/offline_ai")
|
|
parser.add_argument("--final-work-items")
|
|
parser.add_argument("--final-candidates")
|
|
args = parser.parse_args()
|
|
print(json.dumps(run_offline_ai_simulation(args.output_dir, final_work_items_path=args.final_work_items, final_candidates_path=args.final_candidates), indent=2, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|