gti
d21a9d1140
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
131 lines
6.9 KiB
Python
131 lines
6.9 KiB
Python
"""Real LLM provider contract: strict answers, batching safety, cache, metrics attribution."""
|
|
import json
|
|
|
|
from sqlalchemy import create_engine, select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from central.app.models import Base, AiEvalCache
|
|
|
|
|
|
def _item(work_item_id="w1", candidates=None):
|
|
from central.app.ai.llm_provider import LlmEvalItem
|
|
return LlmEvalItem(work_item_id=work_item_id, description="AGUA MINERAL 500ML", normalized_description="AGUA MINERAL 500ML",
|
|
brand=None, unit="ML", segment="mercado", candidates=candidates or [("22011000", "AGUAS MINERAIS", 0.9), ("22019000", "AGUAS OUTRAS", 0.4)])
|
|
|
|
|
|
def _answer(work_item_id="w1", ncm="22011000", confidence=0.9, abstain=False):
|
|
return {"work_item_id": work_item_id, "selected_ncm": ncm, "confidence": confidence, "abstain": abstain, "reason_short": "desc match"}
|
|
|
|
|
|
def test_validate_answer_accepts_only_candidate_or_abstain():
|
|
from central.app.ai.llm_provider import LlmAnswerError, validate_answer
|
|
item = _item()
|
|
assert validate_answer(_answer(), item)["selected_ncm"] == "22011000"
|
|
abstain = validate_answer(_answer(ncm=None, abstain=True), item)
|
|
assert abstain["abstain"] is True and abstain["selected_ncm"] is None
|
|
for bad in (_answer(ncm="99999999"), _answer(work_item_id="other"), _answer(ncm=None), _answer(confidence=1.5), _answer(ncm="22011000", abstain=True)):
|
|
try:
|
|
validate_answer(bad, item)
|
|
raise AssertionError("must reject")
|
|
except LlmAnswerError:
|
|
pass
|
|
|
|
|
|
def test_provider_batch_requires_1_to_1_answers():
|
|
from central.app.ai.llm_provider import LlmAnswerError, OpenAICompatibleProvider
|
|
|
|
def transport(payload):
|
|
content = json.dumps([_answer(), _answer()]) # two answers for one item
|
|
return {"choices": [{"message": {"content": content}}], "usage": {"prompt_tokens": 10, "completion_tokens": 5}}
|
|
|
|
provider = OpenAICompatibleProvider(api_key="test-key", model="m", transport=transport)
|
|
try:
|
|
provider.select_batch([_item()])
|
|
raise AssertionError("must reject")
|
|
except LlmAnswerError:
|
|
pass
|
|
assert provider.http_calls == 1 # contract violations are NOT retried
|
|
|
|
|
|
def test_provider_retries_transport_failures_and_counts_usage():
|
|
from central.app.ai.llm_provider import OpenAICompatibleProvider
|
|
calls = {"n": 0}
|
|
|
|
def transport(payload):
|
|
calls["n"] += 1
|
|
if calls["n"] == 1:
|
|
raise ConnectionError("boom")
|
|
content = json.dumps([_answer()])
|
|
return {"choices": [{"message": {"content": content}}], "usage": {"prompt_tokens": 100, "completion_tokens": 20, "prompt_tokens_details": {"cached_tokens": 50}}}
|
|
|
|
provider = OpenAICompatibleProvider(api_key="k", model="m", transport=transport)
|
|
answers = provider.select_batch([_item()])
|
|
assert answers[0]["selected_ncm"] == "22011000"
|
|
assert provider.http_calls == 2 and provider.retries == 1
|
|
assert provider.usage == {"input_tokens": 100, "output_tokens": 20, "cached_tokens": 50}
|
|
|
|
|
|
def test_prompt_hides_gold_and_carries_candidates():
|
|
from central.app.ai.llm_provider import candidate_set_hash, input_fingerprint, item_prompt_payload
|
|
item = _item()
|
|
payload = item_prompt_payload(item)
|
|
assert "22011000" in json.dumps(payload) # candidates present
|
|
assert item.normalized_description in json.dumps(payload)
|
|
assert len(candidate_set_hash(item)) == 64 and len(input_fingerprint(item)) == 64
|
|
changed = _item(candidates=[("11111111", "X", 0.1)])
|
|
assert candidate_set_hash(changed) != candidate_set_hash(item)
|
|
|
|
|
|
def test_eval_cache_roundtrip_deduplicates():
|
|
engine = create_engine("sqlite://")
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
row = AiEvalCache(input_fingerprint="f", candidate_set_hash="c", prompt_version="p", model="m", dataset_version_ncm="d", phase="gold", answer={"selected_ncm": "22011000"}, gold_ncm="22011000")
|
|
session.add(row); session.commit()
|
|
existing = session.scalar(select(AiEvalCache).where(AiEvalCache.input_fingerprint == "f", AiEvalCache.candidate_set_hash == "c", AiEvalCache.prompt_version == "p", AiEvalCache.model == "m", AiEvalCache.dataset_version_ncm == "d"))
|
|
assert existing.answer["selected_ncm"] == "22011000"
|
|
|
|
|
|
def test_metrics_and_error_attribution():
|
|
from central.app.ai.eval_metrics import attribution, confidence_buckets, metrics
|
|
rows = [
|
|
{"gold_in_candidates": True, "selected": "A", "gold": "A", "confidence": 0.99, "invalid": False},
|
|
{"gold_in_candidates": True, "selected": "B", "gold": "A", "confidence": 0.97, "invalid": False},
|
|
{"gold_in_candidates": True, "selected": None, "gold": "A", "confidence": 0.5, "invalid": False},
|
|
{"gold_in_candidates": False, "selected": "B", "gold": "A", "confidence": 0.9, "invalid": False},
|
|
{"gold_in_candidates": True, "selected": "zzz", "gold": "A", "confidence": 0.9, "invalid": True},
|
|
]
|
|
m = metrics(rows)
|
|
assert m["TOTAL"] == 5 and m["CORRECT"] == 1 and m["WRONG"] == 2 and m["ABSTAIN"] == 1 and m["INVALID"] == 1
|
|
assert m["GOLD_IN_CANDIDATES"] == 4
|
|
assert abs(m["SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE"] - 1 / 3) < 1e-4 # correct / (correct+wrong) among valid answers
|
|
assert abs(m["ABSTENTION_RATE"] - 0.2) < 1e-9
|
|
a = attribution(rows)
|
|
assert a["GOLD_NOT_IN_CANDIDATES"] == 1 and a["GOLD_IN_CANDIDATES_BUT_WRONG"] == 1 and a["GOLD_IN_CANDIDATES_BUT_ABSTAIN"] == 1
|
|
buckets = confidence_buckets(rows)
|
|
assert buckets[">=0.99"] == {"total": 1, "correct": 1} and buckets["0.95-0.9899"] == {"total": 1, "correct": 0}
|
|
|
|
|
|
def test_zai_model_selection_rules():
|
|
import importlib.util
|
|
spec = importlib.util.spec_from_file_location("eval_runner", "central/scripts/run_real_llm_ncm_eval.py")
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
available = ["glm-5.3", "glm-5.3-flash", "glm-4.7", "other-vendor-x"]
|
|
assert module._select_model(available, "glm-5.3") == ("glm-5.3", "requested")
|
|
# requested missing -> current GLM flash fallback, never another vendor, never invented
|
|
assert module._select_model(available, "glm-9.9") == ("glm-5.3-flash", "fallback_glm_flash")
|
|
only_generic = ["glm-4", "glm-4.7"]
|
|
assert module._select_model(only_generic, "glm-9.9") == ("glm-4.7", "fallback_glm_latest")
|
|
assert module._select_model(["gpt-x", "claude-y"], "glm-5.3") == (None, "no_glm_available")
|
|
assert module._diagnose_http(401) == "credential invalid (AI_API_KEY)"
|
|
|
|
|
|
def test_budget_guard_unknown_cost_limited_run():
|
|
from central.app.ai.provider import BudgetGuard
|
|
guard = BudgetGuard(max_run_cost_usd=5.0, price_in_per_m=None, price_out_per_m=None)
|
|
plan = guard.estimate(1000, 100)
|
|
assert plan["estimated_cost"] is None and plan["cost_unknown"] is True
|
|
assert guard.authorize(plan) is False # residual runs refused without prices
|
|
assert guard.authorize(plan, allow_unknown_cost=True) is True # explicitly limited GOLD benchmark may opt in
|