78 lines
3.6 KiB
Python
78 lines
3.6 KiB
Python
"""Deterministic provider responses for offline pipeline tests."""
|
|
|
|
from collections.abc import Iterable
|
|
|
|
from .llm_provider import LlmAnswerError, LlmEvalItem, validate_answer
|
|
from .provider import ProviderContractError, ProviderTransportError
|
|
|
|
|
|
SCENARIOS = frozenset({
|
|
"CORRECT", "WRONG", "ABSTAIN", "INVALID_JSON", "MISSING_ITEM",
|
|
"DUPLICATE_ITEM", "OUT_OF_CANDIDATE", "TIMEOUT", "429", "500",
|
|
})
|
|
|
|
|
|
class DeterministicFakeSemanticProvider:
|
|
"""Return repeatable answers without persistence or network access."""
|
|
|
|
name = "deterministic-fake"
|
|
model = "deterministic-fake-v1"
|
|
prompt_version = "fake-provider-v1"
|
|
|
|
def __init__(self, *, shuffle: bool = False, fail_work_item_ids: Iterable[str] = (), failure_scenario: str = "TIMEOUT"):
|
|
self.shuffle = shuffle
|
|
self.fail_work_item_ids = {str(item_id) for item_id in fail_work_item_ids}
|
|
self.failure_scenario = failure_scenario
|
|
self.calls = 0
|
|
self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
|
|
|
|
def select_batch(self, items: list[LlmEvalItem], scenario: str | None = None) -> list[dict]:
|
|
scenario = scenario or "CORRECT"
|
|
if scenario not in SCENARIOS:
|
|
raise ValueError(f"unknown fake provider scenario: {scenario}")
|
|
self.calls += 1
|
|
if self.fail_work_item_ids and len(items) == 1 and str(items[0].work_item_id) in self.fail_work_item_ids:
|
|
self._raise_transport(self.failure_scenario)
|
|
if scenario in {"TIMEOUT", "429", "500"}:
|
|
self._raise_transport(scenario)
|
|
if scenario == "INVALID_JSON":
|
|
raise LlmAnswerError("fake provider returned invalid JSON")
|
|
|
|
answers = [self._answer(item, scenario) for item in items]
|
|
if scenario == "MISSING_ITEM" and answers:
|
|
answers.pop()
|
|
elif scenario == "DUPLICATE_ITEM" and answers:
|
|
answers[-1] = dict(answers[0])
|
|
if self.shuffle:
|
|
answers.reverse()
|
|
expected_ids = [str(item.work_item_id) for item in items]
|
|
answer_ids = [str(answer.get("work_item_id")) for answer in answers]
|
|
if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids):
|
|
raise ProviderContractError("fake provider response did not contain exactly one answer per work_item_id")
|
|
items_by_id = {str(item.work_item_id): item for item in items}
|
|
try:
|
|
return [validate_answer(answer, items_by_id[str(answer["work_item_id"])]) for answer in answers]
|
|
except (LlmAnswerError, KeyError, TypeError) as error:
|
|
raise ProviderContractError(str(error)) from error
|
|
|
|
@staticmethod
|
|
def _answer(item: LlmEvalItem, scenario: str) -> dict:
|
|
candidates = [str(candidate[0]) for candidate in item.candidates]
|
|
selected = None
|
|
abstain = scenario == "ABSTAIN"
|
|
if not abstain and candidates:
|
|
selected = "99999999" if scenario == "OUT_OF_CANDIDATE" else candidates[min(1, len(candidates) - 1) if scenario == "WRONG" else 0]
|
|
return {
|
|
"work_item_id": item.work_item_id,
|
|
"selected_ncm": selected,
|
|
"confidence": 0.0 if abstain else 0.75,
|
|
"abstain": abstain or selected is None,
|
|
"reason_short": f"fake scenario {scenario}",
|
|
}
|
|
|
|
@staticmethod
|
|
def _raise_transport(scenario: str) -> None:
|
|
if scenario not in {"TIMEOUT", "429", "500"}:
|
|
raise ValueError(f"unknown fake transport scenario: {scenario}")
|
|
raise ProviderTransportError(f"fake provider transport failure: {scenario}")
|