"""Deterministic provider responses for offline pipeline tests.""" from collections.abc import Iterable from .llm_provider import LlmAnswerError, LlmEvalItem, validate_answer from .provider import ProviderContractError, ProviderTransportError SCENARIOS = frozenset({ "CORRECT", "WRONG", "ABSTAIN", "INVALID_JSON", "MISSING_ITEM", "DUPLICATE_ITEM", "OUT_OF_CANDIDATE", "TIMEOUT", "429", "500", }) class DeterministicFakeSemanticProvider: """Return repeatable answers without persistence or network access.""" name = "deterministic-fake" model = "deterministic-fake-v1" prompt_version = "fake-provider-v1" def __init__(self, *, shuffle: bool = False, fail_work_item_ids: Iterable[str] = (), failure_scenario: str = "TIMEOUT"): self.shuffle = shuffle self.fail_work_item_ids = {str(item_id) for item_id in fail_work_item_ids} self.failure_scenario = failure_scenario self.calls = 0 self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0} def select_batch(self, items: list[LlmEvalItem], scenario: str | None = None) -> list[dict]: scenario = scenario or "CORRECT" if scenario not in SCENARIOS: raise ValueError(f"unknown fake provider scenario: {scenario}") self.calls += 1 if self.fail_work_item_ids and len(items) == 1 and str(items[0].work_item_id) in self.fail_work_item_ids: self._raise_transport(self.failure_scenario) if scenario in {"TIMEOUT", "429", "500"}: self._raise_transport(scenario) if scenario == "INVALID_JSON": raise LlmAnswerError("fake provider returned invalid JSON") answers = [self._answer(item, scenario) for item in items] if scenario == "MISSING_ITEM" and answers: answers.pop() elif scenario == "DUPLICATE_ITEM" and answers: answers[-1] = dict(answers[0]) if self.shuffle: answers.reverse() expected_ids = [str(item.work_item_id) for item in items] answer_ids = [str(answer.get("work_item_id")) for answer in answers] if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids): raise ProviderContractError("fake provider response did not contain exactly one answer per work_item_id") items_by_id = {str(item.work_item_id): item for item in items} try: return [validate_answer(answer, items_by_id[str(answer["work_item_id"])]) for answer in answers] except (LlmAnswerError, KeyError, TypeError) as error: raise ProviderContractError(str(error)) from error @staticmethod def _answer(item: LlmEvalItem, scenario: str) -> dict: candidates = [str(candidate[0]) for candidate in item.candidates] selected = None abstain = scenario == "ABSTAIN" if not abstain and candidates: selected = "99999999" if scenario == "OUT_OF_CANDIDATE" else candidates[min(1, len(candidates) - 1) if scenario == "WRONG" else 0] return { "work_item_id": item.work_item_id, "selected_ncm": selected, "confidence": 0.0 if abstain else 0.75, "abstain": abstain or selected is None, "reason_short": f"fake scenario {scenario}", } @staticmethod def _raise_transport(scenario: str) -> None: if scenario not in {"TIMEOUT", "429", "500"}: raise ValueError(f"unknown fake transport scenario: {scenario}") raise ProviderTransportError(f"fake provider transport failure: {scenario}")