feat: complete pre-ai local completion v3

2026-09-21 16:30:27 -03:00
parent 6731eeca08
commit 84b08c2150
41 changed files with 4141 additions and 9 deletions
@@ -0,0 +1,77 @@
+"""Deterministic provider responses for offline pipeline tests."""
+
+from collections.abc import Iterable
+
+from .llm_provider import LlmAnswerError, LlmEvalItem, validate_answer
+from .provider import ProviderContractError, ProviderTransportError
+
+
+SCENARIOS = frozenset({
+    "CORRECT", "WRONG", "ABSTAIN", "INVALID_JSON", "MISSING_ITEM",
+    "DUPLICATE_ITEM", "OUT_OF_CANDIDATE", "TIMEOUT", "429", "500",
+})
+
+
+class DeterministicFakeSemanticProvider:
+    """Return repeatable answers without persistence or network access."""
+
+    name = "deterministic-fake"
+    model = "deterministic-fake-v1"
+    prompt_version = "fake-provider-v1"
+
+    def __init__(self, *, shuffle: bool = False, fail_work_item_ids: Iterable[str] = (), failure_scenario: str = "TIMEOUT"):
+        self.shuffle = shuffle
+        self.fail_work_item_ids = {str(item_id) for item_id in fail_work_item_ids}
+        self.failure_scenario = failure_scenario
+        self.calls = 0
+        self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
+
+    def select_batch(self, items: list[LlmEvalItem], scenario: str | None = None) -> list[dict]:
+        scenario = scenario or "CORRECT"
+        if scenario not in SCENARIOS:
+            raise ValueError(f"unknown fake provider scenario: {scenario}")
+        self.calls += 1
+        if self.fail_work_item_ids and len(items) == 1 and str(items[0].work_item_id) in self.fail_work_item_ids:
+            self._raise_transport(self.failure_scenario)
+        if scenario in {"TIMEOUT", "429", "500"}:
+            self._raise_transport(scenario)
+        if scenario == "INVALID_JSON":
+            raise LlmAnswerError("fake provider returned invalid JSON")
+
+        answers = [self._answer(item, scenario) for item in items]
+        if scenario == "MISSING_ITEM" and answers:
+            answers.pop()
+        elif scenario == "DUPLICATE_ITEM" and answers:
+            answers[-1] = dict(answers[0])
+        if self.shuffle:
+            answers.reverse()
+        expected_ids = [str(item.work_item_id) for item in items]
+        answer_ids = [str(answer.get("work_item_id")) for answer in answers]
+        if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids):
+            raise ProviderContractError("fake provider response did not contain exactly one answer per work_item_id")
+        items_by_id = {str(item.work_item_id): item for item in items}
+        try:
+            return [validate_answer(answer, items_by_id[str(answer["work_item_id"])]) for answer in answers]
+        except (LlmAnswerError, KeyError, TypeError) as error:
+            raise ProviderContractError(str(error)) from error
+
+    @staticmethod
+    def _answer(item: LlmEvalItem, scenario: str) -> dict:
+        candidates = [str(candidate[0]) for candidate in item.candidates]
+        selected = None
+        abstain = scenario == "ABSTAIN"
+        if not abstain and candidates:
+            selected = "99999999" if scenario == "OUT_OF_CANDIDATE" else candidates[min(1, len(candidates) - 1) if scenario == "WRONG" else 0]
+        return {
+            "work_item_id": item.work_item_id,
+            "selected_ncm": selected,
+            "confidence": 0.0 if abstain else 0.75,
+            "abstain": abstain or selected is None,
+            "reason_short": f"fake scenario {scenario}",
+        }
+
+    @staticmethod
+    def _raise_transport(scenario: str) -> None:
+        if scenario not in {"TIMEOUT", "429", "500"}:
+            raise ValueError(f"unknown fake transport scenario: {scenario}")
+        raise ProviderTransportError(f"fake provider transport failure: {scenario}")