feat: complete pre-ai local completion v3

central/tests/test_ai_controls_v3.py

@@ -0,0 +1,33 @@
+from central.app.ai.cache_policy import CONFLICT, NEEDS_REVALIDATION, VALID, cache_key, classify_knowledge_freshness
+from central.app.ai.provider import BudgetGuard
+
+
+def test_cache_key_changes_for_each_input_dimension():
+    base = ("input", "candidates", "model", "prompt", {"ncm": "n1", "tipi": "t1"})
+    key = cache_key(*base)
+    assert key != cache_key("changed", *base[1:])
+    assert key != cache_key(base[0], "changed", *base[2:])
+    assert key != cache_key(base[0], base[1], "changed", *base[3:])
+    assert key != cache_key(base[0], base[1], base[2], "changed", base[4])
+    assert key != cache_key(*base[:4], {"ncm": "n2", "tipi": "t1"})
+    assert key == cache_key(*base)
+
+
+def test_dataset_freshness_is_fail_closed():
+    assert classify_knowledge_freshness({"ncm": "v1"}, {"ncm": "v1"}) == VALID
+    assert classify_knowledge_freshness({"ncm": "v1"}, {"ncm": "v2"}) == CONFLICT
+    assert classify_knowledge_freshness({"ncm": "v1"}, {"ncm": "v1", "tipi": "v2"}) == NEEDS_REVALIDATION
+    assert classify_knowledge_freshness({}, {"ncm": "v1"}) == NEEDS_REVALIDATION
+
+
+def test_budget_blocks_before_call_and_actual_usage_cannot_exceed_limit():
+    guard = BudgetGuard(1.0, 1.0, 2.0)
+    assert guard.authorize(guard.estimate(600_000, 300_000)) is False
+    assert guard.authorize(guard.estimate(100_000, 50_000)) is True
+    assert guard.authorize_usage(100_000, 50_000, prior_cost_usd=0.9) is False
+
+
+def test_unknown_cost_requires_explicit_limited_run():
+    guard = BudgetGuard(5.0, None, None)
+    assert guard.authorize_usage(1000, 100) is False
+    assert guard.authorize_usage(1000, 100, allow_unknown_cost=True) is True

central/tests/test_ai_semantic.py

@@ -99,6 +99,16 @@ def test_work_items_dedup_by_knowledge_needed():
     assert all(i.task_type in ("INVALID_NCM", "CEST", "EX_TIPI", "PRODUCT_ROLE") for i in items)
 
 
+def test_work_items_accept_canonical_identity_map():
+    from central.app.ai.workitems import build_work_items
+    questions = [
+        {"QUESTION_ID": "Q1", "DATABASE": "A", "SCOPE": "PRODUCT", "KEY": "ncm", "CELLS": 1, "DATABASE_CODE": "1"},
+        {"QUESTION_ID": "Q2", "DATABASE": "B", "SCOPE": "PRODUCT", "KEY": "ncm", "CELLS": 1, "DATABASE_CODE": "2"},
+    ]
+    items = build_work_items(questions, canonical_products={"A:1": "GTIN:7891234567895", "B:2": "GTIN:7891234567895"})
+    assert len(items) == 1 and items[0].cells_covered == 2
+
+
 # ---------------- provider contract / structured output / budget ----------------
 
 def test_structured_output_schema_and_baseline_provider():

central/tests/test_batch_runner.py

@@ -0,0 +1,68 @@
+import pytest
+
+from central.app.ai.batch_runner import run_batches
+from central.app.ai.fake_provider import DeterministicFakeSemanticProvider
+from central.app.ai.llm_provider import LlmEvalItem
+
+
+def _item(work_item_id):
+    return LlmEvalItem(
+        work_item_id=work_item_id,
+        description=work_item_id,
+        normalized_description=work_item_id,
+        brand=None,
+        unit=None,
+        segment="test",
+        candidates=[("A", "candidate A", 1.0), ("B", "candidate B", 0.5)],
+    )
+
+
+@pytest.mark.parametrize("batch_size", [5, 10, 20])
+def test_run_batches_correlates_shuffled_answers_by_id(batch_size):
+    items = [_item(f"w{i}") for i in range(batch_size + 1)]
+    provider = DeterministicFakeSemanticProvider(shuffle=True)
+
+    result = run_batches(items, provider, batch_size=batch_size)
+
+    assert set(result.results_by_work_item_id) == {item.work_item_id for item in items}
+    assert all(result.results_by_work_item_id[item.work_item_id]["work_item_id"] == item.work_item_id for item in items)
+    assert result.errors == {}
+    assert result.provider_calls == 2
+
+
+@pytest.mark.parametrize("scenario", ["MISSING_ITEM", "DUPLICATE_ITEM", "INVALID_JSON", "OUT_OF_CANDIDATE"])
+def test_run_batches_preserves_failed_ids_for_contract_errors(scenario):
+    items = [_item("w1"), _item("w2")]
+    provider = DeterministicFakeSemanticProvider()
+
+    result = run_batches(items, provider, batch_size=2, scenario=scenario)
+
+    assert result.results_by_work_item_id == {}
+    assert set(result.errors) == {"w1", "w2"}
+    assert result.retries == 0
+    assert result.provider_calls == 1
+
+
+@pytest.mark.parametrize("scenario", ["TIMEOUT", "429", "500"])
+def test_run_batches_retries_transport_errors_and_preserves_failed_ids(scenario):
+    items = [_item("w1"), _item("w2")]
+    provider = DeterministicFakeSemanticProvider()
+
+    result = run_batches(items, provider, batch_size=2, max_retries=2, scenario=scenario)
+
+    assert result.results_by_work_item_id == {}
+    assert set(result.errors) == {"w1", "w2"}
+    assert result.provider_calls == 3
+    assert result.retries == 2
+
+
+def test_run_batches_allows_partial_failure_by_item():
+    items = [_item("w1"), _item("w2"), _item("w3")]
+    provider = DeterministicFakeSemanticProvider(fail_work_item_ids={"w2"}, failure_scenario="TIMEOUT")
+
+    result = run_batches(items, provider, batch_size=1, max_retries=1)
+
+    assert set(result.results_by_work_item_id) == {"w1", "w3"}
+    assert set(result.errors) == {"w2"}
+    assert result.provider_calls == 4
+    assert result.retries == 1

central/tests/test_fake_provider.py

@@ -0,0 +1,67 @@
+import copy
+
+import pytest
+
+from central.app.ai.fake_provider import DeterministicFakeSemanticProvider
+from central.app.ai.llm_provider import LlmAnswerError, LlmEvalItem
+from central.app.ai.provider import ProviderContractError, ProviderTransportError
+
+
+def _item(work_item_id="w1"):
+    return LlmEvalItem(
+        work_item_id=work_item_id,
+        description="AGUA MINERAL 500ML",
+        normalized_description="AGUA MINERAL 500ML",
+        brand=None,
+        unit="ML",
+        segment="mercado",
+        candidates=[("22011000", "AGUAS MINERAIS", 0.9), ("22019000", "OUTRAS AGUAS", 0.4)],
+    )
+
+
+@pytest.mark.parametrize(
+    ("scenario", "selected", "abstain"),
+    [
+        ("CORRECT", "22011000", False),
+        ("WRONG", "22019000", False),
+        ("ABSTAIN", None, True),
+    ],
+)
+def test_fake_provider_returns_deterministic_valid_answers(scenario, selected, abstain):
+    provider = DeterministicFakeSemanticProvider()
+
+    answers = provider.select_batch([_item()], scenario=scenario)
+
+    assert answers == provider.select_batch([_item()], scenario=scenario)
+    assert answers[0]["work_item_id"] == "w1"
+    assert answers[0]["selected_ncm"] == selected
+    assert answers[0]["abstain"] is abstain
+    assert provider.calls == 2
+
+
+@pytest.mark.parametrize("scenario", ["INVALID_JSON", "MISSING_ITEM", "DUPLICATE_ITEM", "OUT_OF_CANDIDATE"])
+def test_fake_provider_rejects_contract_scenarios_without_mutating_items(scenario):
+    provider = DeterministicFakeSemanticProvider()
+    items = [_item("w1"), _item("w2")]
+    original = copy.deepcopy(items)
+
+    with pytest.raises((LlmAnswerError, ProviderContractError)):
+        provider.select_batch(items, scenario=scenario)
+
+    assert items == original
+    assert provider.calls == 1
+
+
+@pytest.mark.parametrize("scenario", ["TIMEOUT", "429", "500"])
+def test_fake_provider_raises_typed_transport_errors(scenario):
+    provider = DeterministicFakeSemanticProvider()
+
+    with pytest.raises(ProviderTransportError):
+        provider.select_batch([_item()], scenario=scenario)
+
+    assert provider.calls == 1
+
+
+def test_fake_provider_rejects_unknown_scenarios():
+    with pytest.raises(ValueError):
+        DeterministicFakeSemanticProvider().select_batch([_item()], scenario="UNKNOWN")

central/tests/test_kb_consolidation.py

@@ -0,0 +1,31 @@
+def test_canonical_identity_excludes_tenant_fiscal_context():
+    from central.app.ai.kb_consolidation import canonical_product_id
+
+    a = canonical_product_id({"gtin": "7891234567895", "ncm": "22011000", "database": "A"})
+    b = canonical_product_id({"gtin": "7891234567895", "ncm": "22019000", "database": "B"})
+    assert a == b == "GTIN:7891234567895"
+
+
+def test_conflicts_are_conservative():
+    from central.app.ai.kb_consolidation import NCM_CONFLICT, PACKAGING_VARIATION, classify_conflict
+
+    assert classify_conflict([
+        {"gtin": "7891234567895", "ncm": "22011000", "normalized_description": "AGUA MINERAL", "package_quantity": "500"},
+        {"gtin": "7891234567895", "ncm": "22019000", "normalized_description": "AGUA MINERAL", "package_quantity": "500"},
+    ]) == NCM_CONFLICT
+    assert classify_conflict([
+        {"gtin": "7891234567895", "ncm": "22011000", "normalized_description": "AGUA MINERAL", "package_quantity": "500"},
+        {"gtin": "7891234567895", "ncm": "22011000", "normalized_description": "AGUA MINERAL", "package_quantity": "1L"},
+    ]) == PACKAGING_VARIATION
+
+
+def test_payload_audit_reports_observable_coverage():
+    from central.app.ai.kb_consolidation import audit_payload
+
+    report = audit_payload({"products": [
+        {"gtin": "7891234567895", "ncm": "22011000", "normalized_description": "AGUA", "observations": [{"database": "A"}]},
+        {"gtin": "7891234567895", "ncm": "22019000", "normalized_description": "AGUA", "observations": [{"database": "B"}]},
+    ]}, historical_conflicts=163)
+    assert report["historical_conflicts_reported"] == 163
+    assert report["observable_conflicts_audited"] == 1
+    assert report["unobservable_historical_conflicts"] == 162

central/tests/test_offline_ai_simulation.py

@@ -0,0 +1,15 @@
+import json
+
+
+def test_offline_simulation_runs_full_plumbing_without_accepting_fake_answers(tmp_path):
+    from central.scripts.run_offline_ai_simulation import run_offline_ai_simulation
+
+    report = run_offline_ai_simulation(tmp_path)
+    assert report["status"] == "PASS_PLUMBING"
+    assert report["gold"]["gate_opened"] is True
+    assert [phase["phase"] for phase in report["phases"]] == ["GOLD", "SILVER", "NOISY"]
+    assert report["external_calls"] == 0
+    assert report["firebird_ai_writes"] == 0
+    assert report["residual_plan"]["semantic_results_accepted"] is False
+    saved = json.loads((tmp_path / "OFFLINE_AI_SIMULATION_V1.json").read_text(encoding="utf-8"))
+    assert saved == report

central/tests/test_payloads.py

@@ -0,0 +1,8 @@
+def test_payload_compaction_preserves_candidate_codes_and_measures_reduction():
+    from central.app.ai.payloads import compact_payload, measure_payloads
+
+    payload = {"description": "agua", "debug": "remove", "official_candidates": [{"ncm": "22011000", "text": "x"}, {"ncm": "22011000", "text": "x"}, {"ncm": "22019000", "text": "y"}]}
+    compact = compact_payload(payload)
+    assert [row["ncm"] for row in compact["official_candidates"]] == ["22011000", "22019000"]
+    assert "debug" not in compact
+    assert measure_payloads([payload])["tokens_after"] < measure_payloads([payload])["tokens_before"]

central/tests/test_questionnaire.py

@@ -0,0 +1,61 @@
+import json
+
+import pytest
+
+from central.app.ai.questionnaire import build_questionnaire, group_questions, validate_answers
+
+
+def test_questionnaire_groups_by_scope_and_preserves_effects():
+    rows = [
+        {"QUESTION_ID": "Q1", "DATABASE": "ACME", "SCOPE": "COMPANY", "GROUP": "", "KEY": "entry_scope",
+         "VALID_OPTIONS": ["INTRA", "INTER", "MIXED_OR_DYNAMIC"], "PRODUCTS_AFFECTED": 2, "CELLS": 3,
+         "QUESTION": "Escopo?", "EXAMPLES": "P1; P2"},
+        {"QUESTION_ID": "Q2", "DATABASE": "ACME", "SCOPE": "GROUP", "GROUP": "G1", "KEY": "product_role",
+         "VALID_OPTIONS": ["RESALE"], "PRODUCTS_AFFECTED": 1, "CELLS": 1, "QUESTION": "Papel?"},
+    ]
+
+    result = build_questionnaire(rows)
+
+    assert [item["question_id"] for item in result] == ["Q1", "Q2"]
+    assert result[0]["affected"] == {"products": 2, "cells": 3}
+    assert "MIXED_OR_DYNAMIC" in result[0]["options"]
+    assert result[1]["scope_key"] == "G1"
+
+
+def test_answers_are_validated_without_persisting_and_mixed_scope_is_explicit():
+    questions = build_questionnaire([{
+        "QUESTION_ID": "Q1", "DATABASE": "ACME", "SCOPE": "COMPANY", "GROUP": "", "KEY": "entry_scope",
+        "VALID_OPTIONS": ["INTRA", "INTER", "MIXED_OR_DYNAMIC"], "QUESTION": "Escopo?",
+    }])
+
+    answers = validate_answers(questions, {"Q1": "MIXED_OR_DYNAMIC"})
+
+    assert answers["entries"] == [{"scope": "COMPANY", "scope_key": "", "key": "entry_scope", "value": "MIXED_OR_DYNAMIC", "reason": None, "company_key": "ACME"}]
+    assert answers["effects"][0]["behavior"] == "no_static_cfop_entrada"
+
+    with pytest.raises(ValueError, match="nothing was persisted"):
+        validate_answers(questions, {"Q1": "OUTSIDE"})
+
+
+def test_store_ipi_profile_requires_explicit_choice_and_only_plans_rollback():
+    questions = build_questionnaire([{
+        "QUESTION_ID": "QIPI-ACME", "DATABASE": "ACME", "SCOPE": "COMPANY", "GROUP": "", "KEY": "store_ipi_profile",
+        "VALID_OPTIONS": ["POPULATE_REFERENCE_RATE", "LEAVE_NOT_APPLICABLE"], "QUESTION": "IPI?",
+        "PRODUCTS_AFFECTED": 4, "CELLS": 4,
+    }])
+
+    result = validate_answers(questions, {"QIPI-ACME": "LEAVE_NOT_APPLICABLE"})
+
+    assert result["entries"][0]["value"] == "LEAVE_NOT_APPLICABLE"
+    assert result["rollback_plan"] == [{"database": "ACME", "field": "IPI", "affected_cells": 4}]
+    assert json.dumps(result)
+
+
+def test_equivalent_questions_are_grouped_with_aggregated_impact():
+    questions = build_questionnaire([
+        {"QUESTION_ID": "Q1", "DATABASE": "ACME", "SCOPE": "GROUP", "GROUP": "REFRIGERANTES", "KEY": "product_role", "VALID_OPTIONS": ["RESALE"], "CELLS": 10, "PRODUCTS_AFFECTED": 3, "QUESTION": "Qual o papel?"},
+        {"QUESTION_ID": "Q2", "DATABASE": "ACME", "SCOPE": "GROUP", "GROUP": "REFRIGERANTES", "KEY": "product_role", "VALID_OPTIONS": ["RESALE"], "CELLS": 20, "PRODUCTS_AFFECTED": 5, "QUESTION": "Qual o papel?"},
+    ])
+    groups = group_questions(questions)
+    assert len(groups) == 1 and groups[0]["question_ids"] == ["Q1", "Q2"]
+    assert groups[0]["affected"] == {"products": 8, "cells": 30}

central/tests/test_retrieval_v3.py

@@ -0,0 +1,31 @@
+def test_retrieval_v3_gtin_evidence_is_candidate_only():
+    from central.app.ai.retrieval_v3 import retrieval_v3_candidates
+
+    class Corpus:
+        pass
+    corpus = Corpus()
+    corpus.documents = {}
+    corpus.descriptions = {}
+    corpus.positions = {}
+    corpus.children = {}
+    class Index:
+        entries = []
+    # Empty V2 corpus cannot manufacture an NCM, proving no candidate is invented.
+    assert retrieval_v3_candidates(corpus, Index(), "produto", {"gtin_ncm": "22011000"}) == []
+
+
+def test_prepare_candidate_set_is_fingerprinted_and_non_decisional():
+    from central.app.ai.retrieval_v3 import prepare_candidate_set
+
+    work = type("WorkItem", (), {"task_type": "INVALID_NCM", "dedup_key": "x", "payload": {"description": "Agua mineral"}})()
+    prepared = prepare_candidate_set(work, [("22011000", "AGUAS MINERAIS", 0.9, 0.8, 0.1, 2)], dataset_versions={"ncm": "v1"})
+    assert prepared["top20_candidate_ncms"][0]["ncm"] == "22011000"
+    assert prepared["input_fingerprint"]
+    assert "selected_ncm" not in prepared
+
+
+def test_product_role_context_is_compact_and_evidence_only():
+    from central.app.ai.retrieval_v3 import compact_product_role_context
+
+    result = compact_product_role_context([{"description": "Coca Cola"}, {"description": "coca-cola"}, {"description": "Pepsi"}], limit=2)
+    assert len(result["examples"]) == 2 and result["decision"] is None and result["evidence_only"] is True