chore: capture semantic pipeline baseline

central/app/ai/__init__.py

@@ -0,0 +1,8 @@
+"""AI semantic layer V1 — NOT a fiscal authority.
+
+Rules frozen by the operator contract:
+- AI only selects among official candidates; it never invents codes.
+- Cache first: KB exact/fingerprint hits never trigger AI calls.
+- No AI for origin, ST retention, or company-level decisions.
+- AI_WRITE_ALLOWED=false in V1: results stay AI_CANDIDATE_VALIDATED.
+"""

central/app/ai/candidates.py

@@ -0,0 +1,53 @@
+"""Deterministic candidate generation from the ACTIVE official datasets.
+
+The AI never receives an open question: for every task type the Hub first builds
+the official candidate list (NCM top-K by description, CEST links for the NCM,
+TIPI EX rows for the NCM). Providers may only pick from this list.
+"""
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from ..models import CestEntry, CestNcmEntry, Dataset, NcmEntry, TipiEntry
+from .fingerprint import token_stems
+
+
+def _active_dataset(session: Session, kind: str):
+    return session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE"))
+
+
+def ncm_top_k(session: Session, description, k: int = 10):
+    """Top-K official NCM codes by (stemmed) token overlap with the product description."""
+    dataset = _active_dataset(session, "NCM")
+    if dataset is None:
+        return []
+    stems = token_stems(description)
+    if not stems:
+        return []
+    scored = []
+    for entry in session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all():
+        score = len(stems & token_stems(entry.description)) / len(stems)
+        if score > 0:
+            scored.append((entry.code, entry.description, round(score, 4)))
+    scored.sort(key=lambda row: (-row[2], row[0]))
+    return scored[:k]
+
+
+def cest_candidates_for_ncm(session: Session, ncm: str):
+    dataset = _active_dataset(session, "CEST")
+    if dataset is None:
+        return []
+    rows = session.execute(
+        select(CestEntry, CestNcmEntry.ncm_code)
+        .join(CestNcmEntry, CestNcmEntry.cest_entry_id == CestEntry.id)
+        .where(CestEntry.dataset_id == dataset.id)
+    ).all()
+    return list(dict.fromkeys(entry for entry, ncm_code in rows if ncm.startswith(ncm_code)))
+
+
+def ex_tipi_candidates_for_ncm(session: Session, ncm: str):
+    dataset = _active_dataset(session, "TIPI_IPI")
+    if dataset is None:
+        return []
+    return list(dict.fromkeys(
+        entry for entry in session.scalars(select(TipiEntry).where(TipiEntry.dataset_id == dataset.id, TipiEntry.ncm == ncm, TipiEntry.ex.is_not(None))).all()
+    ))

central/app/ai/eval_metrics.py

@@ -0,0 +1,53 @@
+"""Metrics and error attribution for the real LLM NCM evaluation (sections 8-9)."""
+
+
+def metrics(rows) -> dict:
+    """rows: [{gold_in_candidates, selected, gold, confidence, invalid}]"""
+    total = len(rows)
+    correct = sum(1 for row in rows if not row["invalid"] and row["selected"] == row["gold"])
+    wrong = sum(1 for row in rows if not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"])
+    abstain = sum(1 for row in rows if not row["invalid"] and row["selected"] is None)
+    invalid = sum(1 for row in rows if row["invalid"])
+    gold_in = sum(1 for row in rows if row["gold_in_candidates"])
+    answered = correct + wrong
+    high_conf = [row for row in rows if not row["invalid"] and row["confidence"] >= 0.95 and row["selected"] is not None]
+    high_conf_wrong = sum(1 for row in high_conf if row["selected"] != row["gold"])
+    return {
+        "TOTAL": total,
+        "CORRECT": correct,
+        "WRONG": wrong,
+        "ABSTAIN": abstain,
+        "INVALID": invalid,
+        "GOLD_IN_CANDIDATES": gold_in,
+        "ACCURACY_ALL": round(correct / total, 4) if total else None,
+        "SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE": round(correct / answered, 4) if answered else None,
+        "PRECISION_WHEN_HIGH_CONFIDENCE": round(sum(1 for row in high_conf if row["selected"] == row["gold"]) / len(high_conf), 4) if high_conf else None,
+        "HIGH_CONFIDENCE_WRONG_RATE": round(high_conf_wrong / len(high_conf), 4) if high_conf else None,
+        "ABSTENTION_RATE": round(abstain / total, 4) if total else None,
+    }
+
+
+def attribution(rows) -> dict:
+    return {
+        "GOLD_NOT_IN_CANDIDATES": sum(1 for row in rows if not row["gold_in_candidates"]),
+        "GOLD_IN_CANDIDATES_BUT_WRONG": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"]),
+        "GOLD_IN_CANDIDATES_BUT_ABSTAIN": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is None),
+        "INVALID_OUTPUT": sum(1 for row in rows if row["invalid"]),
+    }
+
+
+_BUCKETS = ((">=0.99", 0.99, 1.01), ("0.95-0.9899", 0.95, 0.99), ("0.90-0.9499", 0.90, 0.95), ("0.80-0.8999", 0.80, 0.90), ("<0.80", -0.01, 0.80))
+
+
+def confidence_buckets(rows) -> dict:
+    out = {name: {"total": 0, "correct": 0} for name, _, _ in _BUCKETS}
+    for row in rows:
+        if row["invalid"] or row["selected"] is None:
+            continue
+        for name, low, high in _BUCKETS:
+            if low <= row["confidence"] < high:
+                out[name]["total"] += 1
+                if row["selected"] == row["gold"]:
+                    out[name]["correct"] += 1
+                break
+    return out

central/app/ai/fingerprint.py

@@ -0,0 +1,75 @@
+"""Product identity: stable fingerprints and GTIN validation.
+
+Global identity uses ONLY stable attributes (normalized description, brand, unit,
+package quantity, GTIN). Fiscal/company context (CFOP, CSOSN, CST, rate, ST, tenant)
+never participates: those vary per context and must not merge distinct knowledge.
+"""
+import hashlib
+import re
+import unicodedata
+
+from ..classification import ENGINE_VERSION  # noqa: F401  (re-exported for the AI layer)
+
+
+def normalize_description(value) -> str:
+    """NFKD -> ascii -> A-Z0-9 tokens, space-joined (same canonical form as classification)."""
+    text = unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper()
+    return " ".join(re.sub(r"[^A-Z0-9]+", " ", text).split())
+
+
+def _normalize_token(value) -> str:
+    return normalize_description(value)
+
+
+def _normalize_quantity(value) -> str:
+    digits = re.sub(r"[^0-9.,]", "", str(value or ""))
+    if not digits:
+        return ""
+    try:
+        number = float(digits.replace(",", "."))
+    except ValueError:
+        return digits
+    return f"{number:g}"
+
+
+def product_fingerprint(description, brand=None, unit=None, package_quantity=None) -> str:
+    parts = [_normalize_token(description), _normalize_token(brand), _normalize_token(unit), _normalize_quantity(package_quantity)]
+    return hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()
+
+
+def _stem(token: str) -> str:
+    """Minimal Portuguese plural stemming so AGUAS matches AGUA / MINERAIS matches MINERAL."""
+    if len(token) > 4 and token.endswith("AIS"):
+        return token[:-3] + "AL"
+    if len(token) > 4 and token.endswith("ES"):
+        return token[:-2]
+    if len(token) > 3 and token.endswith("S"):
+        return token[:-1]
+    return token
+
+
+def token_stems(value) -> set:
+    return {_stem(token) for token in normalize_description(value).split()}
+
+
+def overlap_score(query, official) -> float:
+    """Fraction of query description stems present in the official description stems."""
+    stems = token_stems(query)
+    if not stems:
+        return 0.0
+    return len(stems & token_stems(official)) / len(stems)
+
+
+def validate_gtin(value) -> str | None:
+    """Returns the canonical GTIN when the checksum is valid, else None.
+    Accepts GTIN-8/12/13/14 (EAN-8/UPC-12/EAN-13/GTIN-14)."""
+    digits = re.sub(r"\D", "", str(value or ""))
+    if len(digits) not in (8, 12, 13, 14):
+        return None
+    total = 0
+    for index, char in enumerate(reversed(digits)):
+        digit = int(char)
+        total += digit * (1 if index % 2 == 0 else 3)
+    if total % 10 != 0:
+        return None
+    return digits

central/app/ai/kb.py

@@ -0,0 +1,121 @@
+"""Product Knowledge Base service: identity resolution, versioned knowledge, conflicts.
+
+Identity order (operator contract): 1. exact GTIN; 2. strong fingerprint; 3. semantic
+fingerprint candidate; 4. unknown. Conflicting knowledge for the same key is kept
+side by side and surfaced as KNOWLEDGE_CONFLICT — never picked by majority.
+"""
+from datetime import datetime, timezone
+
+from sqlalchemy import func as sa_func, select
+from ..models import KbKnowledge, KbObservation, KbProduct
+
+EXACT, FINGERPRINT = "EXACT", "FINGERPRINT"
+_AUTHORITATIVE = {"OFFICIAL_DATASET", "OPERATOR", "AI_PLUS_OFFICIAL_VALIDATION"}
+
+
+class KnowledgeBase:
+    def __init__(self, session):
+        self.session = session
+
+    # ----- identity -----
+
+    def upsert_product(self, fingerprint, gtin=None, normalized_description="", brand=None, unit=None, package_quantity=None, category=None):
+        gtin = gtin or None
+        product = None
+        if gtin:
+            product = self.session.scalar(select(KbProduct).where(KbProduct.gtin == gtin))
+        if product is None:
+            product = self.session.scalar(select(KbProduct).where(KbProduct.fingerprint == fingerprint))
+        if product is None:
+            product = KbProduct(fingerprint=fingerprint, normalized_description=normalized_description or "")
+            self.session.add(product)
+        # identity upgrades (never downgrades)
+        if gtin and not product.gtin:
+            product.gtin = gtin
+            product.identity_kind = "GTIN"
+        if normalized_description and not product.normalized_description:
+            product.normalized_description = normalized_description
+        for field, value in (("brand", brand), ("unit", unit), ("package_quantity", package_quantity), ("category", category)):
+            if value and not getattr(product, field):
+                setattr(product, field, value)
+        self.session.flush()
+        return product
+
+    def resolve(self, gtin=None, fingerprint=None):
+        """Returns (product|None, hit_kind). Cache-first: EXACT (GTIN) beats FINGERPRINT."""
+        if gtin:
+            product = self.session.scalar(select(KbProduct).where(KbProduct.gtin == gtin))
+            if product:
+                return product, EXACT
+        if fingerprint:
+            product = self.session.scalar(select(KbProduct).where(KbProduct.fingerprint == fingerprint))
+            if product:
+                return product, FINGERPRINT
+        return None, None
+
+    # ----- knowledge -----
+
+    def add_knowledge(self, kb_product_id, knowledge_key, value, source, confidence=0.0, evidence=None, model=None, prompt_version=None, dataset_versions=None):
+        value = str(value)
+        row = self.session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == kb_product_id, KbKnowledge.knowledge_key == knowledge_key, KbKnowledge.value == value))
+        if row is None:
+            row = KbKnowledge(kb_product_id=kb_product_id, knowledge_key=knowledge_key, value=value, source=source, confidence=confidence, evidence=evidence or {}, model=model, prompt_version=prompt_version, dataset_versions=dataset_versions or {})
+            self.session.add(row)
+        else:
+            # authoritative sources upgrade provenance/confidence of the same value; history preserved
+            if source in _AUTHORITATIVE:
+                row.source = source
+                row.confidence = max(row.confidence, confidence)
+                row.evidence = {**row.evidence, **(evidence or {})}
+                if dataset_versions:
+                    row.dataset_versions = {**row.dataset_versions, **dataset_versions}
+            if source == "OFFICIAL_DATASET":
+                row.validated_at = datetime.now(timezone.utc)
+        self.session.flush()
+        return row
+
+    def current_knowledge(self, kb_product_id):
+        rows = self.session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == kb_product_id)).all()
+        by_key: dict = {}
+        for row in rows:
+            by_key.setdefault(row.knowledge_key, []).append(row)
+        result = {}
+        for key, entries in by_key.items():
+            values = {entry.value for entry in entries}
+            if len(values) > 1:
+                result[key] = {"conflict": True, "values": values}
+            else:
+                best = max(entries, key=lambda entry: entry.confidence)
+                result[key] = {"conflict": False, "value": best.value, "source": best.source, "confidence": best.confidence}
+        return result
+
+    # ----- observations -----
+
+    def record_observation(self, kb_product_id, tenant_id, company_key, erp_code, database_label, observed_fields, source="ERP_OBSERVATION"):
+        row = KbObservation(kb_product_id=kb_product_id, tenant_id=tenant_id, company_key=company_key, erp_code=erp_code, database_label=database_label, observed_fields=observed_fields or {}, source=source)
+        self.session.add(row); self.session.flush()
+        return row
+
+    def observation_summary(self, kb_product_id):
+        rows = self.session.scalars(select(KbObservation).where(KbObservation.kb_product_id == kb_product_id)).all()
+        agreement: dict = {}
+        for row in rows:
+            for field, value in (row.observed_fields or {}).items():
+                if value in (None, ""):
+                    continue
+                bucket = agreement.setdefault(field, {})
+                bucket[str(value)] = bucket.get(str(value), 0) + 1
+        conflicts = {field: values for field, values in agreement.items() if len(values) > 1}
+        return {"observation_count": len(rows), "tenant_count": len({row.tenant_id for row in rows}), "agreement": agreement, "conflicts": conflicts}
+
+    # ----- stats -----
+
+    def stats(self):
+        products = self.session.scalar(select(sa_func.count()).select_from(KbProduct)) or 0
+        gtin_products = self.session.scalar(select(sa_func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
+        conflicts = 0
+        for product_id in self.session.scalars(select(KbProduct.id)).all():
+            knowledge = self.current_knowledge(product_id)
+            conflicts += sum(1 for entry in knowledge.values() if entry.get("conflict"))
+        fingerprint_products = products - gtin_products
+        return {"products": products, "gtin_products": gtin_products, "fingerprint_products": fingerprint_products, "conflicts": conflicts}

central/app/ai/llm_provider.py

@@ -0,0 +1,158 @@
+"""Real LLM provider (OpenAI-compatible) for the NCM semantic evaluation.
+
+Contract (mission REAL LLM EVALUATION V1):
+- The API key lives ONLY in the Hub environment (AI_API_KEY). Never committed, logged,
+  written to artifacts, sent to the Agent or Firebird.
+- The gold answer is never part of the prompt: items carry only description/brand/unit/
+  segment-weak-context + the official top-K hybrid candidates with hierarchical context.
+- Strict structured output: per item {work_item_id, selected_ncm, confidence, abstain,
+  reason_short}; selected_ncm must be in the candidate list or null (abstain).
+- Batching is safe only when the response is valid JSON with 1:1 work_item_id matching,
+  no omitted items, no cross-product mixing. Verified progressively (5/10/20).
+"""
+import hashlib
+import json
+import re
+from dataclasses import dataclass
+
+PROMPT_VERSION_LLM = "llm-ncm-eval-v1"
+
+_SYSTEM_PROMPT = (
+    "You are a Brazilian customs classification assistant. For each product below, choose the "
+    "most likely official NCM code from the provided candidate list, or abstain. "
+    "Rules: selected_ncm MUST be one of the candidate codes for that exact item, or null with "
+    "abstain=true when evidence is insufficient. Never invent a code. confidence is your calibrated "
+    "probability in [0,1]. Reply with ONLY a JSON array, one object per item, in the same order, "
+    'each: {"work_item_id": str, "selected_ncm": str|null, "confidence": float, "abstain": bool, "reason_short": str}.'
+)
+
+
+@dataclass
+class LlmEvalItem:
+    work_item_id: str
+    description: str
+    normalized_description: str
+    brand: str | None
+    unit: str | None
+    segment: str
+    candidates: list  # [(code, official_description, hybrid_score, official_score, kb_score, tenant_count)]
+
+
+class LlmAnswerError(ValueError):
+    pass
+
+
+def item_prompt_payload(item: LlmEvalItem) -> dict:
+    return {
+        "work_item_id": item.work_item_id,
+        "description": item.description,
+        "normalized_description": item.normalized_description,
+        "brand": item.brand,
+        "unit": item.unit,
+        "segment_weak_context": item.segment,
+        "official_candidates": [
+            {"ncm": code, "official_description": text}
+            for code, text, *_ in item.candidates
+        ],
+    }
+
+
+def candidate_set_hash(item: LlmEvalItem) -> str:
+    codes = [code for code, *_ in item.candidates]
+    return hashlib.sha256("|".join(codes).encode()).hexdigest()
+
+
+def input_fingerprint(item: LlmEvalItem) -> str:
+    payload = {**item_prompt_payload(item), "system": _SYSTEM_PROMPT}
+    return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
+
+
+def validate_answer(answer: dict, item: LlmEvalItem) -> dict:
+    """Fail-closed validator: schema, work_item_id, candidate-list confinement."""
+    if not isinstance(answer, dict):
+        raise LlmAnswerError("answer must be an object")
+    if str(answer.get("work_item_id")) != str(item.work_item_id):
+        raise LlmAnswerError(f"work_item_id mismatch: {answer.get('work_item_id')!r} != {item.work_item_id!r}")
+    abstain = bool(answer.get("abstain"))
+    selected = answer.get("selected_ncm")
+    selected = None if selected in (None, "") else str(selected)
+    try:
+        confidence = float(answer.get("confidence"))
+    except (TypeError, ValueError):
+        raise LlmAnswerError("confidence must be numeric")
+    if not 0.0 <= confidence <= 1.0:
+        raise LlmAnswerError("confidence out of [0,1]")
+    codes = {code for code, *_ in item.candidates}
+    if abstain:
+        if selected is not None:
+            raise LlmAnswerError("abstain must carry selected_ncm=null")
+    else:
+        if selected is None:
+            raise LlmAnswerError("non-abstain requires selected_ncm")
+        if selected not in codes:
+            raise LlmAnswerError(f"selected_ncm {selected!r} is outside the candidate list")
+    return {"work_item_id": item.work_item_id, "selected_ncm": selected, "confidence": confidence, "abstain": abstain, "reason_short": str(answer.get("reason_short") or "")[:200]}
+
+
+class OpenAICompatibleProvider:
+    """Talks to any OpenAI-compatible /chat/completions endpoint. `transport` is
+    injectable for tests; production default uses httpx with the key from settings only."""
+
+    name = "openai-compatible"
+    prompt_version = PROMPT_VERSION_LLM
+
+    def __init__(self, api_key: str, model: str, base_url: str = "https://api.openai.com/v1", transport=None, timeout: float = 120.0):
+        self.api_key = api_key
+        self.model = model
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self._transport = transport or self._httpx_transport
+        self.http_calls = 0
+        self.retries = 0
+        self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
+
+    def _httpx_transport(self, payload: dict) -> dict:
+        import httpx
+        response = httpx.post(f"{self.base_url}/chat/completions", json=payload, headers={"Authorization": f"Bearer {self.api_key}"}, timeout=self.timeout)
+        response.raise_for_status()
+        return response.json()
+
+    def list_models(self) -> list:
+        """GET /models with the account key — validates auth/base URL/model availability
+        without spending completion tokens. Never raises with the key attached."""
+        import httpx
+        response = httpx.get(f"{self.base_url}/models", headers={"Authorization": f"Bearer {self.api_key}"}, timeout=self.timeout)
+        response.raise_for_status()
+        data = response.json()
+        return sorted(str(item.get("id")) for item in data.get("data", []) if item.get("id"))
+
+    def _request(self, messages: list, max_retries: int = 2) -> dict:
+        payload = {"model": self.model, "messages": messages, "temperature": 0}
+        last_error = None
+        for attempt in range(max_retries + 1):
+            self.http_calls += 1
+            if attempt:
+                self.retries += 1
+            try:
+                data = self._transport(payload)
+                usage = data.get("usage") or {}
+                self.usage["input_tokens"] += int(usage.get("prompt_tokens") or 0)
+                self.usage["output_tokens"] += int(usage.get("completion_tokens") or 0)
+                cached = usage.get("prompt_tokens_details") or {}
+                self.usage["cached_tokens"] += int(cached.get("cached_tokens") or 0)
+                return data
+            except Exception as error:  # transport-level failure only; JSON contract errors are not retried
+                last_error = error
+        raise RuntimeError(f"LLM transport failed after {max_retries + 1} attempts: {last_error}")
+
+    def select_batch(self, items: list[LlmEvalItem]) -> list[dict]:
+        """One HTTP call for the whole batch. Raises LlmAnswerError on any 1:1/schema violation
+        so the harness can shrink the batch size."""
+        user_content = json.dumps([item_prompt_payload(item) for item in items], ensure_ascii=False)
+        data = self._request([{"role": "system", "content": _SYSTEM_PROMPT}, {"role": "user", "content": user_content}])
+        content = data["choices"][0]["message"]["content"]
+        content = re.sub(r"^```(?:json)?|```$", "", content.strip(), flags=re.MULTILINE).strip()
+        answers = json.loads(content)
+        if not isinstance(answers, list) or len(answers) != len(items):
+            raise LlmAnswerError(f"expected {len(items)} answers, got {len(answers) if isinstance(answers, list) else type(answers).__name__}")
+        return [validate_answer(answer, item) for answer, item in zip(answers, items)]

central/app/ai/provider.py

@@ -0,0 +1,139 @@
+"""Provider abstraction with strict structured output and a hard budget guard.
+
+The fiscal engine is never coupled to a vendor: it talks to AISemanticProvider.
+V1 ships BaselineProvider (deterministic, zero-cost) used by the evaluation
+harness and as the no-credential fallback; real providers subclass the same
+contract and are only invoked with AI_ENABLED=true and credentials in the Hub.
+"""
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+from ..classification import ENGINE_VERSION
+from .fingerprint import overlap_score
+
+PROMPT_VERSION = "ai-semantic-v1"
+
+
+class SemanticOutputError(ValueError):
+    """Raised when a provider answer violates the strict structured-output schema."""
+
+
+@dataclass
+class StructuredResult:
+    task_type: str
+    input_fingerprint: str
+    selected_candidate: str            # candidate code, or "NONE"
+    confidence: float
+    reason: str
+    candidate_ids_seen: list[str]
+    model: str
+    prompt_version: str
+    created_at: str
+    dataset_versions: dict = field(default_factory=dict)
+    engine_version: str = ENGINE_VERSION
+
+    def to_dict(self):
+        return {
+            "task_type": self.task_type,
+            "input_fingerprint": self.input_fingerprint,
+            "selected_candidate": self.selected_candidate,
+            "confidence": self.confidence,
+            "reason": self.reason,
+            "candidate_ids_seen": self.candidate_ids_seen,
+            "model": self.model,
+            "prompt_version": self.prompt_version,
+            "created_at": self.created_at,
+            "dataset_versions": self.dataset_versions,
+            "engine_version": self.engine_version,
+        }
+
+
+def result_from_dict(data) -> StructuredResult:
+    """Fail-closed schema check: the selection must be NONE or one of the seen candidates."""
+    result = StructuredResult(
+        task_type=str(data["task_type"]),
+        input_fingerprint=str(data["input_fingerprint"]),
+        selected_candidate=str(data["selected_candidate"]),
+        confidence=float(data["confidence"]),
+        reason=str(data["reason"]),
+        candidate_ids_seen=[str(item) for item in data["candidate_ids_seen"]],
+        model=str(data["model"]),
+        prompt_version=str(data["prompt_version"]),
+        created_at=str(data["created_at"]),
+        dataset_versions=dict(data.get("dataset_versions") or {}),
+    )
+    if result.selected_candidate != "NONE" and result.selected_candidate not in result.candidate_ids_seen:
+        raise SemanticOutputError(f"selected_candidate {result.selected_candidate!r} is not among the official candidates")
+    if not 0.0 <= result.confidence <= 1.0:
+        raise SemanticOutputError("confidence must be within [0,1]")
+    return result
+
+
+class AISemanticProvider:
+    """Contract every provider must honour. `select` receives ONLY official candidates."""
+    name = "abstract"
+    model = "abstract"
+
+    def select(self, work_item, candidates) -> StructuredResult:
+        raise NotImplementedError
+
+    def estimate(self, work_items) -> dict:
+        raise NotImplementedError
+
+
+def _score(text: str, official: str) -> float:
+    return overlap_score(text, official)
+
+
+class BaselineProvider(AISemanticProvider):
+    """Deterministic zero-cost baseline: highest token-overlap candidate, or NONE.
+    Used for the evaluation pipeline and as the mandatory no-credential fallback."""
+
+    name = "baseline"
+    model = "baseline-deterministic-v1"
+
+    def select(self, work_item, candidates, dataset_versions=None) -> StructuredResult:
+        description = work_item.payload.get("description", "") if hasattr(work_item, "payload") else ""
+        scored = [(code, official, _score(description, official)) for code, official in [(str(c[0]), c[1]) for c in candidates]]
+        best = max(scored, key=lambda row: row[2]) if scored else None
+        selected, _, score = best if best else ("NONE", "", 0.0)
+        if not scored or score <= 0.0:
+            selected = "NONE"
+        return StructuredResult(
+            task_type=work_item.task_type,
+            input_fingerprint=work_item.dedup_key,
+            selected_candidate=selected,
+            confidence=round(score, 4),
+            reason="baseline token-overlap top-1" if selected != "NONE" else "no token overlap with any official candidate",
+            candidate_ids_seen=[str(c[0]) for c in candidates],
+            model=self.model,
+            prompt_version=PROMPT_VERSION,
+            created_at=datetime.now(timezone.utc).isoformat(),
+            dataset_versions=dict(dataset_versions or {}),
+        )
+
+    def estimate(self, work_items) -> dict:
+        return {"input_tokens": 0, "output_tokens": 0, "estimated_cost": 0.0}
+
+
+class BudgetGuard:
+    """Refuses execution when the estimated cost exceeds AI_MAX_RUN_COST_USD.
+    Never falls back to a more expensive model. Prices are operator-configured — when
+    absent, cost is UNKNOWN: the residual run is refused, but an explicitly limited
+    benchmark (e.g. the 65-item GOLD eval) may opt in with allow_unknown_cost."""
+
+    def __init__(self, max_run_cost_usd: float, price_in_per_m: float | None, price_out_per_m: float | None):
+        self.max_run_cost_usd = float(max_run_cost_usd)
+        self.price_in_per_m = None if price_in_per_m is None else float(price_in_per_m)
+        self.price_out_per_m = None if price_out_per_m is None else float(price_out_per_m)
+
+    def estimate(self, items_in_tokens: int, items_out_tokens: int) -> dict:
+        if self.price_in_per_m is None or self.price_out_per_m is None:
+            return {"input_tokens": items_in_tokens, "output_tokens": items_out_tokens, "estimated_cost": None, "cost_unknown": True}
+        cost = (items_in_tokens / 1_000_000) * self.price_in_per_m + (items_out_tokens / 1_000_000) * self.price_out_per_m
+        return {"input_tokens": items_in_tokens, "output_tokens": items_out_tokens, "estimated_cost": round(cost, 6)}
+
+    def authorize(self, plan: dict, allow_unknown_cost: bool = False) -> bool:
+        if plan.get("estimated_cost") is None:
+            return allow_unknown_cost
+        return plan["estimated_cost"] <= self.max_run_cost_usd

central/app/ai/retrieval.py

@@ -0,0 +1,245 @@
+"""Semantic candidate retrieval V2 — deterministic, zero LLM.
+
+Three retrieval sources merged into one ranking:
+  A. OFFICIAL: hierarchical search over the official NCM tree. Every 8-digit code is
+     enriched with its ancestors' descriptions (chapter/position/subposition), so the
+     very short legal text carries the vocabulary of its path (section 6).
+  B. KB NEIGHBORS: the Knowledge Base acts as a commercial->official bridge — products
+     whose commercial description resembles the query contribute their VALIDATED NCMs,
+     weighted by similarity, tenant_count, observation_count and agreement (sections 2-3).
+  C. HYBRID: single final ranking; the KB is only evidence for CANDIDATES, never a
+     majority-vote fiscal decision.
+
+Abbreviation lexicon (section 8) is derived statistically from the KB itself; commercial
+normalization (section 7) strips units/sizes/colors/refs deterministically.
+"""
+from dataclasses import dataclass
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from ..models import Dataset, KbKnowledge, KbObservation, KbProduct, NcmEntry, Tenant
+from .fingerprint import normalize_description, token_stems
+
+# ---------------- commercial normalization (section 7) ----------------
+
+_UNITS = {"UN", "UNID", "UNIDADE", "CX", "CAIXA", "FD", "FDO", "FARDO", "PCT", "PACOTE", "PKG",
+          "KG", "G", "GR", "GRAMA", "MG", "ML", "L", "LT", "LITRO", "LATA", "GARRAFA", "PET",
+          "PC", "PECA", "DZ", "DUZIA", "PAR", "KIT", "JG", "PARA", "COM", "SEM", "C", "S"}
+_DISCARD = _UNITS | {"MASC", "MASCULINO", "FEM", "FEMININO", "INF", "INFANTIL", "TAM", "TAMANHO",
+                     "COR", "REF", "COD", "CODIGO", "MOD", "MODELO", "NOVA", "NOVO", "TIPO"}
+_LEXEME_STRIP_PREFIXES = ("1", "2", "3", "4", "5", "6", "7", "8", "9", "0", ".")
+
+
+def _is_size_or_code(token: str) -> bool:
+    if token.isdigit():
+        return True
+    digits = "".join(char for char in token if char.isdigit())
+    letters = "".join(char for char in token if char.isalpha())
+    return bool(digits) and letters in _UNITS  # "1KG", "500ML", "2LT"
+
+
+def commercial_tokens(description) -> set:
+    """Stable commercial lexemes: units/sizes/colors/refs removed, stems kept."""
+    tokens = []
+    skip_next = False
+    for token in normalize_description(description).split():
+        if skip_next:
+            skip_next = False
+            continue
+        if token in ("TAM", "COR", "REF", "COD", "MOD"):
+            skip_next = True
+            continue
+        if token in _DISCARD or _is_size_or_code(token):
+            continue
+        tokens.append(token)
+    return {_stem_safe(token) for token in tokens}
+
+
+def _stem_safe(token: str) -> str:
+    from .fingerprint import _stem
+    return _stem(token)
+
+
+def expand_stems(stems: set, lexicon: dict) -> set:
+    expanded = set(stems)
+    for stem in stems:
+        target = lexicon.get(stem)
+        if target:
+            expanded.add(target)
+    return expanded
+
+
+def derive_abbreviation_lexicon(corpus: dict, min_support: int = 3) -> dict:
+    """corpus: {commercial_description: ncm}. Statistical abbreviations only: a short token
+    (3-5 chars) that is a strict prefix of a longer token, where BOTH forms occur across
+    distinct products that share the same NCM. Support = (#products with the short form) +
+    (#products with the long form); accepted when support >= min_support and both >= 1."""
+    by_ncm: dict = {}
+    for description, ncm in corpus.items():
+        tokens = set(normalize_description(description).split())
+        by_ncm.setdefault(ncm, []).append(tokens)
+    best: dict = {}
+    for descriptions in by_ncm.values():
+        if len(descriptions) < 2:
+            continue
+        all_tokens = set().union(*descriptions)
+        for short in all_tokens:
+            if not (3 <= len(short) <= 5):
+                continue
+            short_count = sum(1 for tokens in descriptions if short in tokens)
+            if short_count < 1:
+                continue
+            for long_token in all_tokens:
+                if len(long_token) > len(short) and long_token.startswith(short):
+                    long_count = sum(1 for tokens in descriptions if long_token in tokens)
+                    support = short_count + long_count
+                    if support >= min_support and long_count >= 1 and short_count >= 1:
+                        current = best.get(short)
+                        if current is None or support > current[1]:
+                            best[short] = (long_token, support)
+    return {short: target for short, (target, _) in best.items()}
+
+
+# ---------------- enriched official corpus (sections 5-6) ----------------
+
+
+@dataclass
+class EnrichedNcmCorpus:
+    documents: dict            # code -> enriched stems set
+    descriptions: dict         # code -> official description
+    positions: dict            # 4-digit position -> enriched stems (chapter + position)
+    children: dict             # 4-digit position -> [8-digit codes]
+
+    @staticmethod
+    def build(session: Session) -> "EnrichedNcmCorpus":
+        dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+        if dataset is None:
+            return EnrichedNcmCorpus({}, {}, {}, {})
+        entries = session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all()
+        by_code = {entry.code: entry.description for entry in entries}
+        documents, positions, children = {}, {}, {}
+        for code, description in by_code.items():
+            if len(code) != 8:
+                continue
+            ancestors = [text for text in (by_code.get(code[:2]), by_code.get(code[:4]), by_code.get(code[:6])) if text]
+            documents[code] = token_stems(description) | set().union(*[token_stems(text) for text in ancestors]) if ancestors else token_stems(description)
+            positions.setdefault(code[:4], token_stems(by_code.get(code[:4], "")) | token_stems(by_code.get(code[:2], "")))
+            children.setdefault(code[:4], []).append(code)
+        return EnrichedNcmCorpus(documents, by_code, positions, children)
+
+    def document(self, code: str) -> str:
+        return " ".join(sorted(self.documents.get(code, set())))
+
+    def items(self):
+        return list(self.documents.items())
+
+
+def _score(stems: set, document_stems: set, idf: dict | None = None) -> float:
+    if not stems:
+        return 0.0
+    if idf:
+        matched = sum(idf.get(stem, 1.0) for stem in stems if stem in document_stems)
+        total = sum(idf.get(stem, 1.0) for stem in stems)
+        return matched / total
+    return len(stems & document_stems) / len(stems)
+
+
+def hierarchical_candidates(corpus: EnrichedNcmCorpus, description, k: int = 20, max_positions: int = 15):
+    """Two-stage: rank 4-digit positions first, then their 8-digit descendants (section 5)."""
+    stems = commercial_tokens(description)
+    if not stems:
+        return []
+    position_scores = sorted(((_score(stems, doc), position) for position, doc in corpus.positions.items()), reverse=True)
+    ranked = []
+    for position_score, position in position_scores[:max_positions]:
+        if position_score <= 0:
+            continue
+        for code in corpus.children.get(position, []):
+            code_score = _score(stems, corpus.documents[code])
+            if code_score > 0:
+                ranked.append((code, corpus.descriptions.get(code, ""), round(code_score, 4)))
+    ranked.sort(key=lambda row: (-row[2], row[0]))
+    return ranked[:k]
+
+
+# ---------------- KB neighbor bridge (sections 2-3) ----------------
+
+
+@dataclass
+class KbNeighborIndex:
+    entries: list  # (product_id, stems, ncm, tenant_ids:set, observation_count, has_conflict)
+
+    @staticmethod
+    def build(session: Session) -> "KbNeighborIndex":
+        validated = {}
+        for knowledge in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "validated_ncm")).all():
+            validated.setdefault(knowledge.kb_product_id, set()).add(knowledge.value)
+        conflicts = {product_id: len(values) > 1 for product_id, values in validated.items()}
+        products = {product.id: product for product in session.scalars(select(KbProduct)).all()}
+        observations = session.scalars(select(KbObservation)).all()
+        per_product_tenants: dict = {}
+        for observation in observations:
+            per_product_tenants.setdefault(observation.kb_product_id, set()).add(observation.tenant_id)
+        entries = []
+        for product_id, values in validated.items():
+            if conflicts.get(product_id):
+                continue  # conflicting knowledge never feeds neighbors
+            for value in values:
+                product = products.get(product_id)
+                if product is None or not product.normalized_description:
+                    continue
+                tenants = per_product_tenants.get(product_id, set())
+                entries.append((product_id, commercial_tokens(product.normalized_description), value, tenants, len(tenants), False))
+        return KbNeighborIndex(entries)
+
+    def corpus(self) -> dict:
+        return {" ".join(sorted(stems)): ncm for _, stems, ncm, _, _, _ in self.entries}
+
+
+def kb_neighbor_candidates(index: KbNeighborIndex, description, k: int = 10, exclude_product_id: int | None = None):
+    """Aggregate NCMs of lexically similar known products. `exclude_product_id` implements
+    leave-one-out evaluation: the queried product never retrieves itself. Returns
+    (ncm, neighbor_score, observation_count, tenant_count) — agreement is the union of
+    distinct tenants behind the contributing neighbors, reported as evidence, never used
+    as an automatic majority decision."""
+    stems = commercial_tokens(description)
+    if not stems:
+        return []
+    best: dict = {}
+    for product_id, neighbor_stems, ncm, tenant_ids, observation_count, _ in index.entries:
+        if exclude_product_id is not None and product_id == exclude_product_id:
+            continue
+        similarity = _score(stems, neighbor_stems)
+        if similarity <= 0:
+            continue
+        current = best.get(ncm)
+        if current is None:
+            best[ncm] = [ncm, round(similarity, 4), observation_count, set(tenant_ids)]
+        else:
+            current[1] = max(current[1], round(similarity, 4))
+            current[3] |= set(tenant_ids)
+    ranked = sorted(((row[0], row[1], row[2], len(row[3])) for row in best.values()), key=lambda row: (-row[1], -row[3], row[0]))
+    return ranked[:k]
+
+
+# ---------------- hybrid ranking (section 4) ----------------
+
+OFFICIAL_WEIGHT, KB_WEIGHT = 0.3, 0.7  # tuned on the V2 benchmark (KB bridge carries the signal;
+# the official component breaks ties and covers products absent from the KB)
+
+
+def hybrid_candidates(corpus: EnrichedNcmCorpus, index: KbNeighborIndex, description, k: int = 20, lexicon: dict | None = None, exclude_product_id: int | None = None):
+    stems = expand_stems(commercial_tokens(description), lexicon or {})
+    official = hierarchical_candidates(corpus, " ".join(sorted(stems)) or description, k=k)
+    neighbors = kb_neighbor_candidates(index, description, k=k, exclude_product_id=exclude_product_id)
+    combined: dict = {}
+    for code, text, score in official:
+        combined[code] = [code, text, OFFICIAL_WEIGHT * score, score, 0.0, 0]
+    for code, similarity, observation_count, tenant_count in neighbors:
+        row = combined.setdefault(code, [code, "", 0.0, 0.0, 0.0, 0])
+        row[2] += KB_WEIGHT * similarity
+        row[4] = similarity
+        row[5] = tenant_count
+    ranked = sorted(combined.values(), key=lambda row: (-round(row[2], 4), row[0]))
+    return [(row[0], row[1], round(row[2], 4), row[3], row[4], row[5]) for row in ranked[:k]]

central/app/ai/workitems.py

@@ -0,0 +1,64 @@
+"""Residual-question dedup: group pending cells by the KNOWLEDGE needed, not per cell.
+
+- INVALID_NCM  -> one work item per product fingerprint (same product in N databases collapses)
+- CEST/EX_TIPI -> one work item per (doubt, NCM): all databases sharing that NCM collapse
+- PRODUCT_ROLE -> tenant-specific by contract: one work item per (company, group)
+- Everything else (origin, ST retention, company profile, duplicate key) never becomes
+  an AI work item.
+"""
+from dataclasses import dataclass, field
+
+_INVALID_NCM, _ROLE = "ncm", "product_role"
+
+
+@dataclass
+class WorkItem:
+    task_type: str
+    dedup_key: str
+    question_refs: list = field(default_factory=list)
+    cells_covered: int = 0
+    payload: dict = field(default_factory=dict)
+    kb_product_id: int | None = None
+
+
+def build_work_items(questions, product_fingerprints=None) -> list[WorkItem]:
+    """`product_fingerprints` maps "{DATABASE}:{erp_code}" -> KB fingerprint (from the seed pass)."""
+    product_fingerprints = product_fingerprints or {}
+    items: dict = {}
+
+    def _get(task_type, dedup_key, ref, cells):
+        item = items.get(dedup_key)
+        if item is None:
+            item = WorkItem(task_type=task_type, dedup_key=dedup_key)
+            items[dedup_key] = item
+        item.question_refs.append(ref)
+        item.cells_covered += cells
+        return item
+
+    for question in questions:
+        key = question.get("KEY")
+        scope = question.get("SCOPE")
+        cells = int(question.get("CELLS") or question.get("PRODUCTS_AFFECTED") or 0)
+        ref = question.get("QUESTION_ID") or f"{question.get('DATABASE')}:{scope}:{question.get('NCM') or question.get('GROUP')}:{key}"
+        if key in ("cest", "ex_tipi") and scope == "NCM":
+            ncm = str(question.get("NCM") or "").strip()
+            if ncm:
+                item = _get(key.upper(), f"{key}:{ncm}", ref, cells)
+                item.payload.setdefault("ncm", ncm)
+                item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
+        elif key == _INVALID_NCM and scope == "PRODUCT":
+            database = question.get("DATABASE", "")
+            erp_code = str(question.get("DATABASE_CODE") or "").strip()
+            fingerprint = product_fingerprints.get(f"{database}:{erp_code}")
+            dedup = f"invalid_ncm:{fingerprint}" if fingerprint else f"invalid_ncm:{database}:{erp_code}"
+            item = _get("INVALID_NCM", dedup, ref, cells)
+            item.payload.setdefault("erp_code", erp_code)
+            item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
+        elif key == _ROLE and scope == "GROUP":
+            company = question.get("COMPANY", "")
+            group = str(question.get("GROUP") or "").strip()
+            if group:
+                item = _get("PRODUCT_ROLE", f"product_role:{company}:{group}", ref, cells)
+                item.payload.setdefault("company_key", company)
+                item.payload["group"] = group
+    return list(items.values())

central/app/config.py

@@ -31,6 +31,20 @@ class Settings:
     cest_dataset_url: str = os.getenv("CEST_DATASET_URL", "https://www.confaz.fazenda.gov.br/legislacao/convenios/2018/CV142_18")
     dataset_storage_path: str = os.getenv("DATASET_STORAGE_PATH", "./data/datasets")
     http_timeout_seconds: float = float(os.getenv("HTTP_TIMEOUT_SECONDS", "30"))
+    # ---- AI semantic layer V1 ----
+    # Credentials live ONLY in the Hub environment; the Windows Agent never sees them.
+    ai_enabled: bool = os.getenv("AI_ENABLED", "0") == "1"
+    ai_provider: str = os.getenv("AI_PROVIDER", "baseline")     # baseline | openai | ... (AISemanticProvider)
+    ai_model: str = os.getenv("AI_MODEL", "baseline-deterministic-v1")
+    ai_max_batch_size: int = int(os.getenv("AI_MAX_BATCH_SIZE", "20"))
+    ai_max_run_cost_usd: float = float(os.getenv("AI_MAX_RUN_COST_USD", "5.0"))
+    ai_min_confidence: float = float(os.getenv("AI_MIN_CONFIDENCE", "0.6"))
+    ai_price_in_per_m: float | None = float(os.getenv("AI_PRICE_IN_PER_M")) if os.getenv("AI_PRICE_IN_PER_M") else None
+    ai_price_out_per_m: float | None = float(os.getenv("AI_PRICE_OUT_PER_M")) if os.getenv("AI_PRICE_OUT_PER_M") else None
+    # Real-LLM evaluation (OpenAI-compatible). The key lives ONLY in this environment.
+    # Z.ai general endpoint (NOT the /api/coding/paas/v4 coding-plan endpoint).
+    ai_api_key: str = os.getenv("AI_API_KEY", "")
+    ai_base_url: str = os.getenv("AI_BASE_URL", "https://api.z.ai/api/paas/v4")
 
 
 settings = Settings()

central/app/context_keys.py

@@ -7,7 +7,9 @@ ENUMS = {
     "st_status": {"SUBJECT", "NOT_SUBJECT"},
     "st_withheld": {"Y", "N"},
     "icms_treatment": {"TAXED", "EXEMPT", "NOT_TAXED", "ST_WITHHELD", "SUSPENDED", "DEFERRED"},
-    "entry_scope": {"INTRA", "INTER"},
+    # MIXED_OR_DYNAMIC: purchases are not predominantly intra- or interstate, so no single
+    # static entry CFOP is fiscally correct for all operations (section 0, AI semantic layer V1).
+    "entry_scope": {"INTRA", "INTER", "MIXED_OR_DYNAMIC"},
     "rounding": {"A", "T"},
     "piscofins": {"N", "S"},
     "incidence": {"NENHUMA"},

central/app/field_contract.py

@@ -362,20 +362,29 @@ def resolve_product(p, d, company, refs, ev, ctx, manual_ncm=None):
         out["CFOP_ENTRADA"] = blocked("CFOP_ENTRADA")
     else:
         exp_e = None
-        if scope_ctx in ("INTRA", "INTER") and role in ("RESALE", "OWN_PRODUCTION", "INPUT") and st != "UNKNOWN" and not (st == "SUBJECT" and withheld is None):
-            first = "1" if scope_ctx == "INTRA" else "2"
-            stw = st == "SUBJECT" and withheld == "Y"
-            exp_e = first + ("403" if stw else "102") if role == "RESALE" else first + ("401" if stw else "101")
-        valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
-        if valid_e and exp_e and cfe != exp_e and cfe in ENTRY_CFOP:
-            out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "PROVEN_ROLE_ST_AND_SCOPE_CONTRADICT_DECLARED_CFOP_ENTRADA")
-        elif valid_e:
-            out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_NOT_CONTRADICTED")
-        elif exp_e:
-            out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "DERIVED_FROM_ROLE_ST_AND_ENTRY_SCOPE")
+        if scope_ctx == "MIXED_OR_DYNAMIC":
+            # Scope answered, but no single static entry CFOP is fiscally correct for every
+            # operation: the ERP default / each operation decides 1xxx vs 2xxx. Never invent one.
+            valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
+            if valid_e:
+                out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_OPERATION_DEPENDENT_MIXED_SCOPE")
+            else:
+                out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", MANUAL, cfe, reason="CFOP_ENTRADA_OPERATION_DEPENDENT_MIXED_SCOPE")
         else:
-            key = "product_role" if role == "UNKNOWN" else ("entry_scope" if scope_ctx not in ("INTRA", "INTER") else ("st_status" if st == "UNKNOWN" else "st_withheld"))
-            out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", MANUAL, cfe, reason="CFOP_ENTRADA_NEEDS_" + key.upper(), key=key, scope="COMPANY" if key == "entry_scope" else "GROUP")
+            if scope_ctx in ("INTRA", "INTER") and role in ("RESALE", "OWN_PRODUCTION", "INPUT") and st != "UNKNOWN" and not (st == "SUBJECT" and withheld is None):
+                first = "1" if scope_ctx == "INTRA" else "2"
+                stw = st == "SUBJECT" and withheld == "Y"
+                exp_e = first + ("403" if stw else "102") if role == "RESALE" else first + ("401" if stw else "101")
+            valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
+            if valid_e and exp_e and cfe != exp_e and cfe in ENTRY_CFOP:
+                out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "PROVEN_ROLE_ST_AND_SCOPE_CONTRADICT_DECLARED_CFOP_ENTRADA")
+            elif valid_e:
+                out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_NOT_CONTRADICTED")
+            elif exp_e:
+                out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "DERIVED_FROM_ROLE_ST_AND_ENTRY_SCOPE")
+            else:
+                key = "product_role" if role == "UNKNOWN" else ("entry_scope" if scope_ctx not in ("INTRA", "INTER") else ("st_status" if st == "UNKNOWN" else "st_withheld"))
+                out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", MANUAL, cfe, reason="CFOP_ENTRADA_NEEDS_" + key.upper(), key=key, scope="COMPANY" if key == "entry_scope" else "GROUP")
 
     # ---------------- origin (needed by CST) ----------------
     cod_o = cur["COD_ORIGEM_MERCADORIA"]

central/app/models.py

@@ -339,3 +339,128 @@ class FiscalContextEntry(Base):
     reason: Mapped[str | None] = mapped_column(Text, nullable=True)
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
     updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+# ================= AI semantic layer V1 =================
+# The AI layer is NOT a fiscal authority: it only selects among official candidates and
+# never writes to Firebird. Knowledge is versioned, provenance-tracked and never deleted.
+
+
+class KbProduct(Base):
+    """GLOBAL product knowledge: only facts inherent to the product (never tenant context)."""
+    __tablename__ = "kb_products"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True)
+    gtin: Mapped[str | None] = mapped_column(String(16), unique=True, nullable=True, index=True)
+    identity_kind: Mapped[str] = mapped_column(String(16), default="FINGERPRINT")  # GTIN | FINGERPRINT | SEMANTIC | UNKNOWN
+    normalized_description: Mapped[str] = mapped_column(Text, default="")
+    brand: Mapped[str | None] = mapped_column(Text, nullable=True)
+    unit: Mapped[str | None] = mapped_column(String(32), nullable=True)
+    package_quantity: Mapped[str | None] = mapped_column(String(32), nullable=True)
+    category: Mapped[str | None] = mapped_column(Text, nullable=True)
+    semantic_attributes: Mapped[dict] = mapped_column(JSON, default=dict)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+
+class KbKnowledge(Base):
+    """Versioned knowledge rows. Unique per (product, key, value): conflicting values for the
+    same key are kept side by side and surfaced as KNOWLEDGE_CONFLICT, never majority-picked."""
+    __tablename__ = "kb_knowledge"
+    __table_args__ = (UniqueConstraint("kb_product_id", "knowledge_key", "value", name="uq_kb_knowledge_value"),)
+    id: Mapped[int] = mapped_column(primary_key=True)
+    kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
+    knowledge_key: Mapped[str] = mapped_column(String(48), index=True)  # candidate_ncm | validated_ncm | candidate_cest | candidate_ex_tipi | ...
+    value: Mapped[str] = mapped_column(Text)
+    source: Mapped[str] = mapped_column(String(32))  # OFFICIAL_DATASET | ERP_OBSERVATION | OPERATOR | AI | AI_PLUS_OFFICIAL_VALIDATION
+    confidence: Mapped[float] = mapped_column(default=0.0)
+    evidence: Mapped[dict] = mapped_column(JSON, default=dict)
+    model: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    prompt_version: Mapped[str | None] = mapped_column(String(32), nullable=True)
+    dataset_versions: Mapped[dict] = mapped_column(JSON, default=dict)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    validated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+
+
+class KbObservation(Base):
+    """Raw per-tenant ERP observation of a KB product (evidence aggregation only)."""
+    __tablename__ = "kb_observations"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
+    tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
+    company_key: Mapped[str] = mapped_column(String(64), index=True)
+    erp_code: Mapped[str] = mapped_column(String(32))
+    database_label: Mapped[str] = mapped_column(String(64))
+    observed_fields: Mapped[dict] = mapped_column(JSON, default=dict)
+    source: Mapped[str] = mapped_column(String(32), default="ERP_OBSERVATION")
+    observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+class TenantProductContext(Base):
+    """TENANT-specific product context: role, ST-withheld, operation profile, operator decisions.
+    Never promoted to global knowledge."""
+    __tablename__ = "tenant_product_context"
+    __table_args__ = (UniqueConstraint("tenant_id", "company_key", "erp_code", name="uq_tenant_product_context"),)
+    id: Mapped[int] = mapped_column(primary_key=True)
+    tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
+    company_key: Mapped[str] = mapped_column(String(64), index=True)
+    erp_code: Mapped[str] = mapped_column(String(32))
+    kb_product_id: Mapped[int | None] = mapped_column(ForeignKey("kb_products.id"), nullable=True)
+    product_role: Mapped[str | None] = mapped_column(String(16), nullable=True)  # RESALE | OWN_PRODUCTION | INPUT | SERVICE | UNKNOWN
+    st_withheld: Mapped[str | None] = mapped_column(String(4), nullable=True)    # Y | N — never AI-decided
+    operation_profile: Mapped[dict] = mapped_column(JSON, default=dict)
+    source: Mapped[str] = mapped_column(String(32), default="ERP_OBSERVATION")
+    confidence: Mapped[float] = mapped_column(default=0.0)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+
+class AiWorkItem(Base):
+    """Deduplicated semantic work item (one knowledge need, many pending cells)."""
+    __tablename__ = "ai_work_items"
+    __table_args__ = (UniqueConstraint("dedup_key", name="uq_ai_work_item_dedup"),)
+    id: Mapped[int] = mapped_column(primary_key=True)
+    task_type: Mapped[str] = mapped_column(String(24), index=True)  # INVALID_NCM | CEST | EX_TIPI | PRODUCT_ROLE
+    dedup_key: Mapped[str] = mapped_column(String(160))
+    question_refs: Mapped[list] = mapped_column(JSON, default=list)
+    payload: Mapped[dict] = mapped_column(JSON, default=dict)
+    status: Mapped[str] = mapped_column(String(32), default="PENDING", index=True)  # PENDING | KB_EXACT_HIT | KB_FINGERPRINT_HIT | HIGH_CONFIDENCE_RETRIEVAL | SINGLE_OFFICIAL_CANDIDATE | LLM_SELECTION_REQUIRED | UNRESOLVED | UNRESOLVED_NO_OFFICIAL_MAPPING | CONFLICT | DONE | SKIPPED
+    kb_product_id: Mapped[int | None] = mapped_column(ForeignKey("kb_products.id"), nullable=True)
+    cells_covered: Mapped[int] = mapped_column(Integer, default=0)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+
+class AiRun(Base):
+    """One provider execution over a batch of work items, with budget-guard evidence."""
+    __tablename__ = "ai_runs"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    provider: Mapped[str] = mapped_column(String(64))
+    model: Mapped[str] = mapped_column(String(64))
+    prompt_version: Mapped[str] = mapped_column(String(32))
+    work_item_ids: Mapped[list] = mapped_column(JSON, default=list)
+    input_tokens: Mapped[int] = mapped_column(Integer, default=0)
+    output_tokens: Mapped[int] = mapped_column(Integer, default=0)
+    cost_usd: Mapped[float] = mapped_column(default=0.0)
+    status: Mapped[str] = mapped_column(String(16), default="RUN")
+    results: Mapped[list] = mapped_column(JSON, default=list)
+    budget_guard: Mapped[dict] = mapped_column(JSON, default=dict)
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+
+
+class AiEvalCache(Base):
+    """Persistent LLM result cache: identical (input_fingerprint, candidate_set_hash,
+    prompt_version, model, dataset_versions) re-execution makes ZERO new calls."""
+    __tablename__ = "ai_eval_cache"
+    __table_args__ = (UniqueConstraint("input_fingerprint", "candidate_set_hash", "prompt_version", "model", "dataset_version_ncm", name="uq_ai_eval_cache"),)
+    id: Mapped[int] = mapped_column(primary_key=True)
+    input_fingerprint: Mapped[str] = mapped_column(String(64), index=True)
+    candidate_set_hash: Mapped[str] = mapped_column(String(64))
+    prompt_version: Mapped[str] = mapped_column(String(32))
+    model: Mapped[str] = mapped_column(String(64))
+    dataset_version_ncm: Mapped[str] = mapped_column(String(64))
+    phase: Mapped[str] = mapped_column(String(16))
+    answer: Mapped[dict] = mapped_column(JSON)
+    gold_ncm: Mapped[str | None] = mapped_column(String(8), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())