chore: capture semantic pipeline baseline

.env.example

@@ -37,3 +37,11 @@ HTTP_TIMEOUT_SECONDS=30
 # COMPANY_CODE=            # C000004.CODIGO of the active company (when C000004 has several rows and EMISSOR cannot disambiguate)
 # COMPANY_CNPJ=            # alternative selector (digits only); a stable, verifiable identifier
 # COMPANY_CRT_SOURCE=      # EMISSOR | C000004 — required when the two CRT values disagree (emission programs read EMISSOR)
+
+# Z.ai (OpenAI-compatible). NEVER commit the real key. Key lives only in .env
+AI_API_KEY=
+AI_BASE_URL=https://api.z.ai/api/paas/v4
+AI_MODEL=glm-5.3
+# Optional prices (USD per 1M tokens) - only set when confirmed with the provider
+AI_PRICE_IN_PER_M=
+AI_PRICE_OUT_PER_M=

central/alembic/versions/0012_ai_semantic_layer.py

@@ -0,0 +1,37 @@
+"""Migration 0012: AI semantic layer V1 (KB, tenant context, work items, runs).
+
+Revision ID: 0012
+Revises: 0011
+"""
+from alembic import op
+from sqlalchemy import Column, DateTime, Float, ForeignKey, Integer, JSON, String, Text, UniqueConstraint, func
+
+revision = "0012"
+down_revision = "0011"
+branch_labels = depends_on = None
+
+
+def upgrade():
+    op.create_table("kb_products", Column("id", Integer, primary_key=True), Column("fingerprint", String(64), nullable=False, unique=True), Column("gtin", String(16), unique=True), Column("identity_kind", String(16), nullable=False, server_default="FINGERPRINT"), Column("normalized_description", Text, nullable=False, server_default=""), Column("brand", Text), Column("unit", String(32)), Column("package_quantity", String(32)), Column("category", Text), Column("semantic_attributes", JSON, nullable=False), Column("created_at", DateTime(timezone=True), server_default=func.now()), Column("updated_at", DateTime(timezone=True), server_default=func.now()))
+    op.create_index("ix_kb_products_gtin", "kb_products", ["gtin"])
+    op.create_table("kb_knowledge", Column("id", Integer, primary_key=True), Column("kb_product_id", Integer, ForeignKey("kb_products.id"), nullable=False), Column("knowledge_key", String(48), nullable=False), Column("value", Text, nullable=False), Column("source", String(32), nullable=False), Column("confidence", Float, nullable=False, server_default="0"), Column("evidence", JSON, nullable=False), Column("model", String(64)), Column("prompt_version", String(32)), Column("dataset_versions", JSON, nullable=False), Column("created_at", DateTime(timezone=True), server_default=func.now()), Column("validated_at", DateTime(timezone=True)), UniqueConstraint("kb_product_id", "knowledge_key", "value", name="uq_kb_knowledge_value"))
+    op.create_index("ix_kb_knowledge_kb_product_id", "kb_knowledge", ["kb_product_id"])
+    op.create_index("ix_kb_knowledge_knowledge_key", "kb_knowledge", ["knowledge_key"])
+    op.create_table("kb_observations", Column("id", Integer, primary_key=True), Column("kb_product_id", Integer, ForeignKey("kb_products.id"), nullable=False), Column("tenant_id", Integer, ForeignKey("tenants.id"), nullable=False), Column("company_key", String(64), nullable=False), Column("erp_code", String(32), nullable=False), Column("database_label", String(64), nullable=False), Column("observed_fields", JSON, nullable=False), Column("source", String(32), nullable=False, server_default="ERP_OBSERVATION"), Column("observed_at", DateTime(timezone=True), server_default=func.now()))
+    op.create_index("ix_kb_observations_kb_product_id", "kb_observations", ["kb_product_id"])
+    op.create_table("tenant_product_context", Column("id", Integer, primary_key=True), Column("tenant_id", Integer, ForeignKey("tenants.id"), nullable=False), Column("company_key", String(64), nullable=False), Column("erp_code", String(32), nullable=False), Column("kb_product_id", Integer, ForeignKey("kb_products.id")), Column("product_role", String(16)), Column("st_withheld", String(4)), Column("operation_profile", JSON, nullable=False), Column("source", String(32), nullable=False, server_default="ERP_OBSERVATION"), Column("confidence", Float, nullable=False, server_default="0"), Column("created_at", DateTime(timezone=True), server_default=func.now()), Column("updated_at", DateTime(timezone=True), server_default=func.now()), UniqueConstraint("tenant_id", "company_key", "erp_code", name="uq_tenant_product_context"))
+    op.create_index("ix_tenant_product_context_tenant_id", "tenant_product_context", ["tenant_id"])
+    op.create_table("ai_work_items", Column("id", Integer, primary_key=True), Column("task_type", String(24), nullable=False), Column("dedup_key", String(160), nullable=False, unique=True), Column("question_refs", JSON, nullable=False), Column("payload", JSON, nullable=False), Column("status", String(24), nullable=False, server_default="PENDING"), Column("kb_product_id", Integer, ForeignKey("kb_products.id")), Column("cells_covered", Integer, nullable=False, server_default="0"), Column("created_at", DateTime(timezone=True), server_default=func.now()), Column("updated_at", DateTime(timezone=True), server_default=func.now()))
+    op.create_index("ix_ai_work_items_task_type", "ai_work_items", ["task_type"])
+    op.create_index("ix_ai_work_items_status", "ai_work_items", ["status"])
+    op.create_table("ai_runs", Column("id", Integer, primary_key=True), Column("provider", String(64), nullable=False), Column("model", String(64), nullable=False), Column("prompt_version", String(32), nullable=False), Column("work_item_ids", JSON, nullable=False), Column("input_tokens", Integer, nullable=False, server_default="0"), Column("output_tokens", Integer, nullable=False, server_default="0"), Column("cost_usd", Float, nullable=False, server_default="0"), Column("status", String(16), nullable=False, server_default="RUN"), Column("results", JSON, nullable=False), Column("budget_guard", JSON, nullable=False), Column("started_at", DateTime(timezone=True), server_default=func.now()), Column("finished_at", DateTime(timezone=True)))
+
+
+def downgrade():
+    op.drop_table("ai_runs")
+    op.drop_table("ai_work_items")
+    op.drop_table("tenant_product_context")
+    op.drop_table("kb_observations")
+    op.drop_table("kb_knowledge")
+    op.drop_index("ix_kb_products_gtin", table_name="kb_products")
+    op.drop_table("kb_products")

central/alembic/versions/0013_widen_ai_work_item_status.py

@@ -0,0 +1,19 @@
+"""Migration 0013: widen ai_work_items.status for V2 classification labels.
+
+Revision ID: 0013
+Revises: 0012
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0013"
+down_revision = "0012"
+branch_labels = depends_on = None
+
+
+def upgrade():
+    op.alter_column("ai_work_items", "status", existing_type=sa.String(24), type_=sa.String(32), nullable=False, existing_server_default="PENDING")
+
+
+def downgrade():
+    op.alter_column("ai_work_items", "status", existing_type=sa.String(32), type_=sa.String(24), nullable=False, existing_server_default="PENDING")

central/alembic/versions/0014_ai_eval_cache.py

@@ -0,0 +1,20 @@
+"""Migration 0014: persistent LLM eval result cache.
+
+Revision ID: 0014
+Revises: 0013
+"""
+from alembic import op
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, JSON, String, UniqueConstraint, func
+
+revision = "0014"
+down_revision = "0013"
+branch_labels = depends_on = None
+
+
+def upgrade():
+    op.create_table("ai_eval_cache", Column("id", Integer, primary_key=True), Column("input_fingerprint", String(64), nullable=False), Column("candidate_set_hash", String(64), nullable=False), Column("prompt_version", String(32), nullable=False), Column("model", String(64), nullable=False), Column("dataset_version_ncm", String(64), nullable=False), Column("phase", String(16), nullable=False), Column("answer", JSON, nullable=False), Column("gold_ncm", String(8)), Column("created_at", DateTime(timezone=True), server_default=func.now()), UniqueConstraint("input_fingerprint", "candidate_set_hash", "prompt_version", "model", "dataset_version_ncm", name="uq_ai_eval_cache"))
+    op.create_index("ix_ai_eval_cache_input_fingerprint", "ai_eval_cache", ["input_fingerprint"])
+
+
+def downgrade():
+    op.drop_table("ai_eval_cache")

central/app/ai/__init__.py

@@ -0,0 +1,8 @@
+"""AI semantic layer V1 — NOT a fiscal authority.
+
+Rules frozen by the operator contract:
+- AI only selects among official candidates; it never invents codes.
+- Cache first: KB exact/fingerprint hits never trigger AI calls.
+- No AI for origin, ST retention, or company-level decisions.
+- AI_WRITE_ALLOWED=false in V1: results stay AI_CANDIDATE_VALIDATED.
+"""

central/app/ai/candidates.py

@@ -0,0 +1,53 @@
+"""Deterministic candidate generation from the ACTIVE official datasets.
+
+The AI never receives an open question: for every task type the Hub first builds
+the official candidate list (NCM top-K by description, CEST links for the NCM,
+TIPI EX rows for the NCM). Providers may only pick from this list.
+"""
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from ..models import CestEntry, CestNcmEntry, Dataset, NcmEntry, TipiEntry
+from .fingerprint import token_stems
+
+
+def _active_dataset(session: Session, kind: str):
+    return session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE"))
+
+
+def ncm_top_k(session: Session, description, k: int = 10):
+    """Top-K official NCM codes by (stemmed) token overlap with the product description."""
+    dataset = _active_dataset(session, "NCM")
+    if dataset is None:
+        return []
+    stems = token_stems(description)
+    if not stems:
+        return []
+    scored = []
+    for entry in session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all():
+        score = len(stems & token_stems(entry.description)) / len(stems)
+        if score > 0:
+            scored.append((entry.code, entry.description, round(score, 4)))
+    scored.sort(key=lambda row: (-row[2], row[0]))
+    return scored[:k]
+
+
+def cest_candidates_for_ncm(session: Session, ncm: str):
+    dataset = _active_dataset(session, "CEST")
+    if dataset is None:
+        return []
+    rows = session.execute(
+        select(CestEntry, CestNcmEntry.ncm_code)
+        .join(CestNcmEntry, CestNcmEntry.cest_entry_id == CestEntry.id)
+        .where(CestEntry.dataset_id == dataset.id)
+    ).all()
+    return list(dict.fromkeys(entry for entry, ncm_code in rows if ncm.startswith(ncm_code)))
+
+
+def ex_tipi_candidates_for_ncm(session: Session, ncm: str):
+    dataset = _active_dataset(session, "TIPI_IPI")
+    if dataset is None:
+        return []
+    return list(dict.fromkeys(
+        entry for entry in session.scalars(select(TipiEntry).where(TipiEntry.dataset_id == dataset.id, TipiEntry.ncm == ncm, TipiEntry.ex.is_not(None))).all()
+    ))

central/app/ai/eval_metrics.py

@@ -0,0 +1,53 @@
+"""Metrics and error attribution for the real LLM NCM evaluation (sections 8-9)."""
+
+
+def metrics(rows) -> dict:
+    """rows: [{gold_in_candidates, selected, gold, confidence, invalid}]"""
+    total = len(rows)
+    correct = sum(1 for row in rows if not row["invalid"] and row["selected"] == row["gold"])
+    wrong = sum(1 for row in rows if not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"])
+    abstain = sum(1 for row in rows if not row["invalid"] and row["selected"] is None)
+    invalid = sum(1 for row in rows if row["invalid"])
+    gold_in = sum(1 for row in rows if row["gold_in_candidates"])
+    answered = correct + wrong
+    high_conf = [row for row in rows if not row["invalid"] and row["confidence"] >= 0.95 and row["selected"] is not None]
+    high_conf_wrong = sum(1 for row in high_conf if row["selected"] != row["gold"])
+    return {
+        "TOTAL": total,
+        "CORRECT": correct,
+        "WRONG": wrong,
+        "ABSTAIN": abstain,
+        "INVALID": invalid,
+        "GOLD_IN_CANDIDATES": gold_in,
+        "ACCURACY_ALL": round(correct / total, 4) if total else None,
+        "SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE": round(correct / answered, 4) if answered else None,
+        "PRECISION_WHEN_HIGH_CONFIDENCE": round(sum(1 for row in high_conf if row["selected"] == row["gold"]) / len(high_conf), 4) if high_conf else None,
+        "HIGH_CONFIDENCE_WRONG_RATE": round(high_conf_wrong / len(high_conf), 4) if high_conf else None,
+        "ABSTENTION_RATE": round(abstain / total, 4) if total else None,
+    }
+
+
+def attribution(rows) -> dict:
+    return {
+        "GOLD_NOT_IN_CANDIDATES": sum(1 for row in rows if not row["gold_in_candidates"]),
+        "GOLD_IN_CANDIDATES_BUT_WRONG": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"]),
+        "GOLD_IN_CANDIDATES_BUT_ABSTAIN": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is None),
+        "INVALID_OUTPUT": sum(1 for row in rows if row["invalid"]),
+    }
+
+
+_BUCKETS = ((">=0.99", 0.99, 1.01), ("0.95-0.9899", 0.95, 0.99), ("0.90-0.9499", 0.90, 0.95), ("0.80-0.8999", 0.80, 0.90), ("<0.80", -0.01, 0.80))
+
+
+def confidence_buckets(rows) -> dict:
+    out = {name: {"total": 0, "correct": 0} for name, _, _ in _BUCKETS}
+    for row in rows:
+        if row["invalid"] or row["selected"] is None:
+            continue
+        for name, low, high in _BUCKETS:
+            if low <= row["confidence"] < high:
+                out[name]["total"] += 1
+                if row["selected"] == row["gold"]:
+                    out[name]["correct"] += 1
+                break
+    return out

central/app/ai/fingerprint.py

@@ -0,0 +1,75 @@
+"""Product identity: stable fingerprints and GTIN validation.
+
+Global identity uses ONLY stable attributes (normalized description, brand, unit,
+package quantity, GTIN). Fiscal/company context (CFOP, CSOSN, CST, rate, ST, tenant)
+never participates: those vary per context and must not merge distinct knowledge.
+"""
+import hashlib
+import re
+import unicodedata
+
+from ..classification import ENGINE_VERSION  # noqa: F401  (re-exported for the AI layer)
+
+
+def normalize_description(value) -> str:
+    """NFKD -> ascii -> A-Z0-9 tokens, space-joined (same canonical form as classification)."""
+    text = unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper()
+    return " ".join(re.sub(r"[^A-Z0-9]+", " ", text).split())
+
+
+def _normalize_token(value) -> str:
+    return normalize_description(value)
+
+
+def _normalize_quantity(value) -> str:
+    digits = re.sub(r"[^0-9.,]", "", str(value or ""))
+    if not digits:
+        return ""
+    try:
+        number = float(digits.replace(",", "."))
+    except ValueError:
+        return digits
+    return f"{number:g}"
+
+
+def product_fingerprint(description, brand=None, unit=None, package_quantity=None) -> str:
+    parts = [_normalize_token(description), _normalize_token(brand), _normalize_token(unit), _normalize_quantity(package_quantity)]
+    return hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()
+
+
+def _stem(token: str) -> str:
+    """Minimal Portuguese plural stemming so AGUAS matches AGUA / MINERAIS matches MINERAL."""
+    if len(token) > 4 and token.endswith("AIS"):
+        return token[:-3] + "AL"
+    if len(token) > 4 and token.endswith("ES"):
+        return token[:-2]
+    if len(token) > 3 and token.endswith("S"):
+        return token[:-1]
+    return token
+
+
+def token_stems(value) -> set:
+    return {_stem(token) for token in normalize_description(value).split()}
+
+
+def overlap_score(query, official) -> float:
+    """Fraction of query description stems present in the official description stems."""
+    stems = token_stems(query)
+    if not stems:
+        return 0.0
+    return len(stems & token_stems(official)) / len(stems)
+
+
+def validate_gtin(value) -> str | None:
+    """Returns the canonical GTIN when the checksum is valid, else None.
+    Accepts GTIN-8/12/13/14 (EAN-8/UPC-12/EAN-13/GTIN-14)."""
+    digits = re.sub(r"\D", "", str(value or ""))
+    if len(digits) not in (8, 12, 13, 14):
+        return None
+    total = 0
+    for index, char in enumerate(reversed(digits)):
+        digit = int(char)
+        total += digit * (1 if index % 2 == 0 else 3)
+    if total % 10 != 0:
+        return None
+    return digits

central/app/ai/kb.py

@@ -0,0 +1,121 @@
+"""Product Knowledge Base service: identity resolution, versioned knowledge, conflicts.
+
+Identity order (operator contract): 1. exact GTIN; 2. strong fingerprint; 3. semantic
+fingerprint candidate; 4. unknown. Conflicting knowledge for the same key is kept
+side by side and surfaced as KNOWLEDGE_CONFLICT — never picked by majority.
+"""
+from datetime import datetime, timezone
+
+from sqlalchemy import func as sa_func, select
+from ..models import KbKnowledge, KbObservation, KbProduct
+
+EXACT, FINGERPRINT = "EXACT", "FINGERPRINT"
+_AUTHORITATIVE = {"OFFICIAL_DATASET", "OPERATOR", "AI_PLUS_OFFICIAL_VALIDATION"}
+
+
+class KnowledgeBase:
+    def __init__(self, session):
+        self.session = session
+
+    # ----- identity -----
+
+    def upsert_product(self, fingerprint, gtin=None, normalized_description="", brand=None, unit=None, package_quantity=None, category=None):
+        gtin = gtin or None
+        product = None
+        if gtin:
+            product = self.session.scalar(select(KbProduct).where(KbProduct.gtin == gtin))
+        if product is None:
+            product = self.session.scalar(select(KbProduct).where(KbProduct.fingerprint == fingerprint))
+        if product is None:
+            product = KbProduct(fingerprint=fingerprint, normalized_description=normalized_description or "")
+            self.session.add(product)
+        # identity upgrades (never downgrades)
+        if gtin and not product.gtin:
+            product.gtin = gtin
+            product.identity_kind = "GTIN"
+        if normalized_description and not product.normalized_description:
+            product.normalized_description = normalized_description
+        for field, value in (("brand", brand), ("unit", unit), ("package_quantity", package_quantity), ("category", category)):
+            if value and not getattr(product, field):
+                setattr(product, field, value)
+        self.session.flush()
+        return product
+
+    def resolve(self, gtin=None, fingerprint=None):
+        """Returns (product|None, hit_kind). Cache-first: EXACT (GTIN) beats FINGERPRINT."""
+        if gtin:
+            product = self.session.scalar(select(KbProduct).where(KbProduct.gtin == gtin))
+            if product:
+                return product, EXACT
+        if fingerprint:
+            product = self.session.scalar(select(KbProduct).where(KbProduct.fingerprint == fingerprint))
+            if product:
+                return product, FINGERPRINT
+        return None, None
+
+    # ----- knowledge -----
+
+    def add_knowledge(self, kb_product_id, knowledge_key, value, source, confidence=0.0, evidence=None, model=None, prompt_version=None, dataset_versions=None):
+        value = str(value)
+        row = self.session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == kb_product_id, KbKnowledge.knowledge_key == knowledge_key, KbKnowledge.value == value))
+        if row is None:
+            row = KbKnowledge(kb_product_id=kb_product_id, knowledge_key=knowledge_key, value=value, source=source, confidence=confidence, evidence=evidence or {}, model=model, prompt_version=prompt_version, dataset_versions=dataset_versions or {})
+            self.session.add(row)
+        else:
+            # authoritative sources upgrade provenance/confidence of the same value; history preserved
+            if source in _AUTHORITATIVE:
+                row.source = source
+                row.confidence = max(row.confidence, confidence)
+                row.evidence = {**row.evidence, **(evidence or {})}
+                if dataset_versions:
+                    row.dataset_versions = {**row.dataset_versions, **dataset_versions}
+            if source == "OFFICIAL_DATASET":
+                row.validated_at = datetime.now(timezone.utc)
+        self.session.flush()
+        return row
+
+    def current_knowledge(self, kb_product_id):
+        rows = self.session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == kb_product_id)).all()
+        by_key: dict = {}
+        for row in rows:
+            by_key.setdefault(row.knowledge_key, []).append(row)
+        result = {}
+        for key, entries in by_key.items():
+            values = {entry.value for entry in entries}
+            if len(values) > 1:
+                result[key] = {"conflict": True, "values": values}
+            else:
+                best = max(entries, key=lambda entry: entry.confidence)
+                result[key] = {"conflict": False, "value": best.value, "source": best.source, "confidence": best.confidence}
+        return result
+
+    # ----- observations -----
+
+    def record_observation(self, kb_product_id, tenant_id, company_key, erp_code, database_label, observed_fields, source="ERP_OBSERVATION"):
+        row = KbObservation(kb_product_id=kb_product_id, tenant_id=tenant_id, company_key=company_key, erp_code=erp_code, database_label=database_label, observed_fields=observed_fields or {}, source=source)
+        self.session.add(row); self.session.flush()
+        return row
+
+    def observation_summary(self, kb_product_id):
+        rows = self.session.scalars(select(KbObservation).where(KbObservation.kb_product_id == kb_product_id)).all()
+        agreement: dict = {}
+        for row in rows:
+            for field, value in (row.observed_fields or {}).items():
+                if value in (None, ""):
+                    continue
+                bucket = agreement.setdefault(field, {})
+                bucket[str(value)] = bucket.get(str(value), 0) + 1
+        conflicts = {field: values for field, values in agreement.items() if len(values) > 1}
+        return {"observation_count": len(rows), "tenant_count": len({row.tenant_id for row in rows}), "agreement": agreement, "conflicts": conflicts}
+
+    # ----- stats -----
+
+    def stats(self):
+        products = self.session.scalar(select(sa_func.count()).select_from(KbProduct)) or 0
+        gtin_products = self.session.scalar(select(sa_func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
+        conflicts = 0
+        for product_id in self.session.scalars(select(KbProduct.id)).all():
+            knowledge = self.current_knowledge(product_id)
+            conflicts += sum(1 for entry in knowledge.values() if entry.get("conflict"))
+        fingerprint_products = products - gtin_products
+        return {"products": products, "gtin_products": gtin_products, "fingerprint_products": fingerprint_products, "conflicts": conflicts}

central/app/ai/llm_provider.py

@@ -0,0 +1,158 @@
+"""Real LLM provider (OpenAI-compatible) for the NCM semantic evaluation.
+
+Contract (mission REAL LLM EVALUATION V1):
+- The API key lives ONLY in the Hub environment (AI_API_KEY). Never committed, logged,
+  written to artifacts, sent to the Agent or Firebird.
+- The gold answer is never part of the prompt: items carry only description/brand/unit/
+  segment-weak-context + the official top-K hybrid candidates with hierarchical context.
+- Strict structured output: per item {work_item_id, selected_ncm, confidence, abstain,
+  reason_short}; selected_ncm must be in the candidate list or null (abstain).
+- Batching is safe only when the response is valid JSON with 1:1 work_item_id matching,
+  no omitted items, no cross-product mixing. Verified progressively (5/10/20).
+"""
+import hashlib
+import json
+import re
+from dataclasses import dataclass
+
+PROMPT_VERSION_LLM = "llm-ncm-eval-v1"
+
+_SYSTEM_PROMPT = (
+    "You are a Brazilian customs classification assistant. For each product below, choose the "
+    "most likely official NCM code from the provided candidate list, or abstain. "
+    "Rules: selected_ncm MUST be one of the candidate codes for that exact item, or null with "
+    "abstain=true when evidence is insufficient. Never invent a code. confidence is your calibrated "
+    "probability in [0,1]. Reply with ONLY a JSON array, one object per item, in the same order, "
+    'each: {"work_item_id": str, "selected_ncm": str|null, "confidence": float, "abstain": bool, "reason_short": str}.'
+)
+
+
+@dataclass
+class LlmEvalItem:
+    work_item_id: str
+    description: str
+    normalized_description: str
+    brand: str | None
+    unit: str | None
+    segment: str
+    candidates: list  # [(code, official_description, hybrid_score, official_score, kb_score, tenant_count)]
+
+
+class LlmAnswerError(ValueError):
+    pass
+
+
+def item_prompt_payload(item: LlmEvalItem) -> dict:
+    return {
+        "work_item_id": item.work_item_id,
+        "description": item.description,
+        "normalized_description": item.normalized_description,
+        "brand": item.brand,
+        "unit": item.unit,
+        "segment_weak_context": item.segment,
+        "official_candidates": [
+            {"ncm": code, "official_description": text}
+            for code, text, *_ in item.candidates
+        ],
+    }
+
+
+def candidate_set_hash(item: LlmEvalItem) -> str:
+    codes = [code for code, *_ in item.candidates]
+    return hashlib.sha256("|".join(codes).encode()).hexdigest()
+
+
+def input_fingerprint(item: LlmEvalItem) -> str:
+    payload = {**item_prompt_payload(item), "system": _SYSTEM_PROMPT}
+    return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
+
+
+def validate_answer(answer: dict, item: LlmEvalItem) -> dict:
+    """Fail-closed validator: schema, work_item_id, candidate-list confinement."""
+    if not isinstance(answer, dict):
+        raise LlmAnswerError("answer must be an object")
+    if str(answer.get("work_item_id")) != str(item.work_item_id):
+        raise LlmAnswerError(f"work_item_id mismatch: {answer.get('work_item_id')!r} != {item.work_item_id!r}")
+    abstain = bool(answer.get("abstain"))
+    selected = answer.get("selected_ncm")
+    selected = None if selected in (None, "") else str(selected)
+    try:
+        confidence = float(answer.get("confidence"))
+    except (TypeError, ValueError):
+        raise LlmAnswerError("confidence must be numeric")
+    if not 0.0 <= confidence <= 1.0:
+        raise LlmAnswerError("confidence out of [0,1]")
+    codes = {code for code, *_ in item.candidates}
+    if abstain:
+        if selected is not None:
+            raise LlmAnswerError("abstain must carry selected_ncm=null")
+    else:
+        if selected is None:
+            raise LlmAnswerError("non-abstain requires selected_ncm")
+        if selected not in codes:
+            raise LlmAnswerError(f"selected_ncm {selected!r} is outside the candidate list")
+    return {"work_item_id": item.work_item_id, "selected_ncm": selected, "confidence": confidence, "abstain": abstain, "reason_short": str(answer.get("reason_short") or "")[:200]}
+
+
+class OpenAICompatibleProvider:
+    """Talks to any OpenAI-compatible /chat/completions endpoint. `transport` is
+    injectable for tests; production default uses httpx with the key from settings only."""
+
+    name = "openai-compatible"
+    prompt_version = PROMPT_VERSION_LLM
+
+    def __init__(self, api_key: str, model: str, base_url: str = "https://api.openai.com/v1", transport=None, timeout: float = 120.0):
+        self.api_key = api_key
+        self.model = model
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self._transport = transport or self._httpx_transport
+        self.http_calls = 0
+        self.retries = 0
+        self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
+
+    def _httpx_transport(self, payload: dict) -> dict:
+        import httpx
+        response = httpx.post(f"{self.base_url}/chat/completions", json=payload, headers={"Authorization": f"Bearer {self.api_key}"}, timeout=self.timeout)
+        response.raise_for_status()
+        return response.json()
+
+    def list_models(self) -> list:
+        """GET /models with the account key — validates auth/base URL/model availability
+        without spending completion tokens. Never raises with the key attached."""
+        import httpx
+        response = httpx.get(f"{self.base_url}/models", headers={"Authorization": f"Bearer {self.api_key}"}, timeout=self.timeout)
+        response.raise_for_status()
+        data = response.json()
+        return sorted(str(item.get("id")) for item in data.get("data", []) if item.get("id"))
+
+    def _request(self, messages: list, max_retries: int = 2) -> dict:
+        payload = {"model": self.model, "messages": messages, "temperature": 0}
+        last_error = None
+        for attempt in range(max_retries + 1):
+            self.http_calls += 1
+            if attempt:
+                self.retries += 1
+            try:
+                data = self._transport(payload)
+                usage = data.get("usage") or {}
+                self.usage["input_tokens"] += int(usage.get("prompt_tokens") or 0)
+                self.usage["output_tokens"] += int(usage.get("completion_tokens") or 0)
+                cached = usage.get("prompt_tokens_details") or {}
+                self.usage["cached_tokens"] += int(cached.get("cached_tokens") or 0)
+                return data
+            except Exception as error:  # transport-level failure only; JSON contract errors are not retried
+                last_error = error
+        raise RuntimeError(f"LLM transport failed after {max_retries + 1} attempts: {last_error}")
+
+    def select_batch(self, items: list[LlmEvalItem]) -> list[dict]:
+        """One HTTP call for the whole batch. Raises LlmAnswerError on any 1:1/schema violation
+        so the harness can shrink the batch size."""
+        user_content = json.dumps([item_prompt_payload(item) for item in items], ensure_ascii=False)
+        data = self._request([{"role": "system", "content": _SYSTEM_PROMPT}, {"role": "user", "content": user_content}])
+        content = data["choices"][0]["message"]["content"]
+        content = re.sub(r"^```(?:json)?|```$", "", content.strip(), flags=re.MULTILINE).strip()
+        answers = json.loads(content)
+        if not isinstance(answers, list) or len(answers) != len(items):
+            raise LlmAnswerError(f"expected {len(items)} answers, got {len(answers) if isinstance(answers, list) else type(answers).__name__}")
+        return [validate_answer(answer, item) for answer, item in zip(answers, items)]

central/app/ai/provider.py

@@ -0,0 +1,139 @@
+"""Provider abstraction with strict structured output and a hard budget guard.
+
+The fiscal engine is never coupled to a vendor: it talks to AISemanticProvider.
+V1 ships BaselineProvider (deterministic, zero-cost) used by the evaluation
+harness and as the no-credential fallback; real providers subclass the same
+contract and are only invoked with AI_ENABLED=true and credentials in the Hub.
+"""
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+from ..classification import ENGINE_VERSION
+from .fingerprint import overlap_score
+
+PROMPT_VERSION = "ai-semantic-v1"
+
+
+class SemanticOutputError(ValueError):
+    """Raised when a provider answer violates the strict structured-output schema."""
+
+
+@dataclass
+class StructuredResult:
+    task_type: str
+    input_fingerprint: str
+    selected_candidate: str            # candidate code, or "NONE"
+    confidence: float
+    reason: str
+    candidate_ids_seen: list[str]
+    model: str
+    prompt_version: str
+    created_at: str
+    dataset_versions: dict = field(default_factory=dict)
+    engine_version: str = ENGINE_VERSION
+
+    def to_dict(self):
+        return {
+            "task_type": self.task_type,
+            "input_fingerprint": self.input_fingerprint,
+            "selected_candidate": self.selected_candidate,
+            "confidence": self.confidence,
+            "reason": self.reason,
+            "candidate_ids_seen": self.candidate_ids_seen,
+            "model": self.model,
+            "prompt_version": self.prompt_version,
+            "created_at": self.created_at,
+            "dataset_versions": self.dataset_versions,
+            "engine_version": self.engine_version,
+        }
+
+
+def result_from_dict(data) -> StructuredResult:
+    """Fail-closed schema check: the selection must be NONE or one of the seen candidates."""
+    result = StructuredResult(
+        task_type=str(data["task_type"]),
+        input_fingerprint=str(data["input_fingerprint"]),
+        selected_candidate=str(data["selected_candidate"]),
+        confidence=float(data["confidence"]),
+        reason=str(data["reason"]),
+        candidate_ids_seen=[str(item) for item in data["candidate_ids_seen"]],
+        model=str(data["model"]),
+        prompt_version=str(data["prompt_version"]),
+        created_at=str(data["created_at"]),
+        dataset_versions=dict(data.get("dataset_versions") or {}),
+    )
+    if result.selected_candidate != "NONE" and result.selected_candidate not in result.candidate_ids_seen:
+        raise SemanticOutputError(f"selected_candidate {result.selected_candidate!r} is not among the official candidates")
+    if not 0.0 <= result.confidence <= 1.0:
+        raise SemanticOutputError("confidence must be within [0,1]")
+    return result
+
+
+class AISemanticProvider:
+    """Contract every provider must honour. `select` receives ONLY official candidates."""
+    name = "abstract"
+    model = "abstract"
+
+    def select(self, work_item, candidates) -> StructuredResult:
+        raise NotImplementedError
+
+    def estimate(self, work_items) -> dict:
+        raise NotImplementedError
+
+
+def _score(text: str, official: str) -> float:
+    return overlap_score(text, official)
+
+
+class BaselineProvider(AISemanticProvider):
+    """Deterministic zero-cost baseline: highest token-overlap candidate, or NONE.
+    Used for the evaluation pipeline and as the mandatory no-credential fallback."""
+
+    name = "baseline"
+    model = "baseline-deterministic-v1"
+
+    def select(self, work_item, candidates, dataset_versions=None) -> StructuredResult:
+        description = work_item.payload.get("description", "") if hasattr(work_item, "payload") else ""
+        scored = [(code, official, _score(description, official)) for code, official in [(str(c[0]), c[1]) for c in candidates]]
+        best = max(scored, key=lambda row: row[2]) if scored else None
+        selected, _, score = best if best else ("NONE", "", 0.0)
+        if not scored or score <= 0.0:
+            selected = "NONE"
+        return StructuredResult(
+            task_type=work_item.task_type,
+            input_fingerprint=work_item.dedup_key,
+            selected_candidate=selected,
+            confidence=round(score, 4),
+            reason="baseline token-overlap top-1" if selected != "NONE" else "no token overlap with any official candidate",
+            candidate_ids_seen=[str(c[0]) for c in candidates],
+            model=self.model,
+            prompt_version=PROMPT_VERSION,
+            created_at=datetime.now(timezone.utc).isoformat(),
+            dataset_versions=dict(dataset_versions or {}),
+        )
+
+    def estimate(self, work_items) -> dict:
+        return {"input_tokens": 0, "output_tokens": 0, "estimated_cost": 0.0}
+
+
+class BudgetGuard:
+    """Refuses execution when the estimated cost exceeds AI_MAX_RUN_COST_USD.
+    Never falls back to a more expensive model. Prices are operator-configured — when
+    absent, cost is UNKNOWN: the residual run is refused, but an explicitly limited
+    benchmark (e.g. the 65-item GOLD eval) may opt in with allow_unknown_cost."""
+
+    def __init__(self, max_run_cost_usd: float, price_in_per_m: float | None, price_out_per_m: float | None):
+        self.max_run_cost_usd = float(max_run_cost_usd)
+        self.price_in_per_m = None if price_in_per_m is None else float(price_in_per_m)
+        self.price_out_per_m = None if price_out_per_m is None else float(price_out_per_m)
+
+    def estimate(self, items_in_tokens: int, items_out_tokens: int) -> dict:
+        if self.price_in_per_m is None or self.price_out_per_m is None:
+            return {"input_tokens": items_in_tokens, "output_tokens": items_out_tokens, "estimated_cost": None, "cost_unknown": True}
+        cost = (items_in_tokens / 1_000_000) * self.price_in_per_m + (items_out_tokens / 1_000_000) * self.price_out_per_m
+        return {"input_tokens": items_in_tokens, "output_tokens": items_out_tokens, "estimated_cost": round(cost, 6)}
+
+    def authorize(self, plan: dict, allow_unknown_cost: bool = False) -> bool:
+        if plan.get("estimated_cost") is None:
+            return allow_unknown_cost
+        return plan["estimated_cost"] <= self.max_run_cost_usd

central/app/ai/retrieval.py

@@ -0,0 +1,245 @@
+"""Semantic candidate retrieval V2 — deterministic, zero LLM.
+
+Three retrieval sources merged into one ranking:
+  A. OFFICIAL: hierarchical search over the official NCM tree. Every 8-digit code is
+     enriched with its ancestors' descriptions (chapter/position/subposition), so the
+     very short legal text carries the vocabulary of its path (section 6).
+  B. KB NEIGHBORS: the Knowledge Base acts as a commercial->official bridge — products
+     whose commercial description resembles the query contribute their VALIDATED NCMs,
+     weighted by similarity, tenant_count, observation_count and agreement (sections 2-3).
+  C. HYBRID: single final ranking; the KB is only evidence for CANDIDATES, never a
+     majority-vote fiscal decision.
+
+Abbreviation lexicon (section 8) is derived statistically from the KB itself; commercial
+normalization (section 7) strips units/sizes/colors/refs deterministically.
+"""
+from dataclasses import dataclass
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from ..models import Dataset, KbKnowledge, KbObservation, KbProduct, NcmEntry, Tenant
+from .fingerprint import normalize_description, token_stems
+
+# ---------------- commercial normalization (section 7) ----------------
+
+_UNITS = {"UN", "UNID", "UNIDADE", "CX", "CAIXA", "FD", "FDO", "FARDO", "PCT", "PACOTE", "PKG",
+          "KG", "G", "GR", "GRAMA", "MG", "ML", "L", "LT", "LITRO", "LATA", "GARRAFA", "PET",
+          "PC", "PECA", "DZ", "DUZIA", "PAR", "KIT", "JG", "PARA", "COM", "SEM", "C", "S"}
+_DISCARD = _UNITS | {"MASC", "MASCULINO", "FEM", "FEMININO", "INF", "INFANTIL", "TAM", "TAMANHO",
+                     "COR", "REF", "COD", "CODIGO", "MOD", "MODELO", "NOVA", "NOVO", "TIPO"}
+_LEXEME_STRIP_PREFIXES = ("1", "2", "3", "4", "5", "6", "7", "8", "9", "0", ".")
+
+
+def _is_size_or_code(token: str) -> bool:
+    if token.isdigit():
+        return True
+    digits = "".join(char for char in token if char.isdigit())
+    letters = "".join(char for char in token if char.isalpha())
+    return bool(digits) and letters in _UNITS  # "1KG", "500ML", "2LT"
+
+
+def commercial_tokens(description) -> set:
+    """Stable commercial lexemes: units/sizes/colors/refs removed, stems kept."""
+    tokens = []
+    skip_next = False
+    for token in normalize_description(description).split():
+        if skip_next:
+            skip_next = False
+            continue
+        if token in ("TAM", "COR", "REF", "COD", "MOD"):
+            skip_next = True
+            continue
+        if token in _DISCARD or _is_size_or_code(token):
+            continue
+        tokens.append(token)
+    return {_stem_safe(token) for token in tokens}
+
+
+def _stem_safe(token: str) -> str:
+    from .fingerprint import _stem
+    return _stem(token)
+
+
+def expand_stems(stems: set, lexicon: dict) -> set:
+    expanded = set(stems)
+    for stem in stems:
+        target = lexicon.get(stem)
+        if target:
+            expanded.add(target)
+    return expanded
+
+
+def derive_abbreviation_lexicon(corpus: dict, min_support: int = 3) -> dict:
+    """corpus: {commercial_description: ncm}. Statistical abbreviations only: a short token
+    (3-5 chars) that is a strict prefix of a longer token, where BOTH forms occur across
+    distinct products that share the same NCM. Support = (#products with the short form) +
+    (#products with the long form); accepted when support >= min_support and both >= 1."""
+    by_ncm: dict = {}
+    for description, ncm in corpus.items():
+        tokens = set(normalize_description(description).split())
+        by_ncm.setdefault(ncm, []).append(tokens)
+    best: dict = {}
+    for descriptions in by_ncm.values():
+        if len(descriptions) < 2:
+            continue
+        all_tokens = set().union(*descriptions)
+        for short in all_tokens:
+            if not (3 <= len(short) <= 5):
+                continue
+            short_count = sum(1 for tokens in descriptions if short in tokens)
+            if short_count < 1:
+                continue
+            for long_token in all_tokens:
+                if len(long_token) > len(short) and long_token.startswith(short):
+                    long_count = sum(1 for tokens in descriptions if long_token in tokens)
+                    support = short_count + long_count
+                    if support >= min_support and long_count >= 1 and short_count >= 1:
+                        current = best.get(short)
+                        if current is None or support > current[1]:
+                            best[short] = (long_token, support)
+    return {short: target for short, (target, _) in best.items()}
+
+
+# ---------------- enriched official corpus (sections 5-6) ----------------
+
+
+@dataclass
+class EnrichedNcmCorpus:
+    documents: dict            # code -> enriched stems set
+    descriptions: dict         # code -> official description
+    positions: dict            # 4-digit position -> enriched stems (chapter + position)
+    children: dict             # 4-digit position -> [8-digit codes]
+
+    @staticmethod
+    def build(session: Session) -> "EnrichedNcmCorpus":
+        dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+        if dataset is None:
+            return EnrichedNcmCorpus({}, {}, {}, {})
+        entries = session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all()
+        by_code = {entry.code: entry.description for entry in entries}
+        documents, positions, children = {}, {}, {}
+        for code, description in by_code.items():
+            if len(code) != 8:
+                continue
+            ancestors = [text for text in (by_code.get(code[:2]), by_code.get(code[:4]), by_code.get(code[:6])) if text]
+            documents[code] = token_stems(description) | set().union(*[token_stems(text) for text in ancestors]) if ancestors else token_stems(description)
+            positions.setdefault(code[:4], token_stems(by_code.get(code[:4], "")) | token_stems(by_code.get(code[:2], "")))
+            children.setdefault(code[:4], []).append(code)
+        return EnrichedNcmCorpus(documents, by_code, positions, children)
+
+    def document(self, code: str) -> str:
+        return " ".join(sorted(self.documents.get(code, set())))
+
+    def items(self):
+        return list(self.documents.items())
+
+
+def _score(stems: set, document_stems: set, idf: dict | None = None) -> float:
+    if not stems:
+        return 0.0
+    if idf:
+        matched = sum(idf.get(stem, 1.0) for stem in stems if stem in document_stems)
+        total = sum(idf.get(stem, 1.0) for stem in stems)
+        return matched / total
+    return len(stems & document_stems) / len(stems)
+
+
+def hierarchical_candidates(corpus: EnrichedNcmCorpus, description, k: int = 20, max_positions: int = 15):
+    """Two-stage: rank 4-digit positions first, then their 8-digit descendants (section 5)."""
+    stems = commercial_tokens(description)
+    if not stems:
+        return []
+    position_scores = sorted(((_score(stems, doc), position) for position, doc in corpus.positions.items()), reverse=True)
+    ranked = []
+    for position_score, position in position_scores[:max_positions]:
+        if position_score <= 0:
+            continue
+        for code in corpus.children.get(position, []):
+            code_score = _score(stems, corpus.documents[code])
+            if code_score > 0:
+                ranked.append((code, corpus.descriptions.get(code, ""), round(code_score, 4)))
+    ranked.sort(key=lambda row: (-row[2], row[0]))
+    return ranked[:k]
+
+
+# ---------------- KB neighbor bridge (sections 2-3) ----------------
+
+
+@dataclass
+class KbNeighborIndex:
+    entries: list  # (product_id, stems, ncm, tenant_ids:set, observation_count, has_conflict)
+
+    @staticmethod
+    def build(session: Session) -> "KbNeighborIndex":
+        validated = {}
+        for knowledge in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "validated_ncm")).all():
+            validated.setdefault(knowledge.kb_product_id, set()).add(knowledge.value)
+        conflicts = {product_id: len(values) > 1 for product_id, values in validated.items()}
+        products = {product.id: product for product in session.scalars(select(KbProduct)).all()}
+        observations = session.scalars(select(KbObservation)).all()
+        per_product_tenants: dict = {}
+        for observation in observations:
+            per_product_tenants.setdefault(observation.kb_product_id, set()).add(observation.tenant_id)
+        entries = []
+        for product_id, values in validated.items():
+            if conflicts.get(product_id):
+                continue  # conflicting knowledge never feeds neighbors
+            for value in values:
+                product = products.get(product_id)
+                if product is None or not product.normalized_description:
+                    continue
+                tenants = per_product_tenants.get(product_id, set())
+                entries.append((product_id, commercial_tokens(product.normalized_description), value, tenants, len(tenants), False))
+        return KbNeighborIndex(entries)
+
+    def corpus(self) -> dict:
+        return {" ".join(sorted(stems)): ncm for _, stems, ncm, _, _, _ in self.entries}
+
+
+def kb_neighbor_candidates(index: KbNeighborIndex, description, k: int = 10, exclude_product_id: int | None = None):
+    """Aggregate NCMs of lexically similar known products. `exclude_product_id` implements
+    leave-one-out evaluation: the queried product never retrieves itself. Returns
+    (ncm, neighbor_score, observation_count, tenant_count) — agreement is the union of
+    distinct tenants behind the contributing neighbors, reported as evidence, never used
+    as an automatic majority decision."""
+    stems = commercial_tokens(description)
+    if not stems:
+        return []
+    best: dict = {}
+    for product_id, neighbor_stems, ncm, tenant_ids, observation_count, _ in index.entries:
+        if exclude_product_id is not None and product_id == exclude_product_id:
+            continue
+        similarity = _score(stems, neighbor_stems)
+        if similarity <= 0:
+            continue
+        current = best.get(ncm)
+        if current is None:
+            best[ncm] = [ncm, round(similarity, 4), observation_count, set(tenant_ids)]
+        else:
+            current[1] = max(current[1], round(similarity, 4))
+            current[3] |= set(tenant_ids)
+    ranked = sorted(((row[0], row[1], row[2], len(row[3])) for row in best.values()), key=lambda row: (-row[1], -row[3], row[0]))
+    return ranked[:k]
+
+
+# ---------------- hybrid ranking (section 4) ----------------
+
+OFFICIAL_WEIGHT, KB_WEIGHT = 0.3, 0.7  # tuned on the V2 benchmark (KB bridge carries the signal;
+# the official component breaks ties and covers products absent from the KB)
+
+
+def hybrid_candidates(corpus: EnrichedNcmCorpus, index: KbNeighborIndex, description, k: int = 20, lexicon: dict | None = None, exclude_product_id: int | None = None):
+    stems = expand_stems(commercial_tokens(description), lexicon or {})
+    official = hierarchical_candidates(corpus, " ".join(sorted(stems)) or description, k=k)
+    neighbors = kb_neighbor_candidates(index, description, k=k, exclude_product_id=exclude_product_id)
+    combined: dict = {}
+    for code, text, score in official:
+        combined[code] = [code, text, OFFICIAL_WEIGHT * score, score, 0.0, 0]
+    for code, similarity, observation_count, tenant_count in neighbors:
+        row = combined.setdefault(code, [code, "", 0.0, 0.0, 0.0, 0])
+        row[2] += KB_WEIGHT * similarity
+        row[4] = similarity
+        row[5] = tenant_count
+    ranked = sorted(combined.values(), key=lambda row: (-round(row[2], 4), row[0]))
+    return [(row[0], row[1], round(row[2], 4), row[3], row[4], row[5]) for row in ranked[:k]]

central/app/ai/workitems.py

@@ -0,0 +1,64 @@
+"""Residual-question dedup: group pending cells by the KNOWLEDGE needed, not per cell.
+
+- INVALID_NCM  -> one work item per product fingerprint (same product in N databases collapses)
+- CEST/EX_TIPI -> one work item per (doubt, NCM): all databases sharing that NCM collapse
+- PRODUCT_ROLE -> tenant-specific by contract: one work item per (company, group)
+- Everything else (origin, ST retention, company profile, duplicate key) never becomes
+  an AI work item.
+"""
+from dataclasses import dataclass, field
+
+_INVALID_NCM, _ROLE = "ncm", "product_role"
+
+
+@dataclass
+class WorkItem:
+    task_type: str
+    dedup_key: str
+    question_refs: list = field(default_factory=list)
+    cells_covered: int = 0
+    payload: dict = field(default_factory=dict)
+    kb_product_id: int | None = None
+
+
+def build_work_items(questions, product_fingerprints=None) -> list[WorkItem]:
+    """`product_fingerprints` maps "{DATABASE}:{erp_code}" -> KB fingerprint (from the seed pass)."""
+    product_fingerprints = product_fingerprints or {}
+    items: dict = {}
+
+    def _get(task_type, dedup_key, ref, cells):
+        item = items.get(dedup_key)
+        if item is None:
+            item = WorkItem(task_type=task_type, dedup_key=dedup_key)
+            items[dedup_key] = item
+        item.question_refs.append(ref)
+        item.cells_covered += cells
+        return item
+
+    for question in questions:
+        key = question.get("KEY")
+        scope = question.get("SCOPE")
+        cells = int(question.get("CELLS") or question.get("PRODUCTS_AFFECTED") or 0)
+        ref = question.get("QUESTION_ID") or f"{question.get('DATABASE')}:{scope}:{question.get('NCM') or question.get('GROUP')}:{key}"
+        if key in ("cest", "ex_tipi") and scope == "NCM":
+            ncm = str(question.get("NCM") or "").strip()
+            if ncm:
+                item = _get(key.upper(), f"{key}:{ncm}", ref, cells)
+                item.payload.setdefault("ncm", ncm)
+                item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
+        elif key == _INVALID_NCM and scope == "PRODUCT":
+            database = question.get("DATABASE", "")
+            erp_code = str(question.get("DATABASE_CODE") or "").strip()
+            fingerprint = product_fingerprints.get(f"{database}:{erp_code}")
+            dedup = f"invalid_ncm:{fingerprint}" if fingerprint else f"invalid_ncm:{database}:{erp_code}"
+            item = _get("INVALID_NCM", dedup, ref, cells)
+            item.payload.setdefault("erp_code", erp_code)
+            item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
+        elif key == _ROLE and scope == "GROUP":
+            company = question.get("COMPANY", "")
+            group = str(question.get("GROUP") or "").strip()
+            if group:
+                item = _get("PRODUCT_ROLE", f"product_role:{company}:{group}", ref, cells)
+                item.payload.setdefault("company_key", company)
+                item.payload["group"] = group
+    return list(items.values())

central/app/config.py

@@ -31,6 +31,20 @@ class Settings:
     cest_dataset_url: str = os.getenv("CEST_DATASET_URL", "https://www.confaz.fazenda.gov.br/legislacao/convenios/2018/CV142_18")
     dataset_storage_path: str = os.getenv("DATASET_STORAGE_PATH", "./data/datasets")
     http_timeout_seconds: float = float(os.getenv("HTTP_TIMEOUT_SECONDS", "30"))
+    # ---- AI semantic layer V1 ----
+    # Credentials live ONLY in the Hub environment; the Windows Agent never sees them.
+    ai_enabled: bool = os.getenv("AI_ENABLED", "0") == "1"
+    ai_provider: str = os.getenv("AI_PROVIDER", "baseline")     # baseline | openai | ... (AISemanticProvider)
+    ai_model: str = os.getenv("AI_MODEL", "baseline-deterministic-v1")
+    ai_max_batch_size: int = int(os.getenv("AI_MAX_BATCH_SIZE", "20"))
+    ai_max_run_cost_usd: float = float(os.getenv("AI_MAX_RUN_COST_USD", "5.0"))
+    ai_min_confidence: float = float(os.getenv("AI_MIN_CONFIDENCE", "0.6"))
+    ai_price_in_per_m: float | None = float(os.getenv("AI_PRICE_IN_PER_M")) if os.getenv("AI_PRICE_IN_PER_M") else None
+    ai_price_out_per_m: float | None = float(os.getenv("AI_PRICE_OUT_PER_M")) if os.getenv("AI_PRICE_OUT_PER_M") else None
+    # Real-LLM evaluation (OpenAI-compatible). The key lives ONLY in this environment.
+    # Z.ai general endpoint (NOT the /api/coding/paas/v4 coding-plan endpoint).
+    ai_api_key: str = os.getenv("AI_API_KEY", "")
+    ai_base_url: str = os.getenv("AI_BASE_URL", "https://api.z.ai/api/paas/v4")
 
 
 settings = Settings()

central/app/context_keys.py

@@ -7,7 +7,9 @@ ENUMS = {
     "st_status": {"SUBJECT", "NOT_SUBJECT"},
     "st_withheld": {"Y", "N"},
     "icms_treatment": {"TAXED", "EXEMPT", "NOT_TAXED", "ST_WITHHELD", "SUSPENDED", "DEFERRED"},
-    "entry_scope": {"INTRA", "INTER"},
+    # MIXED_OR_DYNAMIC: purchases are not predominantly intra- or interstate, so no single
+    # static entry CFOP is fiscally correct for all operations (section 0, AI semantic layer V1).
+    "entry_scope": {"INTRA", "INTER", "MIXED_OR_DYNAMIC"},
     "rounding": {"A", "T"},
     "piscofins": {"N", "S"},
     "incidence": {"NENHUMA"},

central/app/field_contract.py

@@ -362,20 +362,29 @@ def resolve_product(p, d, company, refs, ev, ctx, manual_ncm=None):
         out["CFOP_ENTRADA"] = blocked("CFOP_ENTRADA")
     else:
         exp_e = None
-        if scope_ctx in ("INTRA", "INTER") and role in ("RESALE", "OWN_PRODUCTION", "INPUT") and st != "UNKNOWN" and not (st == "SUBJECT" and withheld is None):
+        if scope_ctx == "MIXED_OR_DYNAMIC":
-            first = "1" if scope_ctx == "INTRA" else "2"
+            # Scope answered, but no single static entry CFOP is fiscally correct for every
-            stw = st == "SUBJECT" and withheld == "Y"
+            # operation: the ERP default / each operation decides 1xxx vs 2xxx. Never invent one.
-            exp_e = first + ("403" if stw else "102") if role == "RESALE" else first + ("401" if stw else "101")
+            valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
-        valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
+            if valid_e:
-        if valid_e and exp_e and cfe != exp_e and cfe in ENTRY_CFOP:
+                out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_OPERATION_DEPENDENT_MIXED_SCOPE")
-            out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "PROVEN_ROLE_ST_AND_SCOPE_CONTRADICT_DECLARED_CFOP_ENTRADA")
+            else:
-        elif valid_e:
+                out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", MANUAL, cfe, reason="CFOP_ENTRADA_OPERATION_DEPENDENT_MIXED_SCOPE")
-            out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_NOT_CONTRADICTED")
-        elif exp_e:
-            out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "DERIVED_FROM_ROLE_ST_AND_ENTRY_SCOPE")
         else:
-            key = "product_role" if role == "UNKNOWN" else ("entry_scope" if scope_ctx not in ("INTRA", "INTER") else ("st_status" if st == "UNKNOWN" else "st_withheld"))
+            if scope_ctx in ("INTRA", "INTER") and role in ("RESALE", "OWN_PRODUCTION", "INPUT") and st != "UNKNOWN" and not (st == "SUBJECT" and withheld is None):
-            out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", MANUAL, cfe, reason="CFOP_ENTRADA_NEEDS_" + key.upper(), key=key, scope="COMPANY" if key == "entry_scope" else "GROUP")
+                first = "1" if scope_ctx == "INTRA" else "2"
+                stw = st == "SUBJECT" and withheld == "Y"
+                exp_e = first + ("403" if stw else "102") if role == "RESALE" else first + ("401" if stw else "101")
+            valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
+            if valid_e and exp_e and cfe != exp_e and cfe in ENTRY_CFOP:
+                out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "PROVEN_ROLE_ST_AND_SCOPE_CONTRADICT_DECLARED_CFOP_ENTRADA")
+            elif valid_e:
+                out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_NOT_CONTRADICTED")
+            elif exp_e:
+                out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "DERIVED_FROM_ROLE_ST_AND_ENTRY_SCOPE")
+            else:
+                key = "product_role" if role == "UNKNOWN" else ("entry_scope" if scope_ctx not in ("INTRA", "INTER") else ("st_status" if st == "UNKNOWN" else "st_withheld"))
+                out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", MANUAL, cfe, reason="CFOP_ENTRADA_NEEDS_" + key.upper(), key=key, scope="COMPANY" if key == "entry_scope" else "GROUP")
 
     # ---------------- origin (needed by CST) ----------------
     cod_o = cur["COD_ORIGEM_MERCADORIA"]

central/app/models.py

@@ -339,3 +339,128 @@ class FiscalContextEntry(Base):
     reason: Mapped[str | None] = mapped_column(Text, nullable=True)
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
     updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+# ================= AI semantic layer V1 =================
+# The AI layer is NOT a fiscal authority: it only selects among official candidates and
+# never writes to Firebird. Knowledge is versioned, provenance-tracked and never deleted.
+
+
+class KbProduct(Base):
+    """GLOBAL product knowledge: only facts inherent to the product (never tenant context)."""
+    __tablename__ = "kb_products"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True)
+    gtin: Mapped[str | None] = mapped_column(String(16), unique=True, nullable=True, index=True)
+    identity_kind: Mapped[str] = mapped_column(String(16), default="FINGERPRINT")  # GTIN | FINGERPRINT | SEMANTIC | UNKNOWN
+    normalized_description: Mapped[str] = mapped_column(Text, default="")
+    brand: Mapped[str | None] = mapped_column(Text, nullable=True)
+    unit: Mapped[str | None] = mapped_column(String(32), nullable=True)
+    package_quantity: Mapped[str | None] = mapped_column(String(32), nullable=True)
+    category: Mapped[str | None] = mapped_column(Text, nullable=True)
+    semantic_attributes: Mapped[dict] = mapped_column(JSON, default=dict)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+
+class KbKnowledge(Base):
+    """Versioned knowledge rows. Unique per (product, key, value): conflicting values for the
+    same key are kept side by side and surfaced as KNOWLEDGE_CONFLICT, never majority-picked."""
+    __tablename__ = "kb_knowledge"
+    __table_args__ = (UniqueConstraint("kb_product_id", "knowledge_key", "value", name="uq_kb_knowledge_value"),)
+    id: Mapped[int] = mapped_column(primary_key=True)
+    kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
+    knowledge_key: Mapped[str] = mapped_column(String(48), index=True)  # candidate_ncm | validated_ncm | candidate_cest | candidate_ex_tipi | ...
+    value: Mapped[str] = mapped_column(Text)
+    source: Mapped[str] = mapped_column(String(32))  # OFFICIAL_DATASET | ERP_OBSERVATION | OPERATOR | AI | AI_PLUS_OFFICIAL_VALIDATION
+    confidence: Mapped[float] = mapped_column(default=0.0)
+    evidence: Mapped[dict] = mapped_column(JSON, default=dict)
+    model: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    prompt_version: Mapped[str | None] = mapped_column(String(32), nullable=True)
+    dataset_versions: Mapped[dict] = mapped_column(JSON, default=dict)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    validated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+
+
+class KbObservation(Base):
+    """Raw per-tenant ERP observation of a KB product (evidence aggregation only)."""
+    __tablename__ = "kb_observations"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
+    tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
+    company_key: Mapped[str] = mapped_column(String(64), index=True)
+    erp_code: Mapped[str] = mapped_column(String(32))
+    database_label: Mapped[str] = mapped_column(String(64))
+    observed_fields: Mapped[dict] = mapped_column(JSON, default=dict)
+    source: Mapped[str] = mapped_column(String(32), default="ERP_OBSERVATION")
+    observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+class TenantProductContext(Base):
+    """TENANT-specific product context: role, ST-withheld, operation profile, operator decisions.
+    Never promoted to global knowledge."""
+    __tablename__ = "tenant_product_context"
+    __table_args__ = (UniqueConstraint("tenant_id", "company_key", "erp_code", name="uq_tenant_product_context"),)
+    id: Mapped[int] = mapped_column(primary_key=True)
+    tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
+    company_key: Mapped[str] = mapped_column(String(64), index=True)
+    erp_code: Mapped[str] = mapped_column(String(32))
+    kb_product_id: Mapped[int | None] = mapped_column(ForeignKey("kb_products.id"), nullable=True)
+    product_role: Mapped[str | None] = mapped_column(String(16), nullable=True)  # RESALE | OWN_PRODUCTION | INPUT | SERVICE | UNKNOWN
+    st_withheld: Mapped[str | None] = mapped_column(String(4), nullable=True)    # Y | N — never AI-decided
+    operation_profile: Mapped[dict] = mapped_column(JSON, default=dict)
+    source: Mapped[str] = mapped_column(String(32), default="ERP_OBSERVATION")
+    confidence: Mapped[float] = mapped_column(default=0.0)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+
+class AiWorkItem(Base):
+    """Deduplicated semantic work item (one knowledge need, many pending cells)."""
+    __tablename__ = "ai_work_items"
+    __table_args__ = (UniqueConstraint("dedup_key", name="uq_ai_work_item_dedup"),)
+    id: Mapped[int] = mapped_column(primary_key=True)
+    task_type: Mapped[str] = mapped_column(String(24), index=True)  # INVALID_NCM | CEST | EX_TIPI | PRODUCT_ROLE
+    dedup_key: Mapped[str] = mapped_column(String(160))
+    question_refs: Mapped[list] = mapped_column(JSON, default=list)
+    payload: Mapped[dict] = mapped_column(JSON, default=dict)
+    status: Mapped[str] = mapped_column(String(32), default="PENDING", index=True)  # PENDING | KB_EXACT_HIT | KB_FINGERPRINT_HIT | HIGH_CONFIDENCE_RETRIEVAL | SINGLE_OFFICIAL_CANDIDATE | LLM_SELECTION_REQUIRED | UNRESOLVED | UNRESOLVED_NO_OFFICIAL_MAPPING | CONFLICT | DONE | SKIPPED
+    kb_product_id: Mapped[int | None] = mapped_column(ForeignKey("kb_products.id"), nullable=True)
+    cells_covered: Mapped[int] = mapped_column(Integer, default=0)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+
+class AiRun(Base):
+    """One provider execution over a batch of work items, with budget-guard evidence."""
+    __tablename__ = "ai_runs"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    provider: Mapped[str] = mapped_column(String(64))
+    model: Mapped[str] = mapped_column(String(64))
+    prompt_version: Mapped[str] = mapped_column(String(32))
+    work_item_ids: Mapped[list] = mapped_column(JSON, default=list)
+    input_tokens: Mapped[int] = mapped_column(Integer, default=0)
+    output_tokens: Mapped[int] = mapped_column(Integer, default=0)
+    cost_usd: Mapped[float] = mapped_column(default=0.0)
+    status: Mapped[str] = mapped_column(String(16), default="RUN")
+    results: Mapped[list] = mapped_column(JSON, default=list)
+    budget_guard: Mapped[dict] = mapped_column(JSON, default=dict)
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+
+
+class AiEvalCache(Base):
+    """Persistent LLM result cache: identical (input_fingerprint, candidate_set_hash,
+    prompt_version, model, dataset_versions) re-execution makes ZERO new calls."""
+    __tablename__ = "ai_eval_cache"
+    __table_args__ = (UniqueConstraint("input_fingerprint", "candidate_set_hash", "prompt_version", "model", "dataset_version_ncm", name="uq_ai_eval_cache"),)
+    id: Mapped[int] = mapped_column(primary_key=True)
+    input_fingerprint: Mapped[str] = mapped_column(String(64), index=True)
+    candidate_set_hash: Mapped[str] = mapped_column(String(64))
+    prompt_version: Mapped[str] = mapped_column(String(32))
+    model: Mapped[str] = mapped_column(String(64))
+    dataset_version_ncm: Mapped[str] = mapped_column(String(64))
+    phase: Mapped[str] = mapped_column(String(16))
+    answer: Mapped[dict] = mapped_column(JSON)
+    gold_ncm: Mapped[str | None] = mapped_column(String(8), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())

central/scripts/ai_cost_plan.py

@@ -0,0 +1,69 @@
+"""Cost plan for the residual AI work items, with budget guard (section 18).
+
+Runs BEFORE any real execution: UNIQUE_AI_WORK_ITEMS, token estimates per task type,
+estimated cost, and the authorize/refuse verdict against AI_MAX_RUN_COST_USD.
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+from central.app.ai.provider import BudgetGuard
+from central.app.config import settings
+from central.app.db import SessionLocal
+from central.app.models import AiWorkItem
+
+TOKENS_PER_ITEM = {  # conservative prompt+schema overhead per single work item
+    "INVALID_NCM": (700, 120),
+    "CEST": (500, 80),
+    "EX_TIPI": (500, 80),
+    "PRODUCT_ROLE": (400, 60),
+}
+
+
+def main(output="docs/AI_COST_PLAN_V1.json") -> dict:
+    session = SessionLocal()
+    items = session.scalars(select(AiWorkItem)).all()
+    guard = BudgetGuard(settings.ai_max_run_cost_usd, settings.ai_price_in_per_m, settings.ai_price_out_per_m)
+    per_task = {}
+    in_total = out_total = 0
+    resolved = {"KB_HIT", "KB_EXACT_HIT", "KB_FINGERPRINT_HIT", "HIGH_CONFIDENCE_RETRIEVAL", "SINGLE_OFFICIAL_CANDIDATE", "DONE", "SKIPPED", "CONFLICT"}
+    for item in items:
+        if item.status in resolved:
+            continue  # cache/official candidate resolved — no AI call needed
+        tokens_in, tokens_out = TOKENS_PER_ITEM.get(item.task_type, (500, 80))
+        bucket = per_task.setdefault(item.task_type, {"items": 0, "input_tokens": 0, "output_tokens": 0, "cells_covered": 0})
+        bucket["items"] += 1; bucket["input_tokens"] += tokens_in; bucket["output_tokens"] += tokens_out
+        bucket["cells_covered"] += item.cells_covered
+        in_total += tokens_in; out_total += tokens_out
+    plan = guard.estimate(in_total, out_total)
+    report = {
+        "artifact": "AI_COST_PLAN_V1",
+        "unique_ai_work_items": sum(bucket["items"] for bucket in per_task.values()),
+        "per_task_type": per_task,
+        "estimated_input_tokens": plan["input_tokens"],
+        "estimated_output_tokens": plan["output_tokens"],
+        "estimated_cost_usd": plan["estimated_cost"],
+        "ai_max_run_cost_usd": guard.max_run_cost_usd,
+        "budget_authorized": guard.authorize(plan),
+        "provider_configured": settings.ai_provider,
+        "model_configured": settings.ai_model,
+        "ai_enabled": settings.ai_enabled,
+        "max_batch_size": settings.ai_max_batch_size,
+        "min_confidence": settings.ai_min_confidence,
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", default="/tmp/ai_cost_plan.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.output), indent=2))
+

central/scripts/audit_gtin_kb.py

@@ -0,0 +1,60 @@
+"""GTIN KB audit (section 10) — validity, cross-tenant matches and conflicts.
+
+Reads the seed payload (ground truth of what was observed per tenant) plus the KB tables.
+"""
+import json
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from central.app.ai.fingerprint import validate_gtin
+
+
+def main(payload_path="docs/runs/ai_kb_seed_payload.json", output="docs/GTIN_KB_AUDIT_V1.json") -> dict:
+    payload = json.loads(Path(payload_path).read_text(encoding="utf8"))
+    by_gtin = defaultdict(list)
+    invalid_gtin_raw = 0
+    for product in payload["products"]:
+        raw = product.get("gtin")
+        if raw is None:
+            continue
+        if validate_gtin(raw) is None:
+            invalid_gtin_raw += 1  # defensive: export already validates
+            continue
+        by_gtin[raw].append(product)
+    duplicated_same_product = sum(1 for entries in by_gtin.values() if len(entries) > 1)
+    conflicting = 0
+    cross_tenant = 0
+    conflict_examples = []
+    for gtin, entries in by_gtin.items():
+        ncms = {entry.get("ncm") for entry in entries if entry.get("ncm")}
+        databases = {obs["database"] for entry in entries for obs in entry["observations"]}
+        if len(ncms) > 1:
+            conflicting += 1
+            if len(conflict_examples) < 10:
+                conflict_examples.append({"gtin": gtin, "ncms": sorted(ncms)})
+        if len(databases) > 1:
+            cross_tenant += 1
+    report = {
+        "artifact": "GTIN_KB_AUDIT_V1",
+        "gtin_valid": len(by_gtin),
+        "gtin_invalid_raw": invalid_gtin_raw,
+        "gtin_duplicated_same_product": duplicated_same_product,
+        "gtin_conflicting_products": conflicting,
+        "gtin_cross_tenant_matches": cross_tenant,
+        "conflict_examples": conflict_examples,
+        "policy": "same GTIN across tenants is STRONG evidence for sharing global product knowledge; tenant fiscal context is never shared",
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    return report
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--payload-path", default="/tmp/ai_kb_seed_payload.json")
+    parser.add_argument("--output", default="/tmp/gtin_audit.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.payload_path, args.output), indent=2))

central/scripts/build_ai_work_items.py

@@ -0,0 +1,83 @@
+"""Build deduplicated AI work items from the residual questions and the seeded KB.
+
+Reads docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json, resolves each invalid-NCM product
+question to its KB fingerprint (same product in several databases collapses into ONE
+work item), persists AiWorkItem rows and writes docs/AI_RESIDUAL_PLAN_V1.json.
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+
+from central.app.ai.kb import KnowledgeBase
+from central.app.ai.workitems import build_work_items
+from central.app.db import SessionLocal
+from central.app.models import AiWorkItem, KbObservation
+
+
+def _erp_code_from_question(question) -> str:
+    import re
+    match = re.search(r"PRODUCT\s+(\d+)", question.get("QUESTION", ""))
+    return match.group(1) if match else ""
+
+
+def main(questions_path="docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json", output="docs/AI_RESIDUAL_PLAN_V1.json") -> dict:
+    session = SessionLocal()
+    questions = json.loads(Path(questions_path).read_text(encoding="utf8"))["questions"]
+    # map "{DATABASE}:{erp_code}" -> fingerprint using the seeded observations
+    rows = session.execute(select(KbObservation.company_key, KbObservation.erp_code, KbObservation.kb_product_id, KbObservation.database_label)).all()
+    product_ids = {f"{row.database_label}:{row.erp_code}": row.kb_product_id for row in rows}
+    questions = [dict(question, DATABASE_CODE=_erp_code_from_question(question)) for question in questions]
+    product_fingerprints = {key: f"kb:{value}" for key, value in product_ids.items()}
+    items = build_work_items(questions, product_fingerprints)
+    raw = sum(len(item.question_refs) for item in items)
+    persisted = {"created": 0, "updated": 0, "kb_hits": 0, "conflicts": 0}
+    kb = KnowledgeBase(session)
+    for item in items:
+        row = session.scalar(select(AiWorkItem).where(AiWorkItem.dedup_key == item.dedup_key))
+        if row is None:
+            row = AiWorkItem(task_type=item.task_type, dedup_key=item.dedup_key, question_refs=item.question_refs, payload=item.payload, cells_covered=item.cells_covered, status="PENDING")
+            session.add(row); persisted["created"] += 1
+        else:
+            row.question_refs = item.question_refs; row.cells_covered = item.cells_covered; row.payload = item.payload
+            persisted["updated"] += 1
+        # cache-first: an existing non-conflicting validated answer makes this a KB hit, no AI call
+        if item.dedup_key.startswith("invalid_ncm:kb:") and item.dedup_key != "invalid_ncm:kb:None":
+            kb_product_id = int(item.dedup_key.rsplit(":", 1)[1])
+            knowledge = kb.current_knowledge(kb_product_id)
+            validated = knowledge.get("validated_ncm", {})
+            if validated and not validated.get("conflict"):
+                row.status = "KB_HIT"; persisted["kb_hits"] += 1
+            elif validated.get("conflict"):
+                row.status = "CONFLICT"; persisted["conflicts"] += 1
+    session.commit()
+    by_task = {}
+    for item in items:
+        bucket = by_task.setdefault(item.task_type, {"items": 0, "cells": 0})
+        bucket["items"] += 1; bucket["cells"] += item.cells_covered
+    report = {
+        "artifact": "AI_RESIDUAL_PLAN_V1",
+        "raw_residual_questions": len(questions),
+        "ai_work_items_raw": raw,
+        "ai_work_items_deduped": len(items),
+        "by_task_type": by_task,
+        "persisted": persisted,
+        "ai_write_allowed": False,
+        "company_decisions_remaining": 14,
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--questions", default="docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json")
+    parser.add_argument("--output", default="docs/AI_RESIDUAL_PLAN_V1.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.questions, args.output), indent=2))
+

central/scripts/reprocess_work_items_v2.py

@@ -0,0 +1,112 @@
+"""Reprocess the 750 AI work items with V2 retrieval (section 16).
+
+Classification per item (INVALID_NCM): identity resolve (GTIN -> fingerprint) against the
+KB; validated non-conflicting knowledge resolves without AI. Otherwise hybrid retrieval:
+a HIGH_CONFIDENCE_RETRIEVAL requires hybrid score >= 0.75, an officially valid NCM and
+cross-tenant agreement >= 2 — it is NOT an automatic fiscal resolution (validator still
+required). Everything else with candidates -> LLM_SELECTION_REQUIRED; without candidates
+-> UNRESOLVED.
+
+CEST/EX_TIPI items never call retrieval semantics beyond the official candidate lists:
+exactly one official candidate resolves deterministically; none is UNRESOLVED
+(NO_OFFICIAL_MAPPING); more than one requires (future) LLM selection.
+PRODUCT_ROLE stays LLM/manual by contract.
+"""
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+
+from central.app.ai.candidates import cest_candidates_for_ncm, ex_tipi_candidates_for_ncm
+from central.app.ai.kb import EXACT, FINGERPRINT, KnowledgeBase
+from central.app.ai.retrieval import (EnrichedNcmCorpus, KbNeighborIndex, derive_abbreviation_lexicon, hybrid_candidates)
+from central.app.db import SessionLocal
+from central.app.models import AiWorkItem, Dataset, KbObservation, KbProduct, NcmEntry
+
+HIGH_CONFIDENCE_THRESHOLD = 0.75
+
+
+def _resolve_product(session, dedup_key: str):
+    """invalid_ncm:kb:{id} -> KB product by id; invalid_ncm:{db}:{code} -> via observation."""
+    if dedup_key.startswith("invalid_ncm:kb:"):
+        return session.get(KbProduct, int(dedup_key.rsplit(":", 1)[1]))
+    parts = dedup_key.split(":")
+    if len(parts) == 3:
+        observation = session.scalar(select(KbObservation).where(KbObservation.database_label == parts[1], KbObservation.erp_code == parts[2]))
+        return session.get(KbProduct, observation.kb_product_id) if observation else None
+    return None
+
+
+def main(output="docs/AI_RESIDUAL_PLAN_V2.json") -> dict:
+    session = SessionLocal()
+    kb = KnowledgeBase(session)
+    corpus = EnrichedNcmCorpus.build(session)
+    index = KbNeighborIndex.build(session)
+    lexicon = derive_abbreviation_lexicon(index.corpus(), min_support=3)
+    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+    valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all()) if dataset else set()
+    items = session.scalars(select(AiWorkItem)).all()
+    counts = {key: 0 for key in ("KB_EXACT_HIT", "KB_FINGERPRINT_HIT", "HIGH_CONFIDENCE_RETRIEVAL", "SINGLE_OFFICIAL_CANDIDATE", "LLM_SELECTION_REQUIRED", "UNRESOLVED")}
+    llm_cells = 0
+    for item in items:
+        if item.task_type == "INVALID_NCM":
+            product = None
+            product = _resolve_product(session, item.dedup_key)
+            if product is not None and product.gtin:
+                resolved, hit = kb.resolve(gtin=product.gtin)
+                if resolved and hit == EXACT:
+                    knowledge = kb.current_knowledge(resolved.id).get("validated_ncm", {})
+                    if knowledge and not knowledge.get("conflict"):
+                        item.status = "KB_EXACT_HIT"; counts["KB_EXACT_HIT"] += 1; continue
+            if product is not None:
+                resolved, hit = kb.resolve(fingerprint=product.fingerprint)
+                if resolved and hit == FINGERPRINT:
+                    knowledge = kb.current_knowledge(resolved.id).get("validated_ncm", {})
+                    if knowledge and not knowledge.get("conflict"):
+                        item.status = "KB_FINGERPRINT_HIT"; counts["KB_FINGERPRINT_HIT"] += 1; continue
+            description = product.normalized_description if product is not None else (item.payload or {}).get("description", "")
+            ranked = hybrid_candidates(corpus, index, description, k=10, lexicon=lexicon, exclude_product_id=(product.id if product else None))
+            if ranked and ranked[0][2] >= HIGH_CONFIDENCE_THRESHOLD and ranked[0][0] in valid_ncm and ranked[0][5] >= 2:
+                item.status = "HIGH_CONFIDENCE_RETRIEVAL"; counts["HIGH_CONFIDENCE_RETRIEVAL"] += 1
+                item.payload = {**item.payload, "retrieval_top1": ranked[0][0], "retrieval_score": ranked[0][2], "tenant_agreement": ranked[0][5]}
+            elif ranked:
+                item.status = "LLM_SELECTION_REQUIRED"; counts["LLM_SELECTION_REQUIRED"] += 1; llm_cells += item.cells_covered
+            else:
+                item.status = "UNRESOLVED"; counts["UNRESOLVED"] += 1
+        elif item.task_type in ("CEST", "EX_TIPI"):
+            ncm = (item.payload or {}).get("ncm", "")
+            candidates = cest_candidates_for_ncm(session, ncm) if item.task_type == "CEST" else ex_tipi_candidates_for_ncm(session, ncm)
+            if len(candidates) == 1:
+                item.status = "SINGLE_OFFICIAL_CANDIDATE"; counts["SINGLE_OFFICIAL_CANDIDATE"] += 1
+            elif not candidates:
+                item.status = "UNRESOLVED_NO_OFFICIAL_MAPPING"; counts["UNRESOLVED"] += 1
+            else:
+                item.status = "LLM_SELECTION_REQUIRED"; counts["LLM_SELECTION_REQUIRED"] += 1; llm_cells += item.cells_covered
+        else:  # PRODUCT_ROLE
+            item.status = "LLM_SELECTION_REQUIRED"; counts["LLM_SELECTION_REQUIRED"] += 1; llm_cells += item.cells_covered
+    session.commit()
+    llm_items = counts["LLM_SELECTION_REQUIRED"]
+    report = {
+        "artifact": "AI_RESIDUAL_PLAN_V2",
+        "work_items": len(items),
+        "classification": counts,
+        "llm_selection_required_items": llm_items,
+        "llm_selection_required_cells": llm_cells,
+        "high_confidence_threshold": HIGH_CONFIDENCE_THRESHOLD,
+        "note": "HIGH_CONFIDENCE_RETRIEVAL is a candidate, not a fiscal resolution: the official validator + operator gate still apply. AI_WRITE_ALLOWED=false.",
+        "dataset_versions": {"ncm": dataset.sha256} if dataset else {},
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", default="/tmp/plan_v2.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.output), indent=2))

central/scripts/run_ai_evaluation.py

@@ -0,0 +1,100 @@
+"""NCM evaluation benchmark (section 19) — same pipeline as production, known answer hidden.
+
+For a stratified sample of products whose NCM is valid in the ACTIVE dataset:
+  candidate generation (ncm_top_k)  ->  provider selection  ->  validator.
+Metrics: EXACT_MATCH / WRONG_SELECTION / NONE_WHEN_KNOWN / INVALID_OUTPUT (+ recall@K),
+per segment (database). With no AI credential the BaselineProvider runs: it measures the
+deterministic floor of the same pipeline and costs nothing.
+"""
+import argparse
+import json
+import random
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+
+from central.app.ai.candidates import ncm_top_k
+from central.app.ai.provider import BaselineProvider, PROMPT_VERSION, SemanticOutputError, result_from_dict
+from central.app.ai.fingerprint import normalize_description
+from central.app.db import SessionLocal
+from central.app.models import Dataset, KbObservation, KbProduct, NcmEntry
+
+SEGMENTS = {"CONSTRUCAO": "construcao", "MMMODAS": "moda", "RINCAO": "restaurante", "UNIAO": "mercado", "TESTESOFICIAIS": "base_testes"}
+
+
+def _samples(session, per_segment=40, seed=42):
+    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+    valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all())
+    rows = session.execute(select(KbProduct.normalized_description, KbObservation.database_label, KbObservation.observed_fields).join(KbObservation, KbObservation.kb_product_id == KbProduct.id)).all()
+    by_segment = {}
+    for description, database, fields in rows:
+        ncm = (fields or {}).get("ncm")
+        if ncm and ncm in valid_ncm and description:
+            by_segment.setdefault(SEGMENTS.get(database, database), set()).add((description, ncm))
+    rng = random.Random(seed)
+    samples = []
+    for segment, values in sorted(by_segment.items()):
+        picked = sorted(values)[: per_segment] if len(values) <= per_segment else rng.sample(sorted(values), per_segment)
+        samples.extend({"segment": segment, "description": description, "true_ncm": ncm} for description, ncm in picked)
+    return samples, dataset.sha256
+
+
+def main(per_segment=40, k=10, output="docs/AI_EVALUATION_V1.json") -> dict:
+    session = SessionLocal()
+    samples, ncm_sha = _samples(session, per_segment)
+    provider = BaselineProvider()
+    metrics = {"EXACT_MATCH": 0, "WRONG_SELECTION": 0, "NONE_WHEN_KNOWN": 0, "INVALID_OUTPUT": 0, "RECALL_AT_K": 0}
+    per_segment = {}
+    for sample in samples:
+        candidates = ncm_top_k(session, sample["description"], k=k)
+        codes = [code for code, _, _ in candidates]
+        bucket = per_segment.setdefault(sample["segment"], {key: 0 for key in metrics} | {"TOTAL": 0})
+        bucket["TOTAL"] += 1
+        if sample["true_ncm"] in codes:
+            metrics["RECALL_AT_K"] += 1; bucket["RECALL_AT_K"] += 1
+        item = type("W", (), {"task_type": "INVALID_NCM", "dedup_key": f"eval:{sample['segment']}:{normalize_description(sample['description'])[:60]}", "payload": {"description": sample["description"]}})()
+        try:
+            result = result_from_dict(provider.select(item, candidates, dataset_versions={"ncm": ncm_sha}).to_dict())
+        except SemanticOutputError:
+            metrics["INVALID_OUTPUT"] += 1; bucket["INVALID_OUTPUT"] += 1
+            continue
+        if result.selected_candidate == sample["true_ncm"]:
+            metrics["EXACT_MATCH"] += 1; bucket["EXACT_MATCH"] += 1
+        elif result.selected_candidate == "NONE":
+            metrics["NONE_WHEN_KNOWN"] += 1; bucket["NONE_WHEN_KNOWN"] += 1
+        else:
+            metrics["WRONG_SELECTION"] += 1; bucket["WRONG_SELECTION"] += 1
+    total = len(samples)
+    report = {
+        "artifact": "AI_EVALUATION_V1",
+        "task": "INVALID_NCM candidate selection",
+        "pipeline": "candidate_generation -> provider_selection -> validator",
+        "provider": provider.name,
+        "model": provider.model,
+        "prompt_version": PROMPT_VERSION,
+        "dataset_versions": {"ncm": ncm_sha},
+        "k": k,
+        "eval_total": total,
+        "metrics": metrics,
+        "accuracy_exact_match": round(metrics["EXACT_MATCH"] / total, 4) if total else None,
+        "recall_at_k": round(metrics["RECALL_AT_K"] / total, 4) if total else None,
+        "by_segment": per_segment,
+        "cost_usd": 0.0,
+        "note": "baseline provider (deterministic, zero-cost): measures the pipeline floor; rerun with a real provider once credentials exist",
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--per-segment", type=int, default=40)
+    parser.add_argument("--k", type=int, default=10)
+    parser.add_argument("--output", default="docs/AI_EVALUATION_V1.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.per_segment, args.k, args.output), indent=2))
+

central/scripts/run_ncm_retrieval_eval_v2.py

@@ -0,0 +1,115 @@
+"""NCM retrieval evaluation V2 (sections 1, 13).
+
+Same sampling as V1 (seed 42, per-segment) with each sample now quality-classified:
+  GOLD   - GTIN product or cross-tenant agreement on the same fingerprint/NCM
+  SILVER - single-tenant valid NCM, no independent confirmation
+  NOISY  - commercial description with < 2 informative tokens (not pursued)
+Retrieval measured separately: OFFICIAL_ONLY (hierarchical enriched corpus),
+KB_ONLY (neighbor bridge), HYBRID; recall@1/5/10/20/50 + MRR, per class and segment.
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+
+from central.app.ai.retrieval import (EnrichedNcmCorpus, KbNeighborIndex, commercial_tokens, derive_abbreviation_lexicon,
+                                      hierarchical_candidates, hybrid_candidates, kb_neighbor_candidates)
+from central.app.db import SessionLocal
+from central.app.models import Dataset, KbObservation, KbProduct, NcmEntry
+
+SEGMENTS = {"CONSTRUCAO": "construcao", "MMMODAS": "moda", "RINCAO": "restaurante", "UNIAO": "mercado", "TESTESOFICIAIS": "base_testes"}
+KS = (1, 5, 10, 20, 50)
+
+
+def _samples(session, per_segment=40, seed=42):
+    import random
+    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+    valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all())
+    rows = session.execute(
+        select(KbProduct, KbObservation).join(KbObservation, KbObservation.kb_product_id == KbProduct.id)
+    ).all()
+    per_product_tenants = {}
+    for product, observation in rows:
+        per_product_tenants.setdefault(product.id, set()).add(observation.tenant_id)
+    by_segment = {}
+    for product, observation in rows:
+        ncm = (observation.observed_fields or {}).get("ncm")
+        if ncm and ncm in valid_ncm and product.normalized_description:
+            by_segment.setdefault(SEGMENTS.get(observation.database_label, observation.database_label), set()).add(
+                (product.id, product.normalized_description, ncm, bool(product.gtin), len(per_product_tenants[product.id]) >= 2)
+            )
+    rng = random.Random(seed)
+    samples = []
+    for segment, values in sorted(by_segment.items()):
+        picked = sorted(values) if len(values) <= per_segment else rng.sample(sorted(values), per_segment)
+        for product_id, description, ncm, has_gtin, cross_tenant in picked:
+            quality = "NOISY" if len(commercial_tokens(description)) < 2 else ("GOLD" if (has_gtin or cross_tenant) else "SILVER")
+            samples.append({"segment": segment, "product_id": product_id, "description": description, "true_ncm": ncm, "quality": quality})
+    return samples
+
+
+def _recall_mrr(runs, samples):
+    """runs: {mode: [ranked_codes_per_sample]}"""
+    out = {}
+    for mode, ranked_lists in runs.items():
+        metrics = {f"recall_at_{k}": 0 for k in KS} | {"mrr": 0.0}
+        total = 0
+        for sample, ranked in zip(samples, ranked_lists):
+            codes = [row[0] for row in ranked]
+            total += 1
+            for k in KS:
+                if sample["true_ncm"] in codes[:k]:
+                    metrics[f"recall_at_{k}"] += 1
+            if sample["true_ncm"] in codes:
+                metrics["mrr"] += 1.0 / (codes.index(sample["true_ncm"]) + 1)
+        if total:
+            for key in metrics:
+                metrics[key] = round(metrics[key] / total, 4)
+        out[mode] = {"total": total, **metrics}
+    return out
+
+
+def main(per_segment=40, output="docs/NCM_RETRIEVAL_EVALUATION_V2.json") -> dict:
+    session = SessionLocal()
+    samples = _samples(session, per_segment)
+    corpus = EnrichedNcmCorpus.build(session)
+    index = KbNeighborIndex.build(session)
+    lexicon = derive_abbreviation_lexicon(index.corpus(), min_support=3)
+    runs = {"OFFICIAL_ONLY": [], "KB_ONLY": [], "HYBRID": []}
+    for sample in samples:
+        # leave-one-out: the queried product never retrieves itself from the KB
+        runs["OFFICIAL_ONLY"].append(hierarchical_candidates(corpus, sample["description"], k=50))
+        runs["KB_ONLY"].append(kb_neighbor_candidates(index, sample["description"], k=50, exclude_product_id=sample["product_id"]))
+        runs["HYBRID"].append(hybrid_candidates(corpus, index, sample["description"], k=50, lexicon=lexicon, exclude_product_id=sample["product_id"]))
+    overall = _recall_mrr(runs, samples)
+    by_quality = {quality: _recall_mrr({mode: [row for row, sample in zip(list_rows, samples) if sample["quality"] == quality] for mode, list_rows in runs.items()},
+                                       [sample for sample in samples if sample["quality"] == quality]) for quality in ("GOLD", "SILVER", "NOISY")}
+    by_segment = {segment: _recall_mrr({mode: [row for row, sample in zip(list_rows, samples) if sample["segment"] == segment] for mode, list_rows in runs.items()},
+                                        [sample for sample in samples if sample["segment"] == segment]) for segment in sorted({sample["segment"] for sample in samples})}
+    report = {
+        "artifact": "NCM_RETRIEVAL_EVALUATION_V2",
+        "eval_total": len(samples),
+        "quality_counts": {quality: sum(1 for sample in samples if sample["quality"] == quality) for quality in ("GOLD", "SILVER", "NOISY")},
+        "lexicon_size": len(lexicon),
+        "lexicon_sample": dict(sorted(lexicon.items())[:40]),
+        "overall": overall,
+        "by_quality": by_quality,
+        "by_segment": by_segment,
+        "llm_liberation_gate": {"gold_recall_at_20_min": 0.80, "gold_recall_at_50_preferred": 0.90},
+        "cost_usd": 0.0,
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--per-segment", type=int, default=40)
+    parser.add_argument("--output", default="docs/NCM_RETRIEVAL_EVALUATION_V2.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.per_segment, args.output), indent=2))

central/scripts/run_real_llm_ncm_eval.py

@@ -0,0 +1,220 @@
+"""Real LLM NCM evaluation over known-answer cases (mission REAL LLM EVALUATION V1).
+
+Phases: gold first (65 cases, gold completely hidden from the prompt); silver and noisy
+run ONLY after the gold quality gate passes (selection accuracy >= 0.95 among valid
+answers AND high-confidence wrong rate <= 0.02). Batching verified progressively
+(5/10/20) with 1:1 id matching; results cached permanently per
+(input_fingerprint, candidate_set_hash, prompt_version, model, dataset_versions) so an
+identical re-execution makes ZERO new calls. AI_WRITE_ALLOWED=false; nothing here can
+produce a Firebird write.
+
+If no AI credential exists the run refuses to call anything and exits with
+BLOCKED_EXTERNAL_AI_CREDENTIAL (no key is ever invented).
+"""
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+
+from central.app.ai.eval_metrics import attribution, confidence_buckets, metrics
+from central.app.ai.llm_provider import (PROMPT_VERSION_LLM, LlmAnswerError, LlmEvalItem, OpenAICompatibleProvider,
+                                         candidate_set_hash, input_fingerprint, validate_answer)
+from central.app.ai.retrieval import EnrichedNcmCorpus, KbNeighborIndex, derive_abbreviation_lexicon, hybrid_candidates
+from central.app.config import settings
+from central.app.db import SessionLocal
+from central.app.models import AiEvalCache, Dataset, NcmEntry
+
+GATE_SELECTION_ACCURACY = 0.95
+GATE_HIGH_CONF_WRONG = 0.02
+BATCH_PROBE_SIZES = (20, 10, 5)
+
+
+def _load_samples(session, per_segment=40):
+    from central.scripts.run_ncm_retrieval_eval_v2 import _samples
+    return _samples(session, per_segment)
+
+
+def _build_items(samples, corpus, index, lexicon, top_k=20):
+    items = []
+    meta = []
+    for sample in samples:
+        ranked = hybrid_candidates(corpus, index, sample["description"], k=top_k, lexicon=lexicon, exclude_product_id=sample["product_id"])
+        item = LlmEvalItem(
+            work_item_id=f"{sample['segment']}:{sample['product_id']}",
+            description=sample["description"],
+            normalized_description=sample["description"],
+            brand=None, unit=None, segment=sample["segment"],
+            candidates=ranked,
+        )
+        items.append(item)
+        meta.append({**sample, "gold_in_candidates": sample["true_ncm"] in {code for code, *_ in ranked}})
+    return items, meta
+
+
+def _cached_answers(session, items, model, ncm_sha, phase):
+    hits = {}
+    for item in items:
+        row = session.scalar(select(AiEvalCache).where(
+            AiEvalCache.input_fingerprint == input_fingerprint(item),
+            AiEvalCache.candidate_set_hash == candidate_set_hash(item),
+            AiEvalCache.prompt_version == PROMPT_VERSION_LLM,
+            AiEvalCache.model == model,
+            AiEvalCache.dataset_version_ncm == ncm_sha,
+            AiEvalCache.phase == phase,
+        ))
+        if row:
+            hits[item.work_item_id] = row.answer
+    return hits
+
+
+def _diagnose_http(status_code: int) -> str:
+    return {401: "credential invalid (AI_API_KEY)", 403: "permission/plan does not allow this endpoint or model", 404: "endpoint or model not found (AI_BASE_URL/AI_MODEL)", 429: "quota or rate limit"}.get(status_code, f"http {status_code}")
+
+
+def _select_model(available: list, requested: str) -> tuple:
+    """Prefer the operator-requested model; else a current GLM suitable for classification.
+    Never invents a name and never switches vendor — returns (model, source)."""
+    if requested in available:
+        return requested, "requested"
+    glm = [model for model in available if model.lower().startswith("glm")]
+    for candidate in glm:
+        if "flash" in candidate.lower():
+            return candidate, "fallback_glm_flash"
+    if glm:
+        return sorted(glm)[-1], "fallback_glm_latest"
+    return None, "no_glm_available"
+
+
+def _preflight(provider: OpenAICompatibleProvider, requested_model: str) -> dict:
+    """Cheap connectivity check (GET /models — no completion tokens). Validates auth,
+    base URL and model availability; diagnoses 401/403/404/429 autonomously."""
+    import httpx
+    try:
+        available = provider.list_models()
+    except httpx.HTTPStatusError as error:
+        return {"zai_connection": "FAIL", "diagnosis": _diagnose_http(error.response.status_code), "model_available": False}
+    except httpx.HTTPError as error:
+        return {"zai_connection": "FAIL", "diagnosis": f"transport: {type(error).__name__}", "model_available": False}
+    model, source = _select_model(available, requested_model)
+    return {"zai_connection": "PASS", "model_available": model is not None, "model": model, "model_source": source,
+            "available_models": available if model is None else None, "base_url": provider.base_url}
+
+
+def _probe_batch_size(provider, items):
+    """Try the largest batch that keeps valid JSON with 1:1 ids on a small probe slice."""
+    for size in BATCH_PROBE_SIZES:
+        probe = items[:size]
+        try:
+            provider.select_batch(probe)
+            return size
+        except (LlmAnswerError, json.JSONDecodeError):
+            continue
+    return 1
+
+
+def run_phase(provider_factory, phase, samples, corpus, index, lexicon, session, ncm_sha, valid_ncm):
+    items, meta = _build_items(samples, corpus, index, lexicon)
+    cached = _cached_answers(session, items, provider_factory.model if hasattr(provider_factory, "model") else settings.ai_model, ncm_sha, phase)
+    todo = [item for item in items if item.work_item_id not in cached]
+    answers = dict(cached)
+    provider = None
+    batch_size = 0
+    if todo:
+        provider = provider_factory()
+        batch_size = _probe_batch_size(provider, todo)
+        for start in range(0, len(todo), batch_size):
+            for answer in provider.select_batch(todo[start:start + batch_size]):
+                answers[answer["work_item_id"]] = answer
+    rows, errors = [], []
+    for item, sample in zip(items, meta):
+        answer = answers.get(item.work_item_id)
+        invalid = False
+        if answer is None:
+            invalid = True
+            answer = {"selected_ncm": None, "confidence": 0.0, "abstain": True, "reason_short": "missing"}
+        else:
+            try:
+                validate_answer(answer, item)
+                if answer["selected_ncm"] is not None and answer["selected_ncm"] not in valid_ncm:
+                    invalid = True
+            except LlmAnswerError:
+                invalid = True
+        selected = None if (invalid or answer["abstain"] or answer["selected_ncm"] is None) else answer["selected_ncm"]
+        rows.append({"gold_in_candidates": sample["gold_in_candidates"], "selected": selected, "gold": sample["true_ncm"], "confidence": answer["confidence"], "invalid": invalid})
+        if selected is not None and selected != sample["true_ncm"]:
+            errors.append({"work_item_id": item.work_item_id, "gold": sample["true_ncm"], "selected": selected, "confidence": answer["confidence"], "gold_in_candidates": sample["gold_in_candidates"], "reason_short": answer.get("reason_short", "")})
+        if invalid or answer.get("abstain") or selected is not None:
+            if item.work_item_id not in cached:
+                session.add(AiEvalCache(input_fingerprint=input_fingerprint(item), candidate_set_hash=candidate_set_hash(item), prompt_version=PROMPT_VERSION_LLM,
+                                        model=provider.model if provider else settings.ai_model, dataset_version_ncm=ncm_sha, phase=phase,
+                                        answer=answer, gold_ncm=sample["true_ncm"]))
+    session.commit()
+    return rows, errors, provider, batch_size, len(cached)
+
+
+def main(output_dir="/tmp", per_segment=40) -> dict:
+    if not settings.ai_api_key:
+        print(json.dumps({"status": "BLOCKED_EXTERNAL_AI_CREDENTIAL",
+                          "operator_action": "set AI_API_KEY (and optionally AI_BASE_URL/AI_MODEL) in the Hub environment (.env); never commit or log it"}, indent=2))
+        sys.exit(2)
+    session = SessionLocal()
+    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+    valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all())
+    corpus = EnrichedNcmCorpus.build(session)
+    index = KbNeighborIndex.build(session)
+    lexicon = derive_abbreviation_lexicon(index.corpus(), min_support=3)
+    samples = _load_samples(session, per_segment)
+    # ---- preflight: cheap /models call, never spends completion tokens ----
+    pre_provider = OpenAICompatibleProvider(api_key=settings.ai_api_key, model=settings.ai_model, base_url=settings.ai_base_url)
+    preflight = _preflight(pre_provider, settings.ai_model)
+    if preflight["zai_connection"] != "PASS" or not preflight["model_available"]:
+        print(json.dumps({"status": "ZAI_PREFLIGHT_FAIL", "preflight": preflight}, indent=2))
+        sys.exit(3)
+    model_name = preflight["model"]
+    factory = lambda: OpenAICompatibleProvider(api_key=settings.ai_api_key, model=model_name, base_url=settings.ai_base_url)
+    gold = [sample for sample in samples if sample["quality"] == "GOLD"]
+    provider = factory()
+    rows, errors, provider_used, batch_size, cache_hits = run_phase(lambda: provider, "gold", gold, corpus, index, lexicon, session, dataset.sha256, valid_ncm)
+    m = metrics(rows)
+    prices_configured = settings.ai_price_in_per_m is not None and settings.ai_price_out_per_m is not None
+    usage = provider.usage if provider is not None else {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
+    cost_total = round(usage["input_tokens"] / 1e6 * settings.ai_price_in_per_m + usage["output_tokens"] / 1e6 * settings.ai_price_out_per_m, 6) if prices_configured else "UNKNOWN"
+    report = {
+        "artifact": "REAL_LLM_NCM_EVALUATION_V1", "phase": "gold",
+        "provider": "z.ai", "base_url": settings.ai_base_url, "model": model_name, "prompt_version": PROMPT_VERSION_LLM,
+        "preflight": preflight,
+        "metrics": m, "attribution": attribution(rows), "confidence_buckets": confidence_buckets(rows),
+        "batch_size_selected": batch_size,
+        "gate": {"selection_accuracy_min": GATE_SELECTION_ACCURACY, "high_conf_wrong_max": GATE_HIGH_CONF_WRONG,
+                 "passed": (m["SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE"] or 0) >= GATE_SELECTION_ACCURACY and (m["HIGH_CONFIDENCE_WRONG_RATE"] or 0) <= GATE_HIGH_CONF_WRONG},
+        "cache_hits_first_run": cache_hits, "ai_write_allowed": False,
+        "usage": {**usage, "http_calls": provider.http_calls, "retries": provider.retries,
+                  "average_items_per_call": round(len(rows) / max(provider.http_calls, 1), 2),
+                  "tokens_per_item": round((usage["input_tokens"] + usage["output_tokens"]) / max(len(rows), 1), 1),
+                  "cost_total": cost_total, "cost_per_item": (round(cost_total / max(len(rows), 1), 6) if prices_configured else "UNKNOWN"),
+                  "prices_configured": prices_configured},
+    }
+    Path(f"{output_dir}/REAL_LLM_NCM_EVALUATION_V1.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    Path(f"{output_dir}/REAL_LLM_NCM_ERRORS_V1.json").write_text(json.dumps(errors, indent=2, ensure_ascii=False), encoding="utf-8")
+    # cache proof: identical re-execution must make zero new calls
+    provider2 = factory()
+    rows2, _, _, _, cache_hits2 = run_phase(lambda: provider2, "gold", gold, corpus, index, lexicon, session, dataset.sha256, valid_ncm)
+    second_calls = provider2.http_calls
+    report["cache_proof"] = {"second_run_calls": second_calls, "second_run_cache_hits": cache_hits2,
+                             "cache_hit_reexecution": second_calls == 0, "identical_metrics": metrics(rows2)["CORRECT"] == m["CORRECT"]}
+    Path(f"{output_dir}/REAL_LLM_NCM_EVALUATION_V1.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    Path(f"{output_dir}/REAL_LLM_COST_V1.json").write_text(json.dumps({"artifact": "REAL_LLM_COST_V1", "usage": report["usage"], "previous_estimate_usd": 0.44, "cache_proof": report["cache_proof"]}, indent=2), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output-dir", default="/tmp")
+    parser.add_argument("--per-segment", type=int, default=40)
+    args = parser.parse_args()
+    print(json.dumps(main(args.output_dir, args.per_segment), indent=2))

central/scripts/seed_product_kb.py

@@ -0,0 +1,158 @@
+"""Seed the Product Knowledge Base deterministically from the 5 Firebird databases.
+
+Two phases (Firebird is only reachable from Windows; Postgres only inside the compose network):
+
+  export (Windows, READ-ONLY SELECT against Firebird):
+      python scripts/seed_product_kb.py export --output docs/runs/ai_kb_seed_payload.json
+  load (inside the fiscal-hub container, against Postgres):
+      python -m central.scripts.seed_product_kb load --input docs/runs/ai_kb_seed_payload.json
+
+Export computes, per physical product row: the stable global fingerprint
+(description|unit; GTIN validated by checksum), and emits observations. Load upserts the
+GLOBAL product, records per-tenant observations, seeds knowledge already proven by the
+gate (validated_ncm when the declared NCM is active-official; ERP-declared values stay
+ERP_OBSERVATION evidence), and links TENANT_PRODUCT_CONTEXT rows. Tenant context is
+never promoted to global knowledge.
+"""
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+DATABASES = [
+    ("TESTESOFICIAIS", None),  # path from FIREBIRD_DATABASE_PATH
+    ("CONSTRUCAO", "Banco/CONSTRUCAO.FDB"),
+    ("MMMODAS", "Banco/MMMODAS.FDB"),
+    ("RINCAO", "Banco/RINCAO.FDB"),
+    ("UNIAO", "Banco/UNIAO.FDB"),
+]
+
+QUERY = "SELECT p.CODIGO, p.PRODUTO, p.UNIDADE, p.CODBARRA, p.GTIN, p.CLASSIFICACAO_FISCAL, p.CEST FROM C000025 p"
+
+
+def _canonical_ncm(value) -> str:
+    digits = "".join(char for char in str(value or "") if char.isdigit())
+    return digits if len(digits) == 8 else ""
+
+
+def export_payload(limit_per_db: int = 0, output: str = "docs/runs/ai_kb_seed_payload.json") -> dict:
+    import fdb
+    from dotenv import dotenv_values
+    from central.app.ai.fingerprint import normalize_description, product_fingerprint, validate_gtin
+    env = dotenv_values(".env")
+    stats = {"databases": [], "rows_read": 0, "firebird_writes": 0}
+    products = {}
+    for label, relative in DATABASES:
+        path = env["FIREBIRD_DATABASE_PATH"] if relative is None else str(Path(relative).resolve())
+        connection = fdb.connect(database=path, user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"))
+        try:
+            cursor = connection.cursor(); cursor.execute(QUERY)
+            columns = [column[0].strip().upper() for column in cursor.description]
+            rows = cursor.fetchmany(limit_per_db) if limit_per_db else cursor.fetchall()
+        finally:
+            connection.close()
+        # collapse the physical duplicate CODIGO rows into observations of the same product
+        seen_codes = set()
+        for row in rows:
+            record = dict(zip(columns, row))
+            code = str(record.get("CODIGO") or "").strip()
+            dedupe = f"{label}:{code}"
+            if dedupe in seen_codes:  # duplicate physical key (e.g. MMMODAS 017421) -> one observation
+                continue
+            seen_codes.add(dedupe)
+            unit = normalize_description(record.get("UNIDADE")) or None
+            fingerprint = product_fingerprint(record.get("PRODUTO"), None, unit)
+            gtin = validate_gtin(record.get("GTIN")) or validate_gtin(record.get("CODBARRA"))
+            entry = products.setdefault(fingerprint, {
+                "fingerprint": fingerprint,
+                "gtin": gtin,
+                "normalized_description": normalize_description(record.get("PRODUTO")),
+                "unit": unit,
+                "ncm": _canonical_ncm(record.get("CLASSIFICACAO_FISCAL")),
+                "cest": str(record.get("CEST") or "").strip() or None,
+                "observations": [],
+            })
+            if gtin and not entry["gtin"]:
+                entry["gtin"] = gtin
+            entry["observations"].append({"database": label, "company_key": label, "erp_code": code})
+        stats["databases"].append({"database": label, "rows": len(rows)})
+        stats["rows_read"] += len(rows)
+    payload = {"artifact": "AI_KB_SEED_PAYLOAD", "exported_at": datetime.now(timezone.utc).isoformat(), "stats": stats, "products": list(products.values())}
+    Path(output).parent.mkdir(parents=True, exist_ok=True)
+    Path(output).write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
+    stats["unique_products"] = len(products)
+    stats["payload_path"] = output
+    return stats
+
+
+def load_payload(input_path: str) -> dict:
+    from sqlalchemy import select
+    from central.app.ai.kb import KnowledgeBase
+    from central.app.db import SessionLocal
+    from central.app.models import Dataset, NcmEntry, Tenant, TenantProductContext
+    payload = json.loads(Path(input_path).read_text(encoding="utf8"))
+    session = SessionLocal()
+    kb = KnowledgeBase(session)
+    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+    active_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all()) if dataset else set()
+    tenant_ids = {}
+    stats = {"products": 0, "observations": 0, "validated_ncm": 0, "erp_ncm_observations": 0, "erp_cest_observations": 0, "tenant_contexts": 0, "firebird_writes": 0}
+    for product in payload["products"]:
+        row = kb.upsert_product(fingerprint=product["fingerprint"], gtin=product.get("gtin"), normalized_description=product.get("normalized_description", ""), unit=product.get("unit"))
+        stats["products"] += 1
+        for observation in product["observations"]:
+            label = observation["database"]
+            if label not in tenant_ids:
+                tenant = session.scalar(select(Tenant).where(Tenant.name == label))
+                if tenant is None:
+                    tenant = Tenant(name=label); session.add(tenant); session.flush()
+                tenant_ids[label] = tenant.id
+            kb.record_observation(row.id, tenant_id=tenant_ids[label], company_key=label, erp_code=observation["erp_code"], database_label=label, observed_fields={"ncm": product.get("ncm") or None, "cest": product.get("cest")})
+            stats["observations"] += 1
+            existing = session.scalar(select(TenantProductContext).where(TenantProductContext.tenant_id == tenant_ids[label], TenantProductContext.company_key == label, TenantProductContext.erp_code == observation["erp_code"]))
+            if existing is None:
+                session.add(TenantProductContext(tenant_id=tenant_ids[label], company_key=label, erp_code=observation["erp_code"], kb_product_id=row.id))
+                stats["tenant_contexts"] += 1
+        ncm = product.get("ncm") or ""
+        if ncm and ncm in active_ncm:
+            kb.add_knowledge(row.id, "validated_ncm", ncm, source="OFFICIAL_DATASET", confidence=1.0, dataset_versions={"ncm": dataset.sha256})
+            stats["validated_ncm"] += 1
+        elif ncm:
+            kb.add_knowledge(row.id, "candidate_ncm", ncm, source="ERP_OBSERVATION", confidence=0.3, evidence={"declared_in": [observation["database"] for observation in product["observations"]]})
+            stats["erp_ncm_observations"] += 1
+        if product.get("cest"):
+            kb.add_knowledge(row.id, "candidate_cest", product["cest"], source="ERP_OBSERVATION", confidence=0.3, evidence={"declared_in": [observation["database"] for observation in product["observations"]]})
+            stats["erp_cest_observations"] += 1
+        if stats["products"] % 2000 == 0:
+            session.commit()
+    session.commit()
+    stats.update(kb.stats())
+    session.close()
+    return stats
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser()
+    sub = parser.add_subparsers(dest="phase", required=True)
+    export_parser = sub.add_parser("export")
+    export_parser.add_argument("--limit", type=int, default=0)
+    export_parser.add_argument("--output", default="docs/runs/ai_kb_seed_payload.json")
+    load_parser = sub.add_parser("load")
+    load_parser.add_argument("--input", default="docs/runs/ai_kb_seed_payload.json")
+    load_parser.add_argument("--output", default="docs/runs/ai_kb_seed.json")
+    args = parser.parse_args(argv)
+    if args.phase == "export":
+        result = export_payload(args.limit, args.output)
+    else:
+        result = load_payload(args.input)
+        result["loaded_at"] = datetime.now(timezone.utc).isoformat()
+        Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.output).write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
+    print(json.dumps(result, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

central/tests/test_ai_semantic.py

@@ -0,0 +1,171 @@
+"""AI semantic layer V1: fingerprint, KB, dedup, provider contract, budget guard, candidates."""
+import json
+
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+
+from central.app.models import Base, Dataset, NcmEntry, CestEntry, CestNcmEntry, TipiEntry
+
+
+def _engine():
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    return engine
+
+
+def _ncm_dataset(session, entries):
+    dataset = Dataset(kind="NCM", provider="Siscomex", sha256="a" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=len(entries))
+    session.add(dataset); session.flush()
+    for code, description in entries:
+        session.add(NcmEntry(dataset_id=dataset.id, code=code, description=description, valid_from="", valid_to=""))
+    session.flush()
+    return dataset
+
+
+# ---------------- fingerprint / identity ----------------
+
+def test_fingerprint_is_stable_and_ignores_unstable_context():
+    from central.app.ai.fingerprint import normalize_description, product_fingerprint, validate_gtin
+    a = product_fingerprint("Coca-Cola 2LT", "COCA-COLA", "UN", "2")
+    b = product_fingerprint("coca cola 2lt  ", "coca-cola", "un", "2.0")
+    assert a == b
+    # fiscal/company context never participates in global identity
+    c = product_fingerprint("Coca-Cola 2LT", "COCA-COLA", "UN", "2")
+    assert a == c
+    assert normalize_description("Água Mineral 500ml") == "AGUA MINERAL 500ML"
+    assert validate_gtin("7891234567895") == "7891234567895"   # valid GTIN-13
+    assert validate_gtin("7891234567890") is None              # bad checksum
+    assert validate_gtin("123") is None
+
+
+# ---------------- knowledge base ----------------
+
+def test_kb_exact_fingerprint_hit_and_conflict_without_majority():
+    from central.app.ai.kb import KnowledgeBase, EXACT, FINGERPRINT
+    engine = _engine()
+    with Session(engine) as session:
+        kb = KnowledgeBase(session)
+        p1 = kb.upsert_product(fingerprint="f1", gtin="7891234567895", normalized_description="AGUA MINERAL 500ML")
+        # same GTIN arrives with a different fingerprint -> same product, exact hit
+        p2, hit = kb.resolve(gtin="7891234567895", fingerprint="other")
+        assert p2.id == p1.id and hit == EXACT
+        # same fingerprint, no GTIN -> fingerprint hit
+        p3, hit = kb.resolve(gtin=None, fingerprint="f2")
+        assert p3 is None and hit is None
+        p4 = kb.upsert_product(fingerprint="f2", normalized_description="OUTRO")
+        _, hit = kb.resolve(gtin=None, fingerprint="f2")
+        assert hit == FINGERPRINT
+        kb.add_knowledge(p1.id, "candidate_ncm", "22011000", source="OFFICIAL_DATASET", confidence=1.0)
+        kb.add_knowledge(p1.id, "candidate_ncm", "22019000", source="OFFICIAL_DATASET", confidence=1.0)
+        current = kb.current_knowledge(p1.id)
+        assert current["candidate_ncm"]["conflict"] is True          # no majority auto-pick
+        assert current["candidate_ncm"]["values"] == {"22011000", "22019000"}
+        stats = kb.stats()
+        assert stats["products"] == 2 and stats["gtin_products"] == 1 and stats["conflicts"] == 1
+
+
+def test_kb_aggregates_observations_across_tenants():
+    from central.app.ai.kb import KnowledgeBase
+    engine = _engine()
+    with Session(engine) as session:
+        kb = KnowledgeBase(session)
+        p = kb.upsert_product(fingerprint="f1", normalized_description="REFRIGERANTE 2L")
+        kb.record_observation(p.id, tenant_id=1, company_key="MMMODAS", erp_code="000001", database_label="MMMODAS", observed_fields={"ncm": "22021000"})
+        kb.record_observation(p.id, tenant_id=2, company_key="UNIAO", erp_code="000009", database_label="UNIAO", observed_fields={"ncm": "22021000"})
+        agg = kb.observation_summary(p.id)
+        assert agg["observation_count"] == 2 and agg["tenant_count"] == 2
+        assert agg["agreement"]["ncm"] == {"22021000": 2} and agg["conflicts"] == {}
+
+
+# ---------------- work item dedup ----------------
+
+def test_work_items_dedup_by_knowledge_needed():
+    from central.app.ai.workitems import build_work_items
+    questions = [
+        {"QUESTION_ID": "Q1", "DATABASE": "MMMODAS", "COMPANY": "MMMODAS", "SCOPE": "NCM", "GROUP": "", "NCM": "33059000", "KEY": "ex_tipi", "CELLS": 10},
+        {"QUESTION_ID": "Q2", "DATABASE": "UNIAO", "COMPANY": "UNIAO", "SCOPE": "NCM", "GROUP": "", "NCM": "33059000", "KEY": "ex_tipi", "CELLS": 5},
+        {"QUESTION_ID": "Q3", "DATABASE": "RINCAO", "COMPANY": "RINCAO", "SCOPE": "GROUP", "GROUP": "000015", "KEY": "product_role", "CELLS": 40},
+        {"QUESTION_ID": "Q4", "DATABASE": "CONSTRUCAO", "COMPANY": "CONSTRUCAO", "SCOPE": "PRODUCT", "GROUP": "", "NCM": "", "KEY": "ncm", "CELLS": 8, "QUESTION": "produto PRODUCT 000054", "DATABASE_CODE": "000054"},
+    ]
+    items = build_work_items(questions, product_fingerprints={"CONSTRUCAO:000054": "fp-a"})
+    # 4 raw questions -> 3 deduped work items (two ex_tipi questions for NCM 33059000 collapse)
+    by_key = {i.dedup_key: i for i in items}
+    assert len(items) == 3
+    assert by_key["ex_tipi:33059000"].cells_covered == 15
+    assert by_key["ex_tipi:33059000"].question_refs == ["Q1", "Q2"]
+    assert by_key["product_role:RINCAO:000015"].cells_covered == 40
+    assert by_key["invalid_ncm:fp-a"].cells_covered == 8
+    # AI-ineligible keys never become work items
+    assert all(i.task_type in ("INVALID_NCM", "CEST", "EX_TIPI", "PRODUCT_ROLE") for i in items)
+
+
+# ---------------- provider contract / structured output / budget ----------------
+
+def test_structured_output_schema_and_baseline_provider():
+    from central.app.ai.provider import BaselineProvider, result_from_dict, SemanticOutputError
+    provider = BaselineProvider()
+    item = type("W", (), {"task_type": "INVALID_NCM", "dedup_key": "invalid_ncm:fp-a", "payload": {"description": "AGUA MINERAL NATURAL 500ML"}})()
+    candidates = [("22011000", "AGUAS MINERAIS NATURAIS", 0.75), ("22019000", "AGUAS ... OUTRAS", 0.25)]
+    result = provider.select(item, candidates, dataset_versions={"ncm": "sha-ncm"})
+    assert result.task_type == "INVALID_NCM" and result.selected_candidate == "22011000"
+    assert result.candidate_ids_seen == ["22011000", "22019000"]
+    assert result.model == "baseline-deterministic-v1" and result.prompt_version == "ai-semantic-v1"
+    assert result.dataset_versions and result.engine_version
+    as_dict = result.to_dict()
+    assert result_from_dict(as_dict).selected_candidate == "22011000"
+    # schema fail-closed: a provider answer outside the candidate list is invalid
+    try:
+        result_from_dict({**as_dict, "selected_candidate": "99999999"})
+        raise AssertionError("must reject")
+    except SemanticOutputError:
+        pass
+    # no candidates at all -> NONE, never invented
+    none_result = provider.select(item, [])
+    assert none_result.selected_candidate == "NONE"
+
+
+def test_budget_guard_blocks_run_over_limit():
+    from central.app.ai.provider import BudgetGuard
+    guard = BudgetGuard(max_run_cost_usd=1.0, price_in_per_m=1.0, price_out_per_m=2.0)
+    plan = guard.estimate(items_in_tokens=600_000, items_out_tokens=300_000)
+    assert plan["estimated_cost"] > 0
+    assert guard.authorize(plan) is False           # (600k*1 + 300k*2)/1M = 1.2 > 1.0
+    assert guard.authorize(guard.estimate(100_000, 50_000)) is True
+
+
+# ---------------- deterministic candidates ----------------
+
+def test_ncm_top_k_and_ex_tipi_candidates():
+    from central.app.ai.candidates import ncm_top_k, ex_tipi_candidates_for_ncm, cest_candidates_for_ncm
+    engine = _engine()
+    with Session(engine) as session:
+        _ncm_dataset(session, [("22011000", "AGUAS MINERAIS NATURAIS"), ("22019000", "AGUAS MINERAIS GASEIFICADAS OUTRAS"), ("08043000", "ABACAXIS FRESCOS")])
+        top = ncm_top_k(session, "AGUA MINERAL NATURAL", k=2)
+        assert [c[0] for c in top] == ["22011000", "22019000"]
+        dataset = Dataset(kind="TIPI_IPI", provider="RFB", sha256="b" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=2)
+        session.add(dataset); session.flush()
+        session.add(TipiEntry(dataset_id=dataset.id, ncm="33059000", ex=None, description="outros", ipi_rate=None, ipi_rate_text="NT", source_row=1))
+        session.add(TipiEntry(dataset_id=dataset.id, ncm="33059000", ex="01", description="preparacoes para alisar", ipi_rate=None, ipi_rate_text="NT", source_row=2))
+        cest_dataset = Dataset(kind="CEST", provider="CONFAZ", sha256="c" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=1)
+        session.add(cest_dataset); session.flush()
+        cest = CestEntry(dataset_id=cest_dataset.id, code="03.999.99", description="x", annex="II")
+        session.add(cest); session.flush()
+        session.add(CestNcmEntry(cest_entry_id=cest.id, ncm_code="22011000"))
+        session.commit()
+        assert [e.ex for e in ex_tipi_candidates_for_ncm(session, "33059000")] == ["01"]
+        assert [c.code for c in cest_candidates_for_ncm(session, "22011000")] == ["03.999.99"]
+
+
+def test_kb_persist_roundtrip_models():
+    from central.app.models import KbProduct, KbKnowledge, KbObservation, TenantProductContext, AiWorkItem, AiRun
+    engine = _engine()
+    with Session(engine) as session:
+        p = KbProduct(fingerprint="f", normalized_description="D")
+        session.add(p); session.flush()
+        session.add(KbKnowledge(kb_product_id=p.id, knowledge_key="candidate_ncm", value="22011000", source="OFFICIAL_DATASET", confidence=1.0))
+        session.add(KbObservation(kb_product_id=p.id, tenant_id=1, company_key="C", erp_code="1", database_label="C", observed_fields={}))
+        session.add(TenantProductContext(tenant_id=1, company_key="C", erp_code="1", kb_product_id=p.id))
+        session.add(AiWorkItem(task_type="CEST", dedup_key="cest:1", payload={}, question_refs=[], cells_covered=1))
+        session.add(AiRun(provider="baseline", model="m", prompt_version="v1", input_tokens=1, output_tokens=1, cost_usd=0.0, status="DONE", results=[]))
+        session.commit()
+        assert session.query(KbProduct).count() == 1 and session.query(AiWorkItem).count() == 1

central/tests/test_field_contract.py

@@ -121,6 +121,18 @@ def test_cfop_entrada_needs_company_entry_scope_then_derives():
     assert run(p=product(CFOP_ENTRADA="1102"))["CFOP_ENTRADA"]["state"] == ALREADY
 
 
+def test_cfop_entrada_mixed_scope_is_operation_dependent_and_never_invents_a_cfop():
+    mixed = [{"scope": "COMPANY", "scope_key": "", "key": "entry_scope", "value": "MIXED_OR_DYNAMIC"}]
+    r = run(p=product(CFOP_ENTRADA=None), ctx=mixed)
+    e = r["CFOP_ENTRADA"]
+    assert e["state"] == MANUAL and e["proposed"] is None and e.get("input_key") is None
+    assert e["reason"] == "CFOP_ENTRADA_OPERATION_DEPENDENT_MIXED_SCOPE"
+    r2 = run(p=product(CFOP_ENTRADA="2102"), ctx=mixed)
+    assert r2["CFOP_ENTRADA"]["state"] == ALREADY and r2["CFOP_ENTRADA"]["reason"] == "DECLARED_VALID_OPERATION_DEPENDENT_MIXED_SCOPE"
+    from central.app.context_keys import validate_entry
+    assert validate_entry({"scope": "COMPANY", "key": "entry_scope", "value": "MIXED_OR_DYNAMIC"})[0]["value"] == "MIXED_OR_DYNAMIC"
+
+
 def test_icms_code_fields_follow_the_regime():
     r1 = run()
     assert r1["CST"]["state"] == NA and r1["CST_ENTRADA"]["state"] == NA and r1["CSOSN"]["state"] == ALREADY

central/tests/test_llm_eval.py

@@ -0,0 +1,130 @@
+"""Real LLM provider contract: strict answers, batching safety, cache, metrics attribution."""
+import json
+
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+
+from central.app.models import Base, AiEvalCache
+
+
+def _item(work_item_id="w1", candidates=None):
+    from central.app.ai.llm_provider import LlmEvalItem
+    return LlmEvalItem(work_item_id=work_item_id, description="AGUA MINERAL 500ML", normalized_description="AGUA MINERAL 500ML",
+                       brand=None, unit="ML", segment="mercado", candidates=candidates or [("22011000", "AGUAS MINERAIS", 0.9), ("22019000", "AGUAS OUTRAS", 0.4)])
+
+
+def _answer(work_item_id="w1", ncm="22011000", confidence=0.9, abstain=False):
+    return {"work_item_id": work_item_id, "selected_ncm": ncm, "confidence": confidence, "abstain": abstain, "reason_short": "desc match"}
+
+
+def test_validate_answer_accepts_only_candidate_or_abstain():
+    from central.app.ai.llm_provider import LlmAnswerError, validate_answer
+    item = _item()
+    assert validate_answer(_answer(), item)["selected_ncm"] == "22011000"
+    abstain = validate_answer(_answer(ncm=None, abstain=True), item)
+    assert abstain["abstain"] is True and abstain["selected_ncm"] is None
+    for bad in (_answer(ncm="99999999"), _answer(work_item_id="other"), _answer(ncm=None), _answer(confidence=1.5), _answer(ncm="22011000", abstain=True)):
+        try:
+            validate_answer(bad, item)
+            raise AssertionError("must reject")
+        except LlmAnswerError:
+            pass
+
+
+def test_provider_batch_requires_1_to_1_answers():
+    from central.app.ai.llm_provider import LlmAnswerError, OpenAICompatibleProvider
+
+    def transport(payload):
+        content = json.dumps([_answer(), _answer()])  # two answers for one item
+        return {"choices": [{"message": {"content": content}}], "usage": {"prompt_tokens": 10, "completion_tokens": 5}}
+
+    provider = OpenAICompatibleProvider(api_key="test-key", model="m", transport=transport)
+    try:
+        provider.select_batch([_item()])
+        raise AssertionError("must reject")
+    except LlmAnswerError:
+        pass
+    assert provider.http_calls == 1  # contract violations are NOT retried
+
+
+def test_provider_retries_transport_failures_and_counts_usage():
+    from central.app.ai.llm_provider import OpenAICompatibleProvider
+    calls = {"n": 0}
+
+    def transport(payload):
+        calls["n"] += 1
+        if calls["n"] == 1:
+            raise ConnectionError("boom")
+        content = json.dumps([_answer()])
+        return {"choices": [{"message": {"content": content}}], "usage": {"prompt_tokens": 100, "completion_tokens": 20, "prompt_tokens_details": {"cached_tokens": 50}}}
+
+    provider = OpenAICompatibleProvider(api_key="k", model="m", transport=transport)
+    answers = provider.select_batch([_item()])
+    assert answers[0]["selected_ncm"] == "22011000"
+    assert provider.http_calls == 2 and provider.retries == 1
+    assert provider.usage == {"input_tokens": 100, "output_tokens": 20, "cached_tokens": 50}
+
+
+def test_prompt_hides_gold_and_carries_candidates():
+    from central.app.ai.llm_provider import candidate_set_hash, input_fingerprint, item_prompt_payload
+    item = _item()
+    payload = item_prompt_payload(item)
+    assert "22011000" in json.dumps(payload)                      # candidates present
+    assert item.normalized_description in json.dumps(payload)
+    assert len(candidate_set_hash(item)) == 64 and len(input_fingerprint(item)) == 64
+    changed = _item(candidates=[("11111111", "X", 0.1)])
+    assert candidate_set_hash(changed) != candidate_set_hash(item)
+
+
+def test_eval_cache_roundtrip_deduplicates():
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        row = AiEvalCache(input_fingerprint="f", candidate_set_hash="c", prompt_version="p", model="m", dataset_version_ncm="d", phase="gold", answer={"selected_ncm": "22011000"}, gold_ncm="22011000")
+        session.add(row); session.commit()
+        existing = session.scalar(select(AiEvalCache).where(AiEvalCache.input_fingerprint == "f", AiEvalCache.candidate_set_hash == "c", AiEvalCache.prompt_version == "p", AiEvalCache.model == "m", AiEvalCache.dataset_version_ncm == "d"))
+        assert existing.answer["selected_ncm"] == "22011000"
+
+
+def test_metrics_and_error_attribution():
+    from central.app.ai.eval_metrics import attribution, confidence_buckets, metrics
+    rows = [
+        {"gold_in_candidates": True, "selected": "A", "gold": "A", "confidence": 0.99, "invalid": False},
+        {"gold_in_candidates": True, "selected": "B", "gold": "A", "confidence": 0.97, "invalid": False},
+        {"gold_in_candidates": True, "selected": None, "gold": "A", "confidence": 0.5, "invalid": False},
+        {"gold_in_candidates": False, "selected": "B", "gold": "A", "confidence": 0.9, "invalid": False},
+        {"gold_in_candidates": True, "selected": "zzz", "gold": "A", "confidence": 0.9, "invalid": True},
+    ]
+    m = metrics(rows)
+    assert m["TOTAL"] == 5 and m["CORRECT"] == 1 and m["WRONG"] == 2 and m["ABSTAIN"] == 1 and m["INVALID"] == 1
+    assert m["GOLD_IN_CANDIDATES"] == 4
+    assert abs(m["SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE"] - 1 / 3) < 1e-4   # correct / (correct+wrong) among valid answers
+    assert abs(m["ABSTENTION_RATE"] - 0.2) < 1e-9
+    a = attribution(rows)
+    assert a["GOLD_NOT_IN_CANDIDATES"] == 1 and a["GOLD_IN_CANDIDATES_BUT_WRONG"] == 1 and a["GOLD_IN_CANDIDATES_BUT_ABSTAIN"] == 1
+    buckets = confidence_buckets(rows)
+    assert buckets[">=0.99"] == {"total": 1, "correct": 1} and buckets["0.95-0.9899"] == {"total": 1, "correct": 0}
+
+
+def test_zai_model_selection_rules():
+    import importlib.util
+    spec = importlib.util.spec_from_file_location("eval_runner", "central/scripts/run_real_llm_ncm_eval.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    available = ["glm-5.3", "glm-5.3-flash", "glm-4.7", "other-vendor-x"]
+    assert module._select_model(available, "glm-5.3") == ("glm-5.3", "requested")
+    # requested missing -> current GLM flash fallback, never another vendor, never invented
+    assert module._select_model(available, "glm-9.9") == ("glm-5.3-flash", "fallback_glm_flash")
+    only_generic = ["glm-4", "glm-4.7"]
+    assert module._select_model(only_generic, "glm-9.9") == ("glm-4.7", "fallback_glm_latest")
+    assert module._select_model(["gpt-x", "claude-y"], "glm-5.3") == (None, "no_glm_available")
+    assert module._diagnose_http(401) == "credential invalid (AI_API_KEY)"
+
+
+def test_budget_guard_unknown_cost_limited_run():
+    from central.app.ai.provider import BudgetGuard
+    guard = BudgetGuard(max_run_cost_usd=5.0, price_in_per_m=None, price_out_per_m=None)
+    plan = guard.estimate(1000, 100)
+    assert plan["estimated_cost"] is None and plan["cost_unknown"] is True
+    assert guard.authorize(plan) is False                       # residual runs refused without prices
+    assert guard.authorize(plan, allow_unknown_cost=True) is True  # explicitly limited GOLD benchmark may opt in

central/tests/test_retrieval_v2.py

@@ -0,0 +1,116 @@
+"""Semantic candidate retrieval V2: hierarchical official corpus + KB-neighbor bridge + hybrid ranking."""
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from central.app.models import Base, Dataset, KbKnowledge, KbObservation, KbProduct, NcmEntry, Tenant
+
+
+def _engine():
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    return engine
+
+
+def _seed(session):
+    ncm = Dataset(kind="NCM", provider="Siscomex", sha256="n" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=7)
+    session.add(ncm); session.flush()
+    rows = [
+        ("22", "BEBIDAS, LIQUIDOS ALCOOLICOS E VINAGRES"),
+        ("2202", "AGUAS, INCLUIDAS AS AGUAS GASEIFICADAS E MINERAIS"),
+        ("2202.10", "AGUAS MINERAIS E GASEIFICADAS"),
+        ("22021000", "AGUAS MINERAIS NAO GASEIFICADAS"),
+        ("61", "VESTUARIO E SEUS ACESSORIOS, DE MALHA"),
+        ("6105", "CAMISAS DE MALHA, DE USO MASCULINO"),
+        ("61051000", "DE ALGODAO"),
+    ]
+    for code, description in rows:
+        session.add(NcmEntry(dataset_id=ncm.id, code=code.replace(".", ""), description=description, valid_from="", valid_to=""))
+    session.flush()
+    # two tenants already selling mineral water with a validated NCM
+    tenant_a, tenant_b = Tenant(name="A"), Tenant(name="B")
+    session.add_all([tenant_a, tenant_b]); session.flush()
+    p1 = KbProduct(fingerprint="f1", normalized_description="AGUA MINERAL SEM GAS 500ML")
+    p2 = KbProduct(fingerprint="f2", normalized_description="AGUA MINERAL GARRAFA 1L")
+    p3 = KbProduct(fingerprint="f3", normalized_description="CAMISA MASCULINA MALHA ALGODAO")
+    session.add_all([p1, p2, p3]); session.flush()
+    for product, ncm_value, tenant in ((p1, "22021000", tenant_a), (p2, "22021000", tenant_b), (p3, "61051000", tenant_a)):
+        session.add(KbKnowledge(kb_product_id=product.id, knowledge_key="validated_ncm", value=ncm_value, source="OFFICIAL_DATASET", confidence=1.0))
+    session.add(KbObservation(kb_product_id=p1.id, tenant_id=tenant_a.id, company_key="A", erp_code="1", database_label="A", observed_fields={}))
+    session.add(KbObservation(kb_product_id=p2.id, tenant_id=tenant_b.id, company_key="B", erp_code="2", database_label="B", observed_fields={}))
+    session.add(KbObservation(kb_product_id=p3.id, tenant_id=tenant_a.id, company_key="A", erp_code="3", database_label="A", observed_fields={}))
+    session.commit()
+    return ncm.sha256
+
+
+def test_enriched_corpus_carries_ancestor_context():
+    from central.app.ai.retrieval import EnrichedNcmCorpus
+    engine = _engine()
+    with Session(engine) as session:
+        _seed(session)
+        corpus = EnrichedNcmCorpus.build(session)
+        doc = corpus.document("22021000")
+        # chapter + position + subposition context enriches the very short legal text (stemmed)
+        assert "BEBIDA" in doc and "AGUA" in doc
+        assert "VESTUARIO" not in doc
+
+
+def test_hierarchical_retrieval_prefers_right_positions():
+    from central.app.ai.retrieval import EnrichedNcmCorpus, hierarchical_candidates
+    engine = _engine()
+    with Session(engine) as session:
+        _seed(session)
+        corpus = EnrichedNcmCorpus.build(session)
+        codes = [code for code, _, _ in hierarchical_candidates(corpus, "AGUA MINERAL", k=2)]
+        assert codes[0] == "22021000"
+
+
+def test_kb_neighbor_bridge_maps_commercial_to_ncm():
+    from central.app.ai.retrieval import KbNeighborIndex, kb_neighbor_candidates
+    engine = _engine()
+    with Session(engine) as session:
+        _seed(session)
+        index = KbNeighborIndex.build(session)
+        neighbors = kb_neighbor_candidates(index, "AGUA MINERAL PET 500ML", k=5)
+        assert neighbors and neighbors[0][0] == "22021000"
+        # cross-tenant agreement is reported, never auto-majority
+        assert neighbors[0][3] >= 2
+
+
+def test_hybrid_merges_official_and_kb_rankings():
+    from central.app.ai.retrieval import EnrichedNcmCorpus, KbNeighborIndex, hybrid_candidates
+    engine = _engine()
+    with Session(engine) as session:
+        _seed(session)
+        corpus = EnrichedNcmCorpus.build(session)
+        index = KbNeighborIndex.build(session)
+        hybrid = hybrid_candidates(corpus, index, "CAMISA MASCULINA ALGODAO", k=2)
+        assert hybrid[0][0] == "61051000"
+        scores = hybrid_candidates(corpus, index, "AGUA MINERAL SEM GAS", k=2)
+        assert scores[0][0] == "22021000"
+
+
+def test_lexicon_expands_abbreviations_supported_by_kb():
+    from central.app.ai.retrieval import derive_abbreviation_lexicon, expand_stems
+    corpus = {
+        "REFRIGERANTE COCA 2L": "22021000",
+        "REFRI GUARANA 1L": "22021000",
+        "REFRI LARANJA 350ML": "22021000",
+        "CAMISA POLO ALGODAO": "61051000",
+        "CAM POLO MASC": "61051000",
+        "CAM POLO AZUL": "61051000",
+        "BOTA COURO MASC": "64039900",
+    }
+    lexicon = derive_abbreviation_lexicon(corpus, min_support=2)
+    assert lexicon.get("REFRI") == "REFRIGERANTE"
+    assert lexicon.get("CAM") == "CAMISA"
+    expanded = expand_stems({"REFRI", "COLA"}, lexicon)
+    assert "REFRIGERANTE" in expanded
+
+
+def test_commercial_tokens_strip_units_and_sizes():
+    from central.app.ai.retrieval import commercial_tokens
+    tokens = commercial_tokens("CREME TRATAMENTO NOVEX 1KG CX 12 COR AZUL TAM M REF 453")
+    assert "NOVEX" in tokens and "CREME" in tokens and "TRATAMENTO" in tokens
+    assert "1KG" not in tokens and "CX" not in tokens and "COR" not in tokens and "TAM" not in tokens
+    assert "AZUL" not in tokens and "REF" not in tokens and "453" not in tokens
+    assert "12" not in tokens

docker-compose.yml

@@ -32,6 +32,12 @@ services:
       LOG_LEVEL: ${LOG_LEVEL:-INFO}
       BASE_URL: ${BASE_URL:-http://localhost:8000}
       AGENT_ONLINE_THRESHOLD_SECONDS: ${AGENT_ONLINE_THRESHOLD_SECONDS:-120}
+      # Z.ai (OpenAI-compatible) — key only from the local .env, never committed/logged.
+      AI_API_KEY: ${AI_API_KEY:-}
+      AI_BASE_URL: ${AI_BASE_URL:-https://api.z.ai/api/paas/v4}
+      AI_MODEL: ${AI_MODEL:-glm-5.3}
+      AI_PRICE_IN_PER_M: ${AI_PRICE_IN_PER_M:-}
+      AI_PRICE_OUT_PER_M: ${AI_PRICE_OUT_PER_M:-}
     volumes:
       # Read-only access to locally-held official source documents (PDFs/XLSX
       # with no stable public direct-download URL) for the one-off dataset

docs/AI_COST_PLAN_V1.json

@@ -0,0 +1,40 @@
+{
+  "artifact": "AI_COST_PLAN_V1",
+  "unique_ai_work_items": 730,
+  "per_task_type": {
+    "EX_TIPI": {
+      "items": 112,
+      "input_tokens": 56000,
+      "output_tokens": 8960,
+      "cells_covered": 4130
+    },
+    "CEST": {
+      "items": 353,
+      "input_tokens": 176500,
+      "output_tokens": 28240,
+      "cells_covered": 2045
+    },
+    "PRODUCT_ROLE": {
+      "items": 41,
+      "input_tokens": 16400,
+      "output_tokens": 2460,
+      "cells_covered": 482
+    },
+    "INVALID_NCM": {
+      "items": 224,
+      "input_tokens": 156800,
+      "output_tokens": 26880,
+      "cells_covered": 1794
+    }
+  },
+  "estimated_input_tokens": 405700,
+  "estimated_output_tokens": 66540,
+  "estimated_cost_usd": 0.53878,
+  "ai_max_run_cost_usd": 5.0,
+  "budget_authorized": true,
+  "provider_configured": "baseline",
+  "model_configured": "baseline-deterministic-v1",
+  "ai_enabled": false,
+  "max_batch_size": 20,
+  "min_confidence": 0.6
+}

docs/AI_EVALUATION_V1.json

@@ -0,0 +1,66 @@
+{
+  "artifact": "AI_EVALUATION_V1",
+  "task": "INVALID_NCM candidate selection",
+  "pipeline": "candidate_generation -> provider_selection -> validator",
+  "provider": "baseline",
+  "model": "baseline-deterministic-v1",
+  "prompt_version": "ai-semantic-v1",
+  "dataset_versions": {
+    "ncm": "3abc0d2eb2501bbd0cc3f5e0ee3774d33f091ea305e8127ee020e4009878987e"
+  },
+  "k": 10,
+  "eval_total": 200,
+  "metrics": {
+    "EXACT_MATCH": 2,
+    "WRONG_SELECTION": 164,
+    "NONE_WHEN_KNOWN": 34,
+    "INVALID_OUTPUT": 0,
+    "RECALL_AT_K": 11
+  },
+  "accuracy_exact_match": 0.01,
+  "recall_at_k": 0.055,
+  "by_segment": {
+    "base_testes": {
+      "EXACT_MATCH": 0,
+      "WRONG_SELECTION": 27,
+      "NONE_WHEN_KNOWN": 13,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 2,
+      "TOTAL": 40
+    },
+    "construcao": {
+      "EXACT_MATCH": 1,
+      "WRONG_SELECTION": 39,
+      "NONE_WHEN_KNOWN": 0,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 4,
+      "TOTAL": 40
+    },
+    "mercado": {
+      "EXACT_MATCH": 1,
+      "WRONG_SELECTION": 35,
+      "NONE_WHEN_KNOWN": 4,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 2,
+      "TOTAL": 40
+    },
+    "moda": {
+      "EXACT_MATCH": 0,
+      "WRONG_SELECTION": 35,
+      "NONE_WHEN_KNOWN": 5,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 0,
+      "TOTAL": 40
+    },
+    "restaurante": {
+      "EXACT_MATCH": 0,
+      "WRONG_SELECTION": 28,
+      "NONE_WHEN_KNOWN": 12,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 3,
+      "TOTAL": 40
+    }
+  },
+  "cost_usd": 0.0,
+  "note": "baseline provider (deterministic, zero-cost): measures the pipeline floor; rerun with a real provider once credentials exist"
+}

docs/AI_PROJECT_STATE.md

@@ -1,6 +1,55 @@
 # AI Project State
 
-## Current status
+## Current status (real LLM evaluation V1 — 2026-09-21)
+
+`BLOCKED_EXTERNAL_AI_CREDENTIAL` (implementation complete). The entire real-LLM
+evaluation pipeline is built and tested offline: OpenAI-compatible provider
+(`central/app/ai/llm_provider.py`, key only from Hub env, injectable transport),
+strict per-item validator (candidate-list confinement, abstain semantics, 1:1
+work_item_id, calibrated confidence in [0,1]), progressive batch probing (20/10/5),
+post-AI validator (active-dataset membership), metrics + A/B/C error attribution +
+confidence-bucket calibration (`central/app/ai/eval_metrics.py`), permanent result
+cache (`ai_eval_cache`, migration 0014) with zero-call re-execution proof, and the
+runner `central/scripts/run_real_llm_ncm_eval.py` (gold-first; silver/noisy only
+after the gate: selection accuracy >= 0.95 AND high-confidence wrong rate <= 0.02;
+artifacts REAL_LLM_NCM_EVALUATION_V1/_ERRORS_/_COST_). The runner refuses to call
+anything without `AI_API_KEY` and exits BLOCKED_EXTERNAL_AI_CREDENTIAL with the exact
+operator action. Zero LLM calls, zero Firebird writes, AI_WRITE_ALLOWED=false.
+Operator action: set AI_API_KEY (optionally AI_BASE_URL/AI_MODEL) in the Hub .env.
+
+## Semantic candidate retrieval V2 (2026-09-21)
+
+`NCM_RETRIEVAL_V2_GATE_PASSED`. Deterministic hybrid retrieval (hierarchical official
+corpus enriched with ancestor descriptions + KB commercial->NCM neighbor bridge +
+KB-derived abbreviation lexicon, 644 entries; weights 0.3/0.7; leave-one-out discipline
+after catching self-retrieval leakage) lifted candidate recall from 7.5%@50 (V1) to
+HYBRID 92%@20 / 93.5%@50, GOLD 95.4%@20/@50, MRR 0.84 — meeting the LLM liberation gate
+(GOLD@20 ≥ 0.80, @50 ≥ 0.90). Benchmark quality audit: GOLD 65 / SILVER 118 / NOISY 17.
+GTIN audit: 8,559 valid, 251 cross-tenant matches, 90 GTIN-conflicting NCMs surfaced.
+Reprocessing the 750 work items: 4 KB_EXACT + 16 KB_FINGERPRINT + 1 high-confidence
+retrieval + 143 single-official-candidate (CEST/EX) resolved deterministically; 511
+items (4,181 cells) still need LLM selection; projected cost $0.54 → $0.44. Zero LLM
+calls, zero Firebird writes, AI_WRITE_ALLOWED remains false. Artifacts:
+docs/NCM_RETRIEVAL_EVALUATION_V2.json, docs/NCM_RETRIEVAL_ARCHITECTURE_V2.md,
+docs/GTIN_KB_AUDIT_V1.json, docs/AI_RESIDUAL_PLAN_V2.json.
+
+## AI semantic layer V1 (2026-09-21)
+
+`AI_SEMANTIC_LAYER_V1_IMPLEMENTED`. The deterministic gate was closed
+(OFFICIAL_RULE_RESOLVABLE_REMAINING=0, ERP_DATA_RESOLVABLE_REMAINING=0) and the Hub now
+owns a Product Knowledge Base plus a deduplicated AI work-item queue. entry_scope now
+admits MIXED_OR_DYNAMIC (CFOP_ENTRADA becomes operation-dependent; no CFOP invented).
+KB seeded read-only from all 5 databases: 30,801 global products (8,559 GTIN), 35,065
+observations, 163 surfaced knowledge conflicts, idempotent load. 850 residual questions
+deduped into 750 work items (20 answered by KB cache alone). Budget-guarded cost plan:
+$0.54 estimated for 730 items (authorized, $0 spent — no AI credential exists here).
+The 200-sample NCM benchmark proved the honest floor: candidate recall@10 = 5.5%
+(ERP commercial wording vs official legal wording), so AI selection over closed
+candidates is capped there — READY_FOR_AI_ASSISTED_WRITE_V2=false with candidate
+generation as the named prerequisite. AI_WRITE_ALLOWED=false in V1: AI results stay
+AI_CANDIDATE_VALIDATED, never Firebird writes. Artifacts:
+docs/PRODUCT_KNOWLEDGE_BASE_V1.md, docs/AI_SEMANTIC_LAYER_V1.md,
+docs/AI_EVALUATION_V1.json, docs/AI_COST_PLAN_V1.json, docs/AI_RESIDUAL_PLAN_V1.json.
 
 `FISCAL_DATA_FOUNDATION_MOSTLY_COMPLETE`. Deploy/pilot work remains paused
 per the current mission's scope. The operator manually placed the 6 official

docs/AI_RESIDUAL_PLAN_V1.json

@@ -0,0 +1,32 @@
+{
+  "artifact": "AI_RESIDUAL_PLAN_V1",
+  "raw_residual_questions": 1000,
+  "ai_work_items_raw": 850,
+  "ai_work_items_deduped": 750,
+  "by_task_type": {
+    "EX_TIPI": {
+      "items": 112,
+      "cells": 4130
+    },
+    "CEST": {
+      "items": 353,
+      "cells": 2045
+    },
+    "PRODUCT_ROLE": {
+      "items": 41,
+      "cells": 482
+    },
+    "INVALID_NCM": {
+      "items": 244,
+      "cells": 1970
+    }
+  },
+  "persisted": {
+    "created": 750,
+    "updated": 0,
+    "kb_hits": 20,
+    "conflicts": 0
+  },
+  "ai_write_allowed": false,
+  "company_decisions_remaining": 14
+}

docs/AI_RESIDUAL_PLAN_V2.json

@@ -0,0 +1,19 @@
+{
+  "artifact": "AI_RESIDUAL_PLAN_V2",
+  "work_items": 750,
+  "classification": {
+    "KB_EXACT_HIT": 4,
+    "KB_FINGERPRINT_HIT": 16,
+    "HIGH_CONFIDENCE_RETRIEVAL": 1,
+    "SINGLE_OFFICIAL_CANDIDATE": 143,
+    "LLM_SELECTION_REQUIRED": 511,
+    "UNRESOLVED": 75
+  },
+  "llm_selection_required_items": 511,
+  "llm_selection_required_cells": 4181,
+  "high_confidence_threshold": 0.75,
+  "note": "HIGH_CONFIDENCE_RETRIEVAL is a candidate, not a fiscal resolution: the official validator + operator gate still apply. AI_WRITE_ALLOWED=false.",
+  "dataset_versions": {
+    "ncm": "3abc0d2eb2501bbd0cc3f5e0ee3774d33f091ea305e8127ee020e4009878987e"
+  }
+}

docs/AI_SEMANTIC_LAYER_V1.md

@@ -0,0 +1,63 @@
+# AI SEMANTIC LAYER V1
+
+Central Hub module `central/app/ai/` + tables `ai_work_items` / `ai_runs`.
+The AI is NOT a fiscal authority, never writes Firebird, never replaces official
+datasets, and never answers company-level questions.
+
+## Contract
+
+- **Cache first**: KB exact/fingerprint hits never trigger AI (20 of 750 work items
+  were answered by the KB alone in the build pass).
+- **Dedup**: pendings are grouped by knowledge needed, not per cell — 850 residual
+  questions collapsed into 750 work items (EX_TIPI 112, CEST 353, PRODUCT_ROLE 41,
+  INVALID_NCM 244).
+- **Closed candidates only**: for every task the Hub first builds the official
+  candidate list (NCM top-K, CEST links for the NCM, TIPI EX rows). A provider may
+  only answer with a candidate code or `NONE`; `result_from_dict` rejects anything
+  else (`SemanticOutputError`). Providers never see an open "what is the NCM?" question.
+- **Structured output** (strict JSON schema): task_type, input_fingerprint,
+  selected_candidate, confidence, reason, candidate_ids_seen, model, prompt_version,
+  created_at, dataset_versions, engine_version.
+- **Provenance**: every persisted AI result is `source=AI` (or
+  `AI_PLUS_OFFICIAL_VALIDATION` after passing the official validator), with model,
+  prompt_version, dataset_versions — versioned history, never overwritten.
+- **Forbidden for AI**: ST retention (ST_WITHHELD), origin (AI_ORIGIN_AUTO_APPLY=false,
+  at most NEEDS_EVIDENCE), company profile questions (PIS/COFINS profile, entry_scope,
+  ICMS profile, store_ipi_profile, CRT, UF), CFOP invention, majority conflict resolution.
+- **Budget guard**: `BudgetGuard` refuses execution above `AI_MAX_RUN_COST_USD`;
+  no fallback to a more expensive model. Credentials live only in the Hub
+  (`AI_PROVIDER`, `AI_MODEL`, `AI_MAX_BATCH_SIZE`, `AI_MAX_RUN_COST_USD`,
+  `AI_MIN_CONFIDENCE`); the Windows Agent never sees them.
+- **AI_WRITE_ALLOWED=false in V1**: validated AI results stay
+  `AI_CANDIDATE_VALIDATED`; no Firebird write is generated from AI output.
+
+## Provider abstraction
+
+`AISemanticProvider` (select / estimate). Shipped: `BaselineProvider`
+(deterministic token-overlap top-1, zero cost) used by the evaluation harness and as
+the mandatory no-credential fallback. Real providers plug in without touching the
+fiscal engine.
+
+## Evaluation result (docs/AI_EVALUATION_V1.json, 200 samples, 5 segments)
+
+Baseline floor: EXACT_MATCH 2/200 (1.0%), NONE_WHEN_KNOWN 34, WRONG_SELECTION 164,
+INVALID_OUTPUT 0, cost $0. RECALL@10=5.5%, RECALL@50=7.5%
+(docs/runs/ai_evaluation_k50.json).
+
+**Root-cause finding**: ERP commercial descriptions ("CREME DE TRATAMENTO NOVEX
+QUERATINA 1KG") share almost no vocabulary with official legal NCM descriptions
+("Outros preparações para uso dos cabelos..."), so the deterministic candidate
+generator cannot surface the true NCM in top-K — the bottleneck is candidate
+recall, not provider selection. Since AI selection is confined to the candidate
+list, any provider would be capped near this floor.
+
+**Consequence**: READY_FOR_AI_ASSISTED_WRITE_V2=false. V2 prerequisite: semantic
+candidate generation (e.g. AI-proposed chapter/heading validated against the official
+NCM tree by prefix — still closed-world), then re-run this benchmark until accuracy
+and cost are jointly healthy.
+
+## Cost plan (docs/AI_COST_PLAN_V1.json)
+
+730 unique AI work items (20 KB hits excluded), 405,700 input / 66,540 output tokens,
+estimated $0.54 against a $5.00 budget — authorized. Actual spend $0 (baseline only;
+no AI credential exists in this environment).

docs/GTIN_KB_AUDIT_V1.json

@@ -0,0 +1,81 @@
+{
+  "artifact": "GTIN_KB_AUDIT_V1",
+  "gtin_valid": 8559,
+  "gtin_invalid_raw": 0,
+  "gtin_duplicated_same_product": 256,
+  "gtin_conflicting_products": 90,
+  "gtin_cross_tenant_matches": 251,
+  "conflict_examples": [
+    {
+      "gtin": "7908429102622",
+      "ncms": [
+        "40169990",
+        "87141000"
+      ]
+    },
+    {
+      "gtin": "7898324238536",
+      "ncms": [
+        "85393100",
+        "85393111"
+      ]
+    },
+    {
+      "gtin": "7898324002410",
+      "ncms": [
+        "85393100",
+        "85393111"
+      ]
+    },
+    {
+      "gtin": "4047025341110",
+      "ncms": [
+        "27101932",
+        "38190000"
+      ]
+    },
+    {
+      "gtin": "4047025341127",
+      "ncms": [
+        "27101932",
+        "38190000"
+      ]
+    },
+    {
+      "gtin": "7898227950016",
+      "ncms": [
+        "27101932",
+        "28070010"
+      ]
+    },
+    {
+      "gtin": "7891117007128",
+      "ncms": [
+        "82015000",
+        "82130000"
+      ]
+    },
+    {
+      "gtin": "7891117097921",
+      "ncms": [
+        "82015000",
+        "82130000"
+      ]
+    },
+    {
+      "gtin": "7899882310993",
+      "ncms": [
+        "38101010",
+        "85094020"
+      ]
+    },
+    {
+      "gtin": "7891114063523",
+      "ncms": [
+        "82041100",
+        "95051000"
+      ]
+    }
+  ],
+  "policy": "same GTIN across tenants is STRONG evidence for sharing global product knowledge; tenant fiscal context is never shared"
+}

docs/NCM_RETRIEVAL_ARCHITECTURE_V2.md

@@ -0,0 +1,84 @@
+# NCM RETRIEVAL ARCHITECTURE V2
+
+Deterministic semantic candidate retrieval (zero LLM) implemented in
+`central/app/ai/retrieval.py`. Goal: surface the correct official code among candidates
+BEFORE spending on LLM selection.
+
+## Sources merged into one ranking
+
+- **A. OFFICIAL hierarchical** — every 8-digit NCM document is enriched with its
+  ancestors' official descriptions (chapter 2-digit, position 4, subposition 6) from the
+  ACTIVE dataset, so short legal texts ("Outros") carry their path vocabulary. Search is
+  two-stage: rank positions, then expand their 8-digit descendants (top-15 positions).
+- **B. KB neighbor bridge** — the Knowledge Base maps COMMERCIAL descriptions to
+  VALIDATED NCMs (conflicting products excluded). Query = commercial-token similarity;
+  evidence per candidate = similarity, observation_count, union of distinct tenants
+  (agreement reported, never a majority decision).
+- **C. HYBRID** — single ranking, weights OFFICIAL 0.3 / KB 0.7 (tuned on the V2
+  benchmark; the KB bridge carries the signal, the official component breaks ties and
+  covers products absent from the KB).
+
+## Commercial normalization (section 7)
+
+`commercial_tokens()`: NFKD/ascii/upper stems with Portuguese plural stemming, minus
+units (UN CX FD PCT KG G ML L LT PET LATA GARRAFA ...), sizes/codes (1KG, 500ML, CX 12,
+pure numbers), and TAM/COR/REF/COD/MOD + their value token. Impact measured on the
+benchmark, not assumed.
+
+## Abbreviation lexicon (section 8)
+
+`derive_abbreviation_lexicon()`: statistical, KB-derived — a 3-5 char token that is a
+strict prefix of a longer token, both occurring across distinct products sharing the
+same validated NCM (support ≥ 3). 644 abbreviations derived (e.g. REFRI→REFRIGERANTE,
+CAM→CAMISA). No LLM involved. Query stems are expanded before official retrieval.
+
+## Leave-one-out discipline
+
+Every KB/eval measurement excludes the queried product itself from the neighbor index
+(`exclude_product_id`); the first run without it showed fake 99.5% recall — self-retrieval
+leakage — and was corrected before any reporting.
+
+## Benchmark (docs/NCM_RETRIEVAL_EVALUATION_V2.json, 200 samples, seed 42)
+
+Quality audit: GOLD 65 (GTIN or cross-tenant agreement), SILVER 118, NOISY 17 (<2
+informative tokens; not pursued).
+
+| Mode | R@1 | R@5 | R@10 | R@20 | R@50 | MRR |
+|---|---|---|---|---|---|---|
+| OFFICIAL_ONLY (hierarchical+enriched) | .015 | .08 | .11 | .14 | .195 | .046 |
+| KB_ONLY (neighbors, leave-one-out) | .665 | .85 | .895 | .925 | .935 | .750 |
+| HYBRID | .635 | .855 | .90 | .92 | .935 | .733 |
+
+GOLD × HYBRID: R@1 .785, R@10 .954, **R@20 .954**, **R@50 .954**, MRR .839.
+
+**Liberation gate (section 14): PASS** — GOLD_RECALL@20 = 0.954 ≥ 0.80 and
+GOLD_RECALL@50 = 0.954 ≥ 0.90. Real-LLM selection may now be evaluated on the same
+benchmark; AI write remains forbidden.
+
+## GTIN audit (docs/GTIN_KB_AUDIT_V1.json)
+
+VALID 8,559 (0 invalid — checksum-validated at ingest) · duplicated-same-product 256 ·
+cross-tenant matches 251 (strong knowledge-sharing evidence) · conflicting NCMs on the
+same GTIN 90 (surfaced as data-quality conflicts, examples in the artifact).
+
+## Why KB_EXACT_HITS was 0 (section 11)
+
+Root cause: the V1 work-item pass resolved identity by fingerprint only — the GTIN
+lookup was never attempted — and INVALID_NCM products have no validated NCM of their own,
+so hits only arise via a shared product with a valid NCM. V2 tries GTIN first
+(KB_EXACT_HIT=4) then fingerprint (KB_FINGERPRINT_HIT=16). No identity loosening.
+
+## Work-item reprocessing (docs/AI_RESIDUAL_PLAN_V2.json)
+
+750 items → 4 KB_EXACT_HIT + 16 KB_FINGERPRINT_HIT + 1 HIGH_CONFIDENCE_RETRIEVAL
+(hybrid ≥ 0.75, official-valid, cross-tenant ≥ 2 — a candidate, NOT an auto fiscal
+resolution) + 143 SINGLE_OFFICIAL_CANDIDATE (CEST/EX with exactly one official
+candidate) + 511 LLM_SELECTION_REQUIRED (4,181 cells) + 75 UNRESOLVED.
+
+Projected LLM cost drops $0.54 → $0.44 (730 → 511 calls).
+
+## CEST / EX TIPI (section 15)
+
+Reuse the same official-candidate lists (`cest_candidates_for_ncm`,
+`ex_tipi_candidates_for_ncm`); 143 items resolved to a single official candidate with no
+AI at all. Multi-candidate items wait for the LLM evaluation pass.

docs/NCM_RETRIEVAL_EVALUATION_V2.json

@@ -0,0 +1,322 @@
+{
+  "artifact": "NCM_RETRIEVAL_EVALUATION_V2",
+  "eval_total": 200,
+  "quality_counts": {
+    "GOLD": 65,
+    "SILVER": 118,
+    "NOISY": 17
+  },
+  "lexicon_size": 644,
+  "lexicon_sample": {
+    "100M": "100MT",
+    "10A": "10AM",
+    "10M": "10MT",
+    "20A": "20AM",
+    "20M": "20MT",
+    "220V": "220VX4600W",
+    "30M": "30MT",
+    "32M": "32MM",
+    "32X25": "32X25MM",
+    "40M": "40MM",
+    "4E0V": "4E0V4",
+    "4PC": "4PCA",
+    "50M": "50MT",
+    "5X100": "5X100MM",
+    "60M": "60MT",
+    "ACES": "ACESSORIO",
+    "ACHOC": "ACHOCOLATADO",
+    "ACO": "ACOPLADO",
+    "ACOPL": "ACOPLADA",
+    "ACQUA": "ACQUAMARINE",
+    "ACR": "ACRILICA",
+    "ACRIL": "ACRILICA",
+    "ADM": "ADMISSAO",
+    "ADVAN": "ADVANC",
+    "AERO": "AEROSOL",
+    "AFRIC": "AFRICANO",
+    "ALAV": "ALAVANCA",
+    "ALAVA": "ALAVANCA",
+    "ALC": "ALCOOL",
+    "ALE": "ALEJANDRA",
+    "ALF": "ALFAIATARIA",
+    "ALFA": "ALFAIATARIA",
+    "ALI": "ALINE",
+    "ALIC": "ALICATE",
+    "ALL": "ALLELA",
+    "ALMOF": "ALMOFADADO",
+    "ALUM": "ALUMASA",
+    "ALUMI": "ALUMINIO",
+    "AMEND": "AMENDOA",
+    "AMOR": "AMORI"
+  },
+  "overall": {
+    "OFFICIAL_ONLY": {
+      "total": 200,
+      "recall_at_1": 0.015,
+      "recall_at_5": 0.08,
+      "recall_at_10": 0.11,
+      "recall_at_20": 0.14,
+      "recall_at_50": 0.195,
+      "mrr": 0.0456
+    },
+    "KB_ONLY": {
+      "total": 200,
+      "recall_at_1": 0.665,
+      "recall_at_5": 0.85,
+      "recall_at_10": 0.895,
+      "recall_at_20": 0.925,
+      "recall_at_50": 0.935,
+      "mrr": 0.7499
+    },
+    "HYBRID": {
+      "total": 200,
+      "recall_at_1": 0.635,
+      "recall_at_5": 0.855,
+      "recall_at_10": 0.9,
+      "recall_at_20": 0.92,
+      "recall_at_50": 0.935,
+      "mrr": 0.7325
+    }
+  },
+  "by_quality": {
+    "GOLD": {
+      "OFFICIAL_ONLY": {
+        "total": 65,
+        "recall_at_1": 0.0308,
+        "recall_at_5": 0.1077,
+        "recall_at_10": 0.1231,
+        "recall_at_20": 0.1538,
+        "recall_at_50": 0.1692,
+        "mrr": 0.0681
+      },
+      "KB_ONLY": {
+        "total": 65,
+        "recall_at_1": 0.7692,
+        "recall_at_5": 0.9077,
+        "recall_at_10": 0.9385,
+        "recall_at_20": 0.9538,
+        "recall_at_50": 0.9538,
+        "mrr": 0.8276
+      },
+      "HYBRID": {
+        "total": 65,
+        "recall_at_1": 0.7846,
+        "recall_at_5": 0.9231,
+        "recall_at_10": 0.9538,
+        "recall_at_20": 0.9538,
+        "recall_at_50": 0.9538,
+        "mrr": 0.8392
+      }
+    },
+    "SILVER": {
+      "OFFICIAL_ONLY": {
+        "total": 118,
+        "recall_at_1": 0.0085,
+        "recall_at_5": 0.0763,
+        "recall_at_10": 0.1186,
+        "recall_at_20": 0.1525,
+        "recall_at_50": 0.2373,
+        "mrr": 0.0398
+      },
+      "KB_ONLY": {
+        "total": 118,
+        "recall_at_1": 0.661,
+        "recall_at_5": 0.8814,
+        "recall_at_10": 0.9237,
+        "recall_at_20": 0.9661,
+        "recall_at_50": 0.9831,
+        "mrr": 0.7656
+      },
+      "HYBRID": {
+        "total": 118,
+        "recall_at_1": 0.6271,
+        "recall_at_5": 0.8898,
+        "recall_at_10": 0.9322,
+        "recall_at_20": 0.9576,
+        "recall_at_50": 0.9831,
+        "mrr": 0.7471
+      }
+    },
+    "NOISY": {
+      "OFFICIAL_ONLY": {
+        "total": 17,
+        "recall_at_1": 0.0,
+        "recall_at_5": 0.0,
+        "recall_at_10": 0.0,
+        "recall_at_20": 0.0,
+        "recall_at_50": 0.0,
+        "mrr": 0.0
+      },
+      "KB_ONLY": {
+        "total": 17,
+        "recall_at_1": 0.2941,
+        "recall_at_5": 0.4118,
+        "recall_at_10": 0.5294,
+        "recall_at_20": 0.5294,
+        "recall_at_50": 0.5294,
+        "mrr": 0.3442
+      },
+      "HYBRID": {
+        "total": 17,
+        "recall_at_1": 0.1176,
+        "recall_at_5": 0.3529,
+        "recall_at_10": 0.4706,
+        "recall_at_20": 0.5294,
+        "recall_at_50": 0.5294,
+        "mrr": 0.2235
+      }
+    }
+  },
+  "by_segment": {
+    "base_testes": {
+      "OFFICIAL_ONLY": {
+        "total": 40,
+        "recall_at_1": 0.0,
+        "recall_at_5": 0.05,
+        "recall_at_10": 0.075,
+        "recall_at_20": 0.075,
+        "recall_at_50": 0.1,
+        "mrr": 0.0254
+      },
+      "KB_ONLY": {
+        "total": 40,
+        "recall_at_1": 0.5,
+        "recall_at_5": 0.75,
+        "recall_at_10": 0.825,
+        "recall_at_20": 0.85,
+        "recall_at_50": 0.85,
+        "mrr": 0.6171
+      },
+      "HYBRID": {
+        "total": 40,
+        "recall_at_1": 0.375,
+        "recall_at_5": 0.7,
+        "recall_at_10": 0.775,
+        "recall_at_20": 0.85,
+        "recall_at_50": 0.85,
+        "mrr": 0.5297
+      }
+    },
+    "construcao": {
+      "OFFICIAL_ONLY": {
+        "total": 40,
+        "recall_at_1": 0.025,
+        "recall_at_5": 0.075,
+        "recall_at_10": 0.1,
+        "recall_at_20": 0.175,
+        "recall_at_50": 0.2,
+        "mrr": 0.0578
+      },
+      "KB_ONLY": {
+        "total": 40,
+        "recall_at_1": 0.875,
+        "recall_at_5": 0.95,
+        "recall_at_10": 0.95,
+        "recall_at_20": 0.975,
+        "recall_at_50": 0.975,
+        "mrr": 0.9106
+      },
+      "HYBRID": {
+        "total": 40,
+        "recall_at_1": 0.85,
+        "recall_at_5": 0.975,
+        "recall_at_10": 0.975,
+        "recall_at_20": 0.975,
+        "recall_at_50": 0.975,
+        "mrr": 0.9008
+      }
+    },
+    "mercado": {
+      "OFFICIAL_ONLY": {
+        "total": 40,
+        "recall_at_1": 0.025,
+        "recall_at_5": 0.125,
+        "recall_at_10": 0.15,
+        "recall_at_20": 0.15,
+        "recall_at_50": 0.175,
+        "mrr": 0.0637
+      },
+      "KB_ONLY": {
+        "total": 40,
+        "recall_at_1": 0.825,
+        "recall_at_5": 0.95,
+        "recall_at_10": 0.975,
+        "recall_at_20": 0.975,
+        "recall_at_50": 0.975,
+        "mrr": 0.8807
+      },
+      "HYBRID": {
+        "total": 40,
+        "recall_at_1": 0.85,
+        "recall_at_5": 0.95,
+        "recall_at_10": 0.975,
+        "recall_at_20": 0.975,
+        "recall_at_50": 0.975,
+        "mrr": 0.8932
+      }
+    },
+    "moda": {
+      "OFFICIAL_ONLY": {
+        "total": 40,
+        "recall_at_1": 0.025,
+        "recall_at_5": 0.15,
+        "recall_at_10": 0.225,
+        "recall_at_20": 0.25,
+        "recall_at_50": 0.425,
+        "mrr": 0.0773
+      },
+      "KB_ONLY": {
+        "total": 40,
+        "recall_at_1": 0.425,
+        "recall_at_5": 0.725,
+        "recall_at_10": 0.85,
+        "recall_at_20": 0.95,
+        "recall_at_50": 1.0,
+        "mrr": 0.5758
+      },
+      "HYBRID": {
+        "total": 40,
+        "recall_at_1": 0.475,
+        "recall_at_5": 0.775,
+        "recall_at_10": 0.9,
+        "recall_at_20": 0.925,
+        "recall_at_50": 1.0,
+        "mrr": 0.6194
+      }
+    },
+    "restaurante": {
+      "OFFICIAL_ONLY": {
+        "total": 40,
+        "recall_at_1": 0.0,
+        "recall_at_5": 0.0,
+        "recall_at_10": 0.0,
+        "recall_at_20": 0.05,
+        "recall_at_50": 0.075,
+        "mrr": 0.0039
+      },
+      "KB_ONLY": {
+        "total": 40,
+        "recall_at_1": 0.7,
+        "recall_at_5": 0.875,
+        "recall_at_10": 0.875,
+        "recall_at_20": 0.875,
+        "recall_at_50": 0.875,
+        "mrr": 0.7654
+      },
+      "HYBRID": {
+        "total": 40,
+        "recall_at_1": 0.625,
+        "recall_at_5": 0.875,
+        "recall_at_10": 0.875,
+        "recall_at_20": 0.875,
+        "recall_at_50": 0.875,
+        "mrr": 0.7196
+      }
+    }
+  },
+  "llm_liberation_gate": {
+    "gold_recall_at_20_min": 0.8,
+    "gold_recall_at_50_preferred": 0.9
+  },
+  "cost_usd": 0.0
+}

docs/PRODUCT_KNOWLEDGE_BASE_V1.md

@@ -0,0 +1,55 @@
+# PRODUCT KNOWLEDGE BASE V1
+
+Global product knowledge for the GTI Fiscal Hub, seeded deterministically from the five
+Firebird databases (read-only) on 2026-09-21. PostgreSQL tables: `kb_products`,
+`kb_knowledge`, `kb_observations`, `tenant_product_context`.
+
+## Global vs tenant separation
+
+- `kb_products` / `kb_knowledge` — GLOBAL: only facts inherent to the product
+  (identity, validated NCM, candidate CEST/EX, semantic attributes).
+- `tenant_product_context` / `kb_observations` — TENANT: role, ST-withheld, operation
+  profile, ERP codes, per-company observations. NEVER promoted to global.
+
+## Identity
+
+Order: (1) valid GTIN (checksum-validated, exact match), (2) strong fingerprint
+sha256(normalized description | brand | unit | package quantity) — brand is not
+resolvable in this ERP generation (CODMARCA has no joinable name table), so V1 identity
+is description|unit, (3) semantic fingerprint candidate (future), (4) unknown.
+Fiscal/company context (CFOP, CSOSN, CST, rates, ST, tenant) never participates.
+
+## Knowledge provenance
+
+Every `kb_knowledge` row records `source` (OFFICIAL_DATASET | ERP_OBSERVATION | OPERATOR
+| AI | AI_PLUS_OFFICIAL_VALIDATION), confidence, evidence, model/prompt_version when
+AI-produced, dataset_versions and created_at/validated_at. History is never deleted.
+
+## Conflicts
+
+Same key with distinct values (e.g. one fingerprint declared with two different NCMs
+across tenants) is surfaced as `KNOWLEDGE_CONFLICT` (`conflicts=163` at seed). Conflicts
+are never resolved by majority and never feed another tenant silently.
+
+## Seed results (docs/runs/ai_kb_seed.json)
+
+| Metric | Value |
+|---|---|
+| Physical rows read (5 databases) | 35,066 |
+| Observations | 35,065 (1 duplicate-key row collapsed) |
+| Unique global products | 30,801 |
+| GTIN-identified products | 8,559 |
+| Fingerprint-only products | 22,242 |
+| validated_ncm (official dataset) | 30,833 knowledge rows |
+| ERP NCM observations (evidence only) | 212 products |
+| ERP CEST observations (evidence only) | 11,719 products |
+| Knowledge conflicts | 163 |
+| Firebird writes | 0 |
+
+Load is idempotent: a second run adds 0 products / 0 contexts.
+
+## Cache-first rule
+
+Any semantic task first resolves identity against the KB: EXACT (GTIN) or FINGERPRINT
+hit with non-conflicting validated knowledge answers without an AI call. Only KB misses
+or eligible conflicts reach the AI resolver.

docs/runs/ai_cost_plan_v2.json

@@ -0,0 +1,40 @@
+{
+  "artifact": "AI_COST_PLAN_V1",
+  "unique_ai_work_items": 586,
+  "per_task_type": {
+    "INVALID_NCM": {
+      "items": 223,
+      "input_tokens": 156100,
+      "output_tokens": 26760,
+      "cells_covered": 1787
+    },
+    "PRODUCT_ROLE": {
+      "items": 41,
+      "input_tokens": 16400,
+      "output_tokens": 2460,
+      "cells_covered": 482
+    },
+    "EX_TIPI": {
+      "items": 20,
+      "input_tokens": 10000,
+      "output_tokens": 1600,
+      "cells_covered": 635
+    },
+    "CEST": {
+      "items": 302,
+      "input_tokens": 151000,
+      "output_tokens": 24160,
+      "cells_covered": 1793
+    }
+  },
+  "estimated_input_tokens": 333500,
+  "estimated_output_tokens": 54980,
+  "estimated_cost_usd": 0.44346,
+  "ai_max_run_cost_usd": 5.0,
+  "budget_authorized": true,
+  "provider_configured": "baseline",
+  "model_configured": "baseline-deterministic-v1",
+  "ai_enabled": false,
+  "max_batch_size": 20,
+  "min_confidence": 0.6
+}

docs/runs/ai_evaluation_k50.json

@@ -0,0 +1,66 @@
+{
+  "artifact": "AI_EVALUATION_V1",
+  "task": "INVALID_NCM candidate selection",
+  "pipeline": "candidate_generation -> provider_selection -> validator",
+  "provider": "baseline",
+  "model": "baseline-deterministic-v1",
+  "prompt_version": "ai-semantic-v1",
+  "dataset_versions": {
+    "ncm": "3abc0d2eb2501bbd0cc3f5e0ee3774d33f091ea305e8127ee020e4009878987e"
+  },
+  "k": 50,
+  "eval_total": 200,
+  "metrics": {
+    "EXACT_MATCH": 2,
+    "WRONG_SELECTION": 164,
+    "NONE_WHEN_KNOWN": 34,
+    "INVALID_OUTPUT": 0,
+    "RECALL_AT_K": 15
+  },
+  "accuracy_exact_match": 0.01,
+  "recall_at_k": 0.075,
+  "by_segment": {
+    "base_testes": {
+      "EXACT_MATCH": 0,
+      "WRONG_SELECTION": 27,
+      "NONE_WHEN_KNOWN": 13,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 2,
+      "TOTAL": 40
+    },
+    "construcao": {
+      "EXACT_MATCH": 1,
+      "WRONG_SELECTION": 39,
+      "NONE_WHEN_KNOWN": 0,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 5,
+      "TOTAL": 40
+    },
+    "mercado": {
+      "EXACT_MATCH": 1,
+      "WRONG_SELECTION": 35,
+      "NONE_WHEN_KNOWN": 4,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 4,
+      "TOTAL": 40
+    },
+    "moda": {
+      "EXACT_MATCH": 0,
+      "WRONG_SELECTION": 35,
+      "NONE_WHEN_KNOWN": 5,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 1,
+      "TOTAL": 40
+    },
+    "restaurante": {
+      "EXACT_MATCH": 0,
+      "WRONG_SELECTION": 28,
+      "NONE_WHEN_KNOWN": 12,
+      "INVALID_OUTPUT": 0,
+      "RECALL_AT_K": 3,
+      "TOTAL": 40
+    }
+  },
+  "cost_usd": 0.0,
+  "note": "baseline provider (deterministic, zero-cost): measures the pipeline floor; rerun with a real provider once credentials exist"
+}

docs/runs/ai_kb_seed.json

@@ -0,0 +1,13 @@
+{
+  "products": 30801,
+  "observations": 35065,
+  "validated_ncm": 30833,
+  "erp_ncm_observations": 212,
+  "erp_cest_observations": 11719,
+  "tenant_contexts": 0,
+  "firebird_writes": 0,
+  "gtin_products": 8559,
+  "fingerprint_products": 22242,
+  "conflicts": 163,
+  "loaded_at": "2026-09-21T15:22:11.333546+00:00"
+}

docs/superpowers/plans/2026-09-21-pre-ai-local-completion-v3.md

@@ -0,0 +1,187 @@
+# PRE-AI Local Completion V3 Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Complete and verify the local semantic pipeline, knowledge preparation, and operator tooling without external AI or uncertain Firebird writes.
+
+**Architecture:** Extend the existing `central.app.ai` contracts with a deterministic scenario provider and an ID-correlated batch harness. Add conservative, versioned KB/retrieval preparation artifacts and CLI-first operational commands that compose existing read-only mechanisms.
+
+**Tech Stack:** Python 3.11+, SQLAlchemy, PostgreSQL/SQLite test engines, pytest, existing Alembic models, JSON/CSV artifacts, Docker only for existing local test execution.
+
+## Global Constraints
+
+- Do not call paid/external AI APIs.
+- Keep `DETERMINISTIC_GATE_CLOSED=true`.
+- Keep `AI_WRITE_ALLOWED=false` and `FIREBIRD_AI_WRITES=0`.
+- Never use fake answers as semantic decisions.
+- Never write uncertain semantic values to Firebird.
+- Preserve tenant-specific fiscal context outside global KB identity.
+- Preserve all conflicts and historical evidence.
+- Do not add arbitrary fiscal rules.
+
+---
+
+### Task 1: Fake Provider Contract
+
+**Files:**
+- Create: `central/app/ai/fake_provider.py`
+- Modify: `central/app/ai/provider.py`
+- Test: `central/tests/test_fake_provider.py`
+
+**Interfaces:**
+- `DeterministicFakeSemanticProvider.select_batch(items, scenario=None) -> list[dict]`
+- Scenarios: `CORRECT`, `WRONG`, `ABSTAIN`, `INVALID_JSON`, `MISSING_ITEM`, `DUPLICATE_ITEM`, `OUT_OF_CANDIDATE`, `TIMEOUT`, `429`, `500`.
+- The provider returns only test responses and never persists them.
+
+- [ ] Write tests asserting every scenario and deterministic call counters.
+- [ ] Run `python -m pytest central/tests/test_fake_provider.py -q` and observe failure.
+- [ ] Implement the provider using the existing `LlmEvalItem`/validator contract.
+- [ ] Ensure failures use typed transport/contract errors and do not mutate input items.
+- [ ] Run the focused tests and confirm pass.
+
+### Task 2: ID-Correlated Batch Harness
+
+**Files:**
+- Create: `central/app/ai/batch_runner.py`
+- Test: `central/tests/test_batch_runner.py`
+
+**Interfaces:**
+- `run_batches(items, provider, batch_size, max_retries=2) -> BatchRunResult`.
+- `BatchRunResult` exposes `results_by_work_item_id`, `errors`, `provider_calls`, `retries`, and `usage`.
+
+- [ ] Add failing tests for batch sizes 5, 10, and 20, shuffled responses, missing IDs, duplicates, invalid JSON, timeout, 429, 500, and partial failure.
+- [ ] Implement correlation by `work_item_id`, never array position.
+- [ ] Retry only transport failures and preserve failed item IDs.
+- [ ] Shrink/retry only according to explicit policy; never silently drop items.
+- [ ] Run focused tests.
+
+### Task 3: Budget, Cache, and Dataset Drift Tests
+
+**Files:**
+- Modify: `central/app/ai/provider.py`
+- Modify: `central/app/ai/llm_provider.py`
+- Create: `central/app/ai/cache_policy.py`
+- Test: `central/tests/test_ai_controls_v3.py`
+
+**Interfaces:**
+- `cache_key(input_fingerprint, candidate_set_hash, model, prompt_version, dataset_versions) -> str`.
+- `classify_knowledge_freshness(record_versions, active_versions) -> VALID | NEEDS_REVALIDATION | CONFLICT`.
+
+- [ ] Test sufficient and exceeded budgets, including zero provider calls on pre-call rejection.
+- [ ] Test input/output/cached token accounting and unknown prices.
+- [ ] Test cache hit/miss for each key component with `provider_calls=0` on hit.
+- [ ] Test dataset drift classification without deleting history.
+- [ ] Implement minimal helpers and connect them to existing cache/provider paths.
+- [ ] Run focused tests.
+
+### Task 4: Offline E2E Simulation
+
+**Files:**
+- Modify: `central/scripts/run_real_llm_ncm_eval.py`
+- Create: `central/scripts/run_offline_ai_simulation.py`
+- Test: `central/tests/test_offline_ai_simulation.py`
+
+**Interfaces:**
+- `run_offline_ai_simulation(output_dir, scenario, batch_sizes=(5,10,20)) -> dict`.
+- Output explicitly marks fake results as plumbing-only and `FIREBIRD_AI_WRITES=0`.
+
+- [ ] Test GOLD -> gate -> SILVER/NOISY -> residual plan sequencing.
+- [ ] Test that fake selections never enter Firebird or KB semantic truth tables.
+- [ ] Implement the command using final existing work-item/candidate contracts.
+- [ ] Emit sanitized metrics and zero-write evidence.
+- [ ] Run the command and focused tests.
+
+### Task 5: Canonical KB and Conflict Audit
+
+**Files:**
+- Modify: `central/app/models.py`
+- Create: `central/alembic/versions/0015_kb_canonical_identity.py`
+- Create: `central/app/ai/kb_consolidation.py`
+- Create: `central/scripts/audit_kb_conflicts.py`
+- Create: `central/tests/test_kb_consolidation.py`
+- Generate: `docs/KB_CONFLICT_AUDIT_V1.json`
+
+**Interfaces:**
+- `canonical_identity(observation) -> str` using GTIN first, then conservative fingerprint fields.
+- `classify_conflict(observations) -> str` using the six requested classifications.
+- `audit_conflicts(observations) -> dict` with counts, decisions, and evidence.
+
+- [ ] Test GTIN-equivalent abbreviation merges and incompatible same-GTIN conflicts.
+- [ ] Test packaging variations remain distinct or explicitly classified.
+- [ ] Add additive canonical identity without promoting tenant fiscal fields.
+- [ ] Audit all available KB conflicts and generate the JSON artifact.
+- [ ] Run focused tests and validate artifact counts.
+
+### Task 6: Retrieval V3 Evaluation and Candidate Preparation
+
+**Files:**
+- Modify: `central/app/ai/retrieval.py`
+- Create: `central/app/ai/retrieval_v3.py`
+- Create: `central/scripts/run_retrieval_eval_v3.py`
+- Create: `central/scripts/prepare_ai_candidates.py`
+- Test: `central/tests/test_retrieval_v3.py`
+- Generate: `docs/NCM_RETRIEVAL_EVALUATION_V3.json`
+
+**Interfaces:**
+- `retrieval_v3_candidates(corpus, index, description, metadata, k=20) -> list`.
+- `prepare_candidate_set(work_item, active_datasets) -> dict` with normalized product, top-20, scores, official descriptions, hierarchy, neighbors, GTIN evidence, and input fingerprint.
+
+- [ ] Add holdout tests for BM25/TF-IDF/character/rarity/hierarchy/KB/GTIN/group evidence.
+- [ ] Compare V3 to V2 and activate only if GOLD recall@20 is at least 0.954 and no required metric regresses.
+- [ ] Prepare NCM invalid, CEST, EX_TIPI, and PRODUCT_ROLE payloads without making semantic decisions.
+- [ ] Persist versioned candidate artifacts idempotently.
+- [ ] Run retrieval evaluation and focused tests.
+
+### Task 7: Payload Optimization and Final Phase-1 Rerun
+
+**Files:**
+- Create: `central/app/ai/payloads.py`
+- Modify: `central/scripts/run_offline_ai_simulation.py`
+- Test: `central/tests/test_payloads.py`
+- Generate: `docs/AI_PAYLOAD_OPTIMIZATION_V1.json`
+
+- [ ] Test compaction removes only redundant repeated evidence and preserves candidate recall.
+- [ ] Measure before/after token estimates for all 511 selection items.
+- [ ] Rerun Phase 1 simulation against final work items and candidate sets.
+- [ ] Confirm fake outputs remain non-authoritative and zero-write.
+
+### Task 8: Guided Company and Group Questionnaires
+
+**Files:**
+- Create: `scripts/answer_company_context.py`
+- Create: `scripts/answer_group_questions.py`
+- Create: `central/app/ai/questionnaire.py`
+- Test: `central/tests/test_questionnaire.py`
+
+- [ ] Test validation, review-before-import, cancel-without-write, JSON output, and CSV output.
+- [ ] Implement one-question-at-a-time Portuguese prompts with affected products/cells/options.
+- [ ] Group questions by company, group, subgroup, NCM, question, and consequence; show examples.
+- [ ] Enforce `ENTRY_SCOPE` values and document `MIXED_OR_DYNAMIC` behavior.
+- [ ] Implement `STORE_IPI_PROFILE` as explicit choice plus preview/rollback plan only.
+
+### Task 9: Rollback Tooling and Full Local Validation
+
+**Files:**
+- Modify: `scripts/rollback_field.py`
+- Create: `scripts/rollback_run.py`
+- Create: `scripts/run_local_validation.py`
+- Test: `tests/test_rollback_run.py`
+- Test: `tests/test_local_validation.py`
+
+- [ ] Test preview fields: run_id, database, field, product, before, current, target, affected rows.
+- [ ] Enforce `current == expected_after`; emit `CONFLICT` and never overwrite drift.
+- [ ] Compose preflight, sources, company context, KB, deterministic engine, residual classification, AI-ready preparation, and reports.
+- [ ] Make repeated runs idempotent and assert no semantic Firebird writes.
+
+### Task 10: Health Report and Final Verification
+
+**Files:**
+- Create: `scripts/generate_local_health.py`
+- Generate: `docs/LOCAL_PRODUCT_HEALTH_V1.json`
+- Modify: `docs/AI_PROJECT_STATE.md`
+
+- [ ] Generate all requested health fields with measured values, not placeholders.
+- [ ] Run Phase-1 E2E again using final Phase-2 work items/candidates.
+- [ ] Run `python -m pytest tests -q` and `python -m pytest central/tests -q`.
+- [ ] Run `git diff --check` and verify no API key or token appears in tracked artifacts.
+- [ ] Report remaining blockers precisely: external Z.ai balance/credential, missing source files, unresolved human decisions, and any measurable failures.

docs/superpowers/specs/2026-09-21-pre-ai-local-completion-v3-design.md

@@ -0,0 +1,74 @@
+# PRE-AI Local Completion V3 Design
+
+## Goal
+
+Finish and verify every local, deterministic part of the semantic completion
+product without external LLM calls, Firebird semantic writes, deployment, or
+new fiscal rules. When credentials become available, only the real benchmark
+execution should remain.
+
+## Scope and Order
+
+1. Offline AI plumbing: deterministic fake provider, batch correlation,
+   retries, budget enforcement, cache invalidation, and end-to-end dry-run.
+2. Knowledge base: canonical identity and conflict audit, conservative
+   deduplication, retrieval V3 evaluation, final candidate preparation, and
+   payload compaction.
+3. Operations: guided company/group questionnaires, rollback preview/guard,
+   idempotent local validation, and a concise health artifact.
+
+The phases are sequential. Phase 1 is rerun after Phase 2 with the final work
+items and candidate sets. No fake answer is persisted as semantic truth.
+
+## Safety Invariants
+
+- `DETERMINISTIC_GATE_CLOSED=true` remains unchanged.
+- `AI_WRITE_ALLOWED=false` and `FIREBIRD_AI_WRITES=0` are asserted by tests and
+  reports.
+- External HTTP/API calls are forbidden in offline commands and tests.
+- Firebird access remains read-only for discovery; no uncertain semantic value
+  is written to Firebird.
+- Tenant fiscal context is never promoted into global product identity.
+- Conflicts are retained and classified; automatic consolidation requires
+  exact structural evidence and equivalent normalized descriptions.
+- `MIXED_OR_DYNAMIC` never produces a single invented `CFOP_ENTRADA`.
+- Rollback changes only rows whose current value equals the recorded expected
+  post-write value.
+
+## Architecture
+
+The fake provider implements the existing semantic selection contract and is
+driven by named scenarios. A batch harness maps responses by `work_item_id`,
+validates candidate confinement, retries only transport failures, and records
+provider calls, token usage, and budget decisions. Cache keys include input,
+candidate set, model, prompt version, and all relevant dataset versions.
+
+KB consolidation remains additive: `canonical_product_id` is a stable global
+identity for equivalent observations, while tenant-specific values remain in
+tenant context. Conflict audit and candidate preparation are persisted as
+versioned JSON artifacts or Hub records; no fiscal decision is inferred.
+
+Operational tools are CLI-first and preview-first. They reuse the existing
+question and context persistence mechanisms, produce reviewable JSON/CSV, and
+are idempotent. The full validation command composes existing read-only
+scripts plus the new preparation/report steps.
+
+## Acceptance Criteria
+
+- Fake scenarios cover correct, wrong, abstain, invalid JSON, missing,
+  duplicate, out-of-candidate, timeout, 429, and 500 responses.
+- Batch sizes 5, 10, and 20 are tested with out-of-order responses and
+  partial/retry failures, always correlated by ID.
+- Budget overrun blocks before provider invocation; unknown prices remain
+  explicitly unknown.
+- Cache hit performs zero provider calls; each key component independently
+  causes a miss; dataset drift yields `NEEDS_REVALIDATION` or `CONFLICT`.
+- KB conflict audit contains all 163 baseline conflicts or an explicit
+  measured current count, with no silent merge.
+- Retrieval V3 never lowers GOLD recall@20 below 0.954 on the holdout and only
+  becomes active when the comparison artifact proves improvement or equal
+  recall with better recall@1/MRR.
+- Final E2E simulation completes GOLD -> gate -> SILVER/NOISY -> residual plan
+  with zero Firebird writes.
+- Full local validation is idempotent and emits `docs/LOCAL_PRODUCT_HEALTH_V1.json`.
+- Full test suite passes.

scripts/seed_product_kb.py

@@ -0,0 +1,15 @@
+"""Thin Windows delegator: Firebird is only reachable from the host.
+
+All logic lives in central/scripts/seed_product_kb.py (export = Windows/Firebird,
+load = container/Postgres). Run from the repository root:
+    python scripts/seed_product_kb.py export|load ...
+"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from central.scripts.seed_product_kb import main  # noqa: E402
+
+if __name__ == "__main__":
+    main()