chore: capture semantic pipeline baseline

central/scripts/ai_cost_plan.py

@@ -0,0 +1,69 @@
+"""Cost plan for the residual AI work items, with budget guard (section 18).
+
+Runs BEFORE any real execution: UNIQUE_AI_WORK_ITEMS, token estimates per task type,
+estimated cost, and the authorize/refuse verdict against AI_MAX_RUN_COST_USD.
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+from central.app.ai.provider import BudgetGuard
+from central.app.config import settings
+from central.app.db import SessionLocal
+from central.app.models import AiWorkItem
+
+TOKENS_PER_ITEM = {  # conservative prompt+schema overhead per single work item
+    "INVALID_NCM": (700, 120),
+    "CEST": (500, 80),
+    "EX_TIPI": (500, 80),
+    "PRODUCT_ROLE": (400, 60),
+}
+
+
+def main(output="docs/AI_COST_PLAN_V1.json") -> dict:
+    session = SessionLocal()
+    items = session.scalars(select(AiWorkItem)).all()
+    guard = BudgetGuard(settings.ai_max_run_cost_usd, settings.ai_price_in_per_m, settings.ai_price_out_per_m)
+    per_task = {}
+    in_total = out_total = 0
+    resolved = {"KB_HIT", "KB_EXACT_HIT", "KB_FINGERPRINT_HIT", "HIGH_CONFIDENCE_RETRIEVAL", "SINGLE_OFFICIAL_CANDIDATE", "DONE", "SKIPPED", "CONFLICT"}
+    for item in items:
+        if item.status in resolved:
+            continue  # cache/official candidate resolved — no AI call needed
+        tokens_in, tokens_out = TOKENS_PER_ITEM.get(item.task_type, (500, 80))
+        bucket = per_task.setdefault(item.task_type, {"items": 0, "input_tokens": 0, "output_tokens": 0, "cells_covered": 0})
+        bucket["items"] += 1; bucket["input_tokens"] += tokens_in; bucket["output_tokens"] += tokens_out
+        bucket["cells_covered"] += item.cells_covered
+        in_total += tokens_in; out_total += tokens_out
+    plan = guard.estimate(in_total, out_total)
+    report = {
+        "artifact": "AI_COST_PLAN_V1",
+        "unique_ai_work_items": sum(bucket["items"] for bucket in per_task.values()),
+        "per_task_type": per_task,
+        "estimated_input_tokens": plan["input_tokens"],
+        "estimated_output_tokens": plan["output_tokens"],
+        "estimated_cost_usd": plan["estimated_cost"],
+        "ai_max_run_cost_usd": guard.max_run_cost_usd,
+        "budget_authorized": guard.authorize(plan),
+        "provider_configured": settings.ai_provider,
+        "model_configured": settings.ai_model,
+        "ai_enabled": settings.ai_enabled,
+        "max_batch_size": settings.ai_max_batch_size,
+        "min_confidence": settings.ai_min_confidence,
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", default="/tmp/ai_cost_plan.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.output), indent=2))
+

central/scripts/audit_gtin_kb.py

@@ -0,0 +1,60 @@
+"""GTIN KB audit (section 10) — validity, cross-tenant matches and conflicts.
+
+Reads the seed payload (ground truth of what was observed per tenant) plus the KB tables.
+"""
+import json
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from central.app.ai.fingerprint import validate_gtin
+
+
+def main(payload_path="docs/runs/ai_kb_seed_payload.json", output="docs/GTIN_KB_AUDIT_V1.json") -> dict:
+    payload = json.loads(Path(payload_path).read_text(encoding="utf8"))
+    by_gtin = defaultdict(list)
+    invalid_gtin_raw = 0
+    for product in payload["products"]:
+        raw = product.get("gtin")
+        if raw is None:
+            continue
+        if validate_gtin(raw) is None:
+            invalid_gtin_raw += 1  # defensive: export already validates
+            continue
+        by_gtin[raw].append(product)
+    duplicated_same_product = sum(1 for entries in by_gtin.values() if len(entries) > 1)
+    conflicting = 0
+    cross_tenant = 0
+    conflict_examples = []
+    for gtin, entries in by_gtin.items():
+        ncms = {entry.get("ncm") for entry in entries if entry.get("ncm")}
+        databases = {obs["database"] for entry in entries for obs in entry["observations"]}
+        if len(ncms) > 1:
+            conflicting += 1
+            if len(conflict_examples) < 10:
+                conflict_examples.append({"gtin": gtin, "ncms": sorted(ncms)})
+        if len(databases) > 1:
+            cross_tenant += 1
+    report = {
+        "artifact": "GTIN_KB_AUDIT_V1",
+        "gtin_valid": len(by_gtin),
+        "gtin_invalid_raw": invalid_gtin_raw,
+        "gtin_duplicated_same_product": duplicated_same_product,
+        "gtin_conflicting_products": conflicting,
+        "gtin_cross_tenant_matches": cross_tenant,
+        "conflict_examples": conflict_examples,
+        "policy": "same GTIN across tenants is STRONG evidence for sharing global product knowledge; tenant fiscal context is never shared",
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    return report
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--payload-path", default="/tmp/ai_kb_seed_payload.json")
+    parser.add_argument("--output", default="/tmp/gtin_audit.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.payload_path, args.output), indent=2))

central/scripts/build_ai_work_items.py

@@ -0,0 +1,83 @@
+"""Build deduplicated AI work items from the residual questions and the seeded KB.
+
+Reads docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json, resolves each invalid-NCM product
+question to its KB fingerprint (same product in several databases collapses into ONE
+work item), persists AiWorkItem rows and writes docs/AI_RESIDUAL_PLAN_V1.json.
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+
+from central.app.ai.kb import KnowledgeBase
+from central.app.ai.workitems import build_work_items
+from central.app.db import SessionLocal
+from central.app.models import AiWorkItem, KbObservation
+
+
+def _erp_code_from_question(question) -> str:
+    import re
+    match = re.search(r"PRODUCT\s+(\d+)", question.get("QUESTION", ""))
+    return match.group(1) if match else ""
+
+
+def main(questions_path="docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json", output="docs/AI_RESIDUAL_PLAN_V1.json") -> dict:
+    session = SessionLocal()
+    questions = json.loads(Path(questions_path).read_text(encoding="utf8"))["questions"]
+    # map "{DATABASE}:{erp_code}" -> fingerprint using the seeded observations
+    rows = session.execute(select(KbObservation.company_key, KbObservation.erp_code, KbObservation.kb_product_id, KbObservation.database_label)).all()
+    product_ids = {f"{row.database_label}:{row.erp_code}": row.kb_product_id for row in rows}
+    questions = [dict(question, DATABASE_CODE=_erp_code_from_question(question)) for question in questions]
+    product_fingerprints = {key: f"kb:{value}" for key, value in product_ids.items()}
+    items = build_work_items(questions, product_fingerprints)
+    raw = sum(len(item.question_refs) for item in items)
+    persisted = {"created": 0, "updated": 0, "kb_hits": 0, "conflicts": 0}
+    kb = KnowledgeBase(session)
+    for item in items:
+        row = session.scalar(select(AiWorkItem).where(AiWorkItem.dedup_key == item.dedup_key))
+        if row is None:
+            row = AiWorkItem(task_type=item.task_type, dedup_key=item.dedup_key, question_refs=item.question_refs, payload=item.payload, cells_covered=item.cells_covered, status="PENDING")
+            session.add(row); persisted["created"] += 1
+        else:
+            row.question_refs = item.question_refs; row.cells_covered = item.cells_covered; row.payload = item.payload
+            persisted["updated"] += 1
+        # cache-first: an existing non-conflicting validated answer makes this a KB hit, no AI call
+        if item.dedup_key.startswith("invalid_ncm:kb:") and item.dedup_key != "invalid_ncm:kb:None":
+            kb_product_id = int(item.dedup_key.rsplit(":", 1)[1])
+            knowledge = kb.current_knowledge(kb_product_id)
+            validated = knowledge.get("validated_ncm", {})
+            if validated and not validated.get("conflict"):
+                row.status = "KB_HIT"; persisted["kb_hits"] += 1
+            elif validated.get("conflict"):
+                row.status = "CONFLICT"; persisted["conflicts"] += 1
+    session.commit()
+    by_task = {}
+    for item in items:
+        bucket = by_task.setdefault(item.task_type, {"items": 0, "cells": 0})
+        bucket["items"] += 1; bucket["cells"] += item.cells_covered
+    report = {
+        "artifact": "AI_RESIDUAL_PLAN_V1",
+        "raw_residual_questions": len(questions),
+        "ai_work_items_raw": raw,
+        "ai_work_items_deduped": len(items),
+        "by_task_type": by_task,
+        "persisted": persisted,
+        "ai_write_allowed": False,
+        "company_decisions_remaining": 14,
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--questions", default="docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json")
+    parser.add_argument("--output", default="docs/AI_RESIDUAL_PLAN_V1.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.questions, args.output), indent=2))
+

central/scripts/reprocess_work_items_v2.py

@@ -0,0 +1,112 @@
+"""Reprocess the 750 AI work items with V2 retrieval (section 16).
+
+Classification per item (INVALID_NCM): identity resolve (GTIN -> fingerprint) against the
+KB; validated non-conflicting knowledge resolves without AI. Otherwise hybrid retrieval:
+a HIGH_CONFIDENCE_RETRIEVAL requires hybrid score >= 0.75, an officially valid NCM and
+cross-tenant agreement >= 2 — it is NOT an automatic fiscal resolution (validator still
+required). Everything else with candidates -> LLM_SELECTION_REQUIRED; without candidates
+-> UNRESOLVED.
+
+CEST/EX_TIPI items never call retrieval semantics beyond the official candidate lists:
+exactly one official candidate resolves deterministically; none is UNRESOLVED
+(NO_OFFICIAL_MAPPING); more than one requires (future) LLM selection.
+PRODUCT_ROLE stays LLM/manual by contract.
+"""
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+
+from central.app.ai.candidates import cest_candidates_for_ncm, ex_tipi_candidates_for_ncm
+from central.app.ai.kb import EXACT, FINGERPRINT, KnowledgeBase
+from central.app.ai.retrieval import (EnrichedNcmCorpus, KbNeighborIndex, derive_abbreviation_lexicon, hybrid_candidates)
+from central.app.db import SessionLocal
+from central.app.models import AiWorkItem, Dataset, KbObservation, KbProduct, NcmEntry
+
+HIGH_CONFIDENCE_THRESHOLD = 0.75
+
+
+def _resolve_product(session, dedup_key: str):
+    """invalid_ncm:kb:{id} -> KB product by id; invalid_ncm:{db}:{code} -> via observation."""
+    if dedup_key.startswith("invalid_ncm:kb:"):
+        return session.get(KbProduct, int(dedup_key.rsplit(":", 1)[1]))
+    parts = dedup_key.split(":")
+    if len(parts) == 3:
+        observation = session.scalar(select(KbObservation).where(KbObservation.database_label == parts[1], KbObservation.erp_code == parts[2]))
+        return session.get(KbProduct, observation.kb_product_id) if observation else None
+    return None
+
+
+def main(output="docs/AI_RESIDUAL_PLAN_V2.json") -> dict:
+    session = SessionLocal()
+    kb = KnowledgeBase(session)
+    corpus = EnrichedNcmCorpus.build(session)
+    index = KbNeighborIndex.build(session)
+    lexicon = derive_abbreviation_lexicon(index.corpus(), min_support=3)
+    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+    valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all()) if dataset else set()
+    items = session.scalars(select(AiWorkItem)).all()
+    counts = {key: 0 for key in ("KB_EXACT_HIT", "KB_FINGERPRINT_HIT", "HIGH_CONFIDENCE_RETRIEVAL", "SINGLE_OFFICIAL_CANDIDATE", "LLM_SELECTION_REQUIRED", "UNRESOLVED")}
+    llm_cells = 0
+    for item in items:
+        if item.task_type == "INVALID_NCM":
+            product = None
+            product = _resolve_product(session, item.dedup_key)
+            if product is not None and product.gtin:
+                resolved, hit = kb.resolve(gtin=product.gtin)
+                if resolved and hit == EXACT:
+                    knowledge = kb.current_knowledge(resolved.id).get("validated_ncm", {})
+                    if knowledge and not knowledge.get("conflict"):
+                        item.status = "KB_EXACT_HIT"; counts["KB_EXACT_HIT"] += 1; continue
+            if product is not None:
+                resolved, hit = kb.resolve(fingerprint=product.fingerprint)
+                if resolved and hit == FINGERPRINT:
+                    knowledge = kb.current_knowledge(resolved.id).get("validated_ncm", {})
+                    if knowledge and not knowledge.get("conflict"):
+                        item.status = "KB_FINGERPRINT_HIT"; counts["KB_FINGERPRINT_HIT"] += 1; continue
+            description = product.normalized_description if product is not None else (item.payload or {}).get("description", "")
+            ranked = hybrid_candidates(corpus, index, description, k=10, lexicon=lexicon, exclude_product_id=(product.id if product else None))
+            if ranked and ranked[0][2] >= HIGH_CONFIDENCE_THRESHOLD and ranked[0][0] in valid_ncm and ranked[0][5] >= 2:
+                item.status = "HIGH_CONFIDENCE_RETRIEVAL"; counts["HIGH_CONFIDENCE_RETRIEVAL"] += 1
+                item.payload = {**item.payload, "retrieval_top1": ranked[0][0], "retrieval_score": ranked[0][2], "tenant_agreement": ranked[0][5]}
+            elif ranked:
+                item.status = "LLM_SELECTION_REQUIRED"; counts["LLM_SELECTION_REQUIRED"] += 1; llm_cells += item.cells_covered
+            else:
+                item.status = "UNRESOLVED"; counts["UNRESOLVED"] += 1
+        elif item.task_type in ("CEST", "EX_TIPI"):
+            ncm = (item.payload or {}).get("ncm", "")
+            candidates = cest_candidates_for_ncm(session, ncm) if item.task_type == "CEST" else ex_tipi_candidates_for_ncm(session, ncm)
+            if len(candidates) == 1:
+                item.status = "SINGLE_OFFICIAL_CANDIDATE"; counts["SINGLE_OFFICIAL_CANDIDATE"] += 1
+            elif not candidates:
+                item.status = "UNRESOLVED_NO_OFFICIAL_MAPPING"; counts["UNRESOLVED"] += 1
+            else:
+                item.status = "LLM_SELECTION_REQUIRED"; counts["LLM_SELECTION_REQUIRED"] += 1; llm_cells += item.cells_covered
+        else:  # PRODUCT_ROLE
+            item.status = "LLM_SELECTION_REQUIRED"; counts["LLM_SELECTION_REQUIRED"] += 1; llm_cells += item.cells_covered
+    session.commit()
+    llm_items = counts["LLM_SELECTION_REQUIRED"]
+    report = {
+        "artifact": "AI_RESIDUAL_PLAN_V2",
+        "work_items": len(items),
+        "classification": counts,
+        "llm_selection_required_items": llm_items,
+        "llm_selection_required_cells": llm_cells,
+        "high_confidence_threshold": HIGH_CONFIDENCE_THRESHOLD,
+        "note": "HIGH_CONFIDENCE_RETRIEVAL is a candidate, not a fiscal resolution: the official validator + operator gate still apply. AI_WRITE_ALLOWED=false.",
+        "dataset_versions": {"ncm": dataset.sha256} if dataset else {},
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", default="/tmp/plan_v2.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.output), indent=2))

central/scripts/run_ai_evaluation.py

@@ -0,0 +1,100 @@
+"""NCM evaluation benchmark (section 19) — same pipeline as production, known answer hidden.
+
+For a stratified sample of products whose NCM is valid in the ACTIVE dataset:
+  candidate generation (ncm_top_k)  ->  provider selection  ->  validator.
+Metrics: EXACT_MATCH / WRONG_SELECTION / NONE_WHEN_KNOWN / INVALID_OUTPUT (+ recall@K),
+per segment (database). With no AI credential the BaselineProvider runs: it measures the
+deterministic floor of the same pipeline and costs nothing.
+"""
+import argparse
+import json
+import random
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+
+from central.app.ai.candidates import ncm_top_k
+from central.app.ai.provider import BaselineProvider, PROMPT_VERSION, SemanticOutputError, result_from_dict
+from central.app.ai.fingerprint import normalize_description
+from central.app.db import SessionLocal
+from central.app.models import Dataset, KbObservation, KbProduct, NcmEntry
+
+SEGMENTS = {"CONSTRUCAO": "construcao", "MMMODAS": "moda", "RINCAO": "restaurante", "UNIAO": "mercado", "TESTESOFICIAIS": "base_testes"}
+
+
+def _samples(session, per_segment=40, seed=42):
+    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+    valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all())
+    rows = session.execute(select(KbProduct.normalized_description, KbObservation.database_label, KbObservation.observed_fields).join(KbObservation, KbObservation.kb_product_id == KbProduct.id)).all()
+    by_segment = {}
+    for description, database, fields in rows:
+        ncm = (fields or {}).get("ncm")
+        if ncm and ncm in valid_ncm and description:
+            by_segment.setdefault(SEGMENTS.get(database, database), set()).add((description, ncm))
+    rng = random.Random(seed)
+    samples = []
+    for segment, values in sorted(by_segment.items()):
+        picked = sorted(values)[: per_segment] if len(values) <= per_segment else rng.sample(sorted(values), per_segment)
+        samples.extend({"segment": segment, "description": description, "true_ncm": ncm} for description, ncm in picked)
+    return samples, dataset.sha256
+
+
+def main(per_segment=40, k=10, output="docs/AI_EVALUATION_V1.json") -> dict:
+    session = SessionLocal()
+    samples, ncm_sha = _samples(session, per_segment)
+    provider = BaselineProvider()
+    metrics = {"EXACT_MATCH": 0, "WRONG_SELECTION": 0, "NONE_WHEN_KNOWN": 0, "INVALID_OUTPUT": 0, "RECALL_AT_K": 0}
+    per_segment = {}
+    for sample in samples:
+        candidates = ncm_top_k(session, sample["description"], k=k)
+        codes = [code for code, _, _ in candidates]
+        bucket = per_segment.setdefault(sample["segment"], {key: 0 for key in metrics} | {"TOTAL": 0})
+        bucket["TOTAL"] += 1
+        if sample["true_ncm"] in codes:
+            metrics["RECALL_AT_K"] += 1; bucket["RECALL_AT_K"] += 1
+        item = type("W", (), {"task_type": "INVALID_NCM", "dedup_key": f"eval:{sample['segment']}:{normalize_description(sample['description'])[:60]}", "payload": {"description": sample["description"]}})()
+        try:
+            result = result_from_dict(provider.select(item, candidates, dataset_versions={"ncm": ncm_sha}).to_dict())
+        except SemanticOutputError:
+            metrics["INVALID_OUTPUT"] += 1; bucket["INVALID_OUTPUT"] += 1
+            continue
+        if result.selected_candidate == sample["true_ncm"]:
+            metrics["EXACT_MATCH"] += 1; bucket["EXACT_MATCH"] += 1
+        elif result.selected_candidate == "NONE":
+            metrics["NONE_WHEN_KNOWN"] += 1; bucket["NONE_WHEN_KNOWN"] += 1
+        else:
+            metrics["WRONG_SELECTION"] += 1; bucket["WRONG_SELECTION"] += 1
+    total = len(samples)
+    report = {
+        "artifact": "AI_EVALUATION_V1",
+        "task": "INVALID_NCM candidate selection",
+        "pipeline": "candidate_generation -> provider_selection -> validator",
+        "provider": provider.name,
+        "model": provider.model,
+        "prompt_version": PROMPT_VERSION,
+        "dataset_versions": {"ncm": ncm_sha},
+        "k": k,
+        "eval_total": total,
+        "metrics": metrics,
+        "accuracy_exact_match": round(metrics["EXACT_MATCH"] / total, 4) if total else None,
+        "recall_at_k": round(metrics["RECALL_AT_K"] / total, 4) if total else None,
+        "by_segment": per_segment,
+        "cost_usd": 0.0,
+        "note": "baseline provider (deterministic, zero-cost): measures the pipeline floor; rerun with a real provider once credentials exist",
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--per-segment", type=int, default=40)
+    parser.add_argument("--k", type=int, default=10)
+    parser.add_argument("--output", default="docs/AI_EVALUATION_V1.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.per_segment, args.k, args.output), indent=2))
+

central/scripts/run_ncm_retrieval_eval_v2.py

@@ -0,0 +1,115 @@
+"""NCM retrieval evaluation V2 (sections 1, 13).
+
+Same sampling as V1 (seed 42, per-segment) with each sample now quality-classified:
+  GOLD   - GTIN product or cross-tenant agreement on the same fingerprint/NCM
+  SILVER - single-tenant valid NCM, no independent confirmation
+  NOISY  - commercial description with < 2 informative tokens (not pursued)
+Retrieval measured separately: OFFICIAL_ONLY (hierarchical enriched corpus),
+KB_ONLY (neighbor bridge), HYBRID; recall@1/5/10/20/50 + MRR, per class and segment.
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+
+from central.app.ai.retrieval import (EnrichedNcmCorpus, KbNeighborIndex, commercial_tokens, derive_abbreviation_lexicon,
+                                      hierarchical_candidates, hybrid_candidates, kb_neighbor_candidates)
+from central.app.db import SessionLocal
+from central.app.models import Dataset, KbObservation, KbProduct, NcmEntry
+
+SEGMENTS = {"CONSTRUCAO": "construcao", "MMMODAS": "moda", "RINCAO": "restaurante", "UNIAO": "mercado", "TESTESOFICIAIS": "base_testes"}
+KS = (1, 5, 10, 20, 50)
+
+
+def _samples(session, per_segment=40, seed=42):
+    import random
+    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+    valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all())
+    rows = session.execute(
+        select(KbProduct, KbObservation).join(KbObservation, KbObservation.kb_product_id == KbProduct.id)
+    ).all()
+    per_product_tenants = {}
+    for product, observation in rows:
+        per_product_tenants.setdefault(product.id, set()).add(observation.tenant_id)
+    by_segment = {}
+    for product, observation in rows:
+        ncm = (observation.observed_fields or {}).get("ncm")
+        if ncm and ncm in valid_ncm and product.normalized_description:
+            by_segment.setdefault(SEGMENTS.get(observation.database_label, observation.database_label), set()).add(
+                (product.id, product.normalized_description, ncm, bool(product.gtin), len(per_product_tenants[product.id]) >= 2)
+            )
+    rng = random.Random(seed)
+    samples = []
+    for segment, values in sorted(by_segment.items()):
+        picked = sorted(values) if len(values) <= per_segment else rng.sample(sorted(values), per_segment)
+        for product_id, description, ncm, has_gtin, cross_tenant in picked:
+            quality = "NOISY" if len(commercial_tokens(description)) < 2 else ("GOLD" if (has_gtin or cross_tenant) else "SILVER")
+            samples.append({"segment": segment, "product_id": product_id, "description": description, "true_ncm": ncm, "quality": quality})
+    return samples
+
+
+def _recall_mrr(runs, samples):
+    """runs: {mode: [ranked_codes_per_sample]}"""
+    out = {}
+    for mode, ranked_lists in runs.items():
+        metrics = {f"recall_at_{k}": 0 for k in KS} | {"mrr": 0.0}
+        total = 0
+        for sample, ranked in zip(samples, ranked_lists):
+            codes = [row[0] for row in ranked]
+            total += 1
+            for k in KS:
+                if sample["true_ncm"] in codes[:k]:
+                    metrics[f"recall_at_{k}"] += 1
+            if sample["true_ncm"] in codes:
+                metrics["mrr"] += 1.0 / (codes.index(sample["true_ncm"]) + 1)
+        if total:
+            for key in metrics:
+                metrics[key] = round(metrics[key] / total, 4)
+        out[mode] = {"total": total, **metrics}
+    return out
+
+
+def main(per_segment=40, output="docs/NCM_RETRIEVAL_EVALUATION_V2.json") -> dict:
+    session = SessionLocal()
+    samples = _samples(session, per_segment)
+    corpus = EnrichedNcmCorpus.build(session)
+    index = KbNeighborIndex.build(session)
+    lexicon = derive_abbreviation_lexicon(index.corpus(), min_support=3)
+    runs = {"OFFICIAL_ONLY": [], "KB_ONLY": [], "HYBRID": []}
+    for sample in samples:
+        # leave-one-out: the queried product never retrieves itself from the KB
+        runs["OFFICIAL_ONLY"].append(hierarchical_candidates(corpus, sample["description"], k=50))
+        runs["KB_ONLY"].append(kb_neighbor_candidates(index, sample["description"], k=50, exclude_product_id=sample["product_id"]))
+        runs["HYBRID"].append(hybrid_candidates(corpus, index, sample["description"], k=50, lexicon=lexicon, exclude_product_id=sample["product_id"]))
+    overall = _recall_mrr(runs, samples)
+    by_quality = {quality: _recall_mrr({mode: [row for row, sample in zip(list_rows, samples) if sample["quality"] == quality] for mode, list_rows in runs.items()},
+                                       [sample for sample in samples if sample["quality"] == quality]) for quality in ("GOLD", "SILVER", "NOISY")}
+    by_segment = {segment: _recall_mrr({mode: [row for row, sample in zip(list_rows, samples) if sample["segment"] == segment] for mode, list_rows in runs.items()},
+                                        [sample for sample in samples if sample["segment"] == segment]) for segment in sorted({sample["segment"] for sample in samples})}
+    report = {
+        "artifact": "NCM_RETRIEVAL_EVALUATION_V2",
+        "eval_total": len(samples),
+        "quality_counts": {quality: sum(1 for sample in samples if sample["quality"] == quality) for quality in ("GOLD", "SILVER", "NOISY")},
+        "lexicon_size": len(lexicon),
+        "lexicon_sample": dict(sorted(lexicon.items())[:40]),
+        "overall": overall,
+        "by_quality": by_quality,
+        "by_segment": by_segment,
+        "llm_liberation_gate": {"gold_recall_at_20_min": 0.80, "gold_recall_at_50_preferred": 0.90},
+        "cost_usd": 0.0,
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--per-segment", type=int, default=40)
+    parser.add_argument("--output", default="docs/NCM_RETRIEVAL_EVALUATION_V2.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.per_segment, args.output), indent=2))

central/scripts/run_real_llm_ncm_eval.py

@@ -0,0 +1,220 @@
+"""Real LLM NCM evaluation over known-answer cases (mission REAL LLM EVALUATION V1).
+
+Phases: gold first (65 cases, gold completely hidden from the prompt); silver and noisy
+run ONLY after the gold quality gate passes (selection accuracy >= 0.95 among valid
+answers AND high-confidence wrong rate <= 0.02). Batching verified progressively
+(5/10/20) with 1:1 id matching; results cached permanently per
+(input_fingerprint, candidate_set_hash, prompt_version, model, dataset_versions) so an
+identical re-execution makes ZERO new calls. AI_WRITE_ALLOWED=false; nothing here can
+produce a Firebird write.
+
+If no AI credential exists the run refuses to call anything and exits with
+BLOCKED_EXTERNAL_AI_CREDENTIAL (no key is ever invented).
+"""
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+
+from central.app.ai.eval_metrics import attribution, confidence_buckets, metrics
+from central.app.ai.llm_provider import (PROMPT_VERSION_LLM, LlmAnswerError, LlmEvalItem, OpenAICompatibleProvider,
+                                         candidate_set_hash, input_fingerprint, validate_answer)
+from central.app.ai.retrieval import EnrichedNcmCorpus, KbNeighborIndex, derive_abbreviation_lexicon, hybrid_candidates
+from central.app.config import settings
+from central.app.db import SessionLocal
+from central.app.models import AiEvalCache, Dataset, NcmEntry
+
+GATE_SELECTION_ACCURACY = 0.95
+GATE_HIGH_CONF_WRONG = 0.02
+BATCH_PROBE_SIZES = (20, 10, 5)
+
+
+def _load_samples(session, per_segment=40):
+    from central.scripts.run_ncm_retrieval_eval_v2 import _samples
+    return _samples(session, per_segment)
+
+
+def _build_items(samples, corpus, index, lexicon, top_k=20):
+    items = []
+    meta = []
+    for sample in samples:
+        ranked = hybrid_candidates(corpus, index, sample["description"], k=top_k, lexicon=lexicon, exclude_product_id=sample["product_id"])
+        item = LlmEvalItem(
+            work_item_id=f"{sample['segment']}:{sample['product_id']}",
+            description=sample["description"],
+            normalized_description=sample["description"],
+            brand=None, unit=None, segment=sample["segment"],
+            candidates=ranked,
+        )
+        items.append(item)
+        meta.append({**sample, "gold_in_candidates": sample["true_ncm"] in {code for code, *_ in ranked}})
+    return items, meta
+
+
+def _cached_answers(session, items, model, ncm_sha, phase):
+    hits = {}
+    for item in items:
+        row = session.scalar(select(AiEvalCache).where(
+            AiEvalCache.input_fingerprint == input_fingerprint(item),
+            AiEvalCache.candidate_set_hash == candidate_set_hash(item),
+            AiEvalCache.prompt_version == PROMPT_VERSION_LLM,
+            AiEvalCache.model == model,
+            AiEvalCache.dataset_version_ncm == ncm_sha,
+            AiEvalCache.phase == phase,
+        ))
+        if row:
+            hits[item.work_item_id] = row.answer
+    return hits
+
+
+def _diagnose_http(status_code: int) -> str:
+    return {401: "credential invalid (AI_API_KEY)", 403: "permission/plan does not allow this endpoint or model", 404: "endpoint or model not found (AI_BASE_URL/AI_MODEL)", 429: "quota or rate limit"}.get(status_code, f"http {status_code}")
+
+
+def _select_model(available: list, requested: str) -> tuple:
+    """Prefer the operator-requested model; else a current GLM suitable for classification.
+    Never invents a name and never switches vendor — returns (model, source)."""
+    if requested in available:
+        return requested, "requested"
+    glm = [model for model in available if model.lower().startswith("glm")]
+    for candidate in glm:
+        if "flash" in candidate.lower():
+            return candidate, "fallback_glm_flash"
+    if glm:
+        return sorted(glm)[-1], "fallback_glm_latest"
+    return None, "no_glm_available"
+
+
+def _preflight(provider: OpenAICompatibleProvider, requested_model: str) -> dict:
+    """Cheap connectivity check (GET /models — no completion tokens). Validates auth,
+    base URL and model availability; diagnoses 401/403/404/429 autonomously."""
+    import httpx
+    try:
+        available = provider.list_models()
+    except httpx.HTTPStatusError as error:
+        return {"zai_connection": "FAIL", "diagnosis": _diagnose_http(error.response.status_code), "model_available": False}
+    except httpx.HTTPError as error:
+        return {"zai_connection": "FAIL", "diagnosis": f"transport: {type(error).__name__}", "model_available": False}
+    model, source = _select_model(available, requested_model)
+    return {"zai_connection": "PASS", "model_available": model is not None, "model": model, "model_source": source,
+            "available_models": available if model is None else None, "base_url": provider.base_url}
+
+
+def _probe_batch_size(provider, items):
+    """Try the largest batch that keeps valid JSON with 1:1 ids on a small probe slice."""
+    for size in BATCH_PROBE_SIZES:
+        probe = items[:size]
+        try:
+            provider.select_batch(probe)
+            return size
+        except (LlmAnswerError, json.JSONDecodeError):
+            continue
+    return 1
+
+
+def run_phase(provider_factory, phase, samples, corpus, index, lexicon, session, ncm_sha, valid_ncm):
+    items, meta = _build_items(samples, corpus, index, lexicon)
+    cached = _cached_answers(session, items, provider_factory.model if hasattr(provider_factory, "model") else settings.ai_model, ncm_sha, phase)
+    todo = [item for item in items if item.work_item_id not in cached]
+    answers = dict(cached)
+    provider = None
+    batch_size = 0
+    if todo:
+        provider = provider_factory()
+        batch_size = _probe_batch_size(provider, todo)
+        for start in range(0, len(todo), batch_size):
+            for answer in provider.select_batch(todo[start:start + batch_size]):
+                answers[answer["work_item_id"]] = answer
+    rows, errors = [], []
+    for item, sample in zip(items, meta):
+        answer = answers.get(item.work_item_id)
+        invalid = False
+        if answer is None:
+            invalid = True
+            answer = {"selected_ncm": None, "confidence": 0.0, "abstain": True, "reason_short": "missing"}
+        else:
+            try:
+                validate_answer(answer, item)
+                if answer["selected_ncm"] is not None and answer["selected_ncm"] not in valid_ncm:
+                    invalid = True
+            except LlmAnswerError:
+                invalid = True
+        selected = None if (invalid or answer["abstain"] or answer["selected_ncm"] is None) else answer["selected_ncm"]
+        rows.append({"gold_in_candidates": sample["gold_in_candidates"], "selected": selected, "gold": sample["true_ncm"], "confidence": answer["confidence"], "invalid": invalid})
+        if selected is not None and selected != sample["true_ncm"]:
+            errors.append({"work_item_id": item.work_item_id, "gold": sample["true_ncm"], "selected": selected, "confidence": answer["confidence"], "gold_in_candidates": sample["gold_in_candidates"], "reason_short": answer.get("reason_short", "")})
+        if invalid or answer.get("abstain") or selected is not None:
+            if item.work_item_id not in cached:
+                session.add(AiEvalCache(input_fingerprint=input_fingerprint(item), candidate_set_hash=candidate_set_hash(item), prompt_version=PROMPT_VERSION_LLM,
+                                        model=provider.model if provider else settings.ai_model, dataset_version_ncm=ncm_sha, phase=phase,
+                                        answer=answer, gold_ncm=sample["true_ncm"]))
+    session.commit()
+    return rows, errors, provider, batch_size, len(cached)
+
+
+def main(output_dir="/tmp", per_segment=40) -> dict:
+    if not settings.ai_api_key:
+        print(json.dumps({"status": "BLOCKED_EXTERNAL_AI_CREDENTIAL",
+                          "operator_action": "set AI_API_KEY (and optionally AI_BASE_URL/AI_MODEL) in the Hub environment (.env); never commit or log it"}, indent=2))
+        sys.exit(2)
+    session = SessionLocal()
+    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+    valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all())
+    corpus = EnrichedNcmCorpus.build(session)
+    index = KbNeighborIndex.build(session)
+    lexicon = derive_abbreviation_lexicon(index.corpus(), min_support=3)
+    samples = _load_samples(session, per_segment)
+    # ---- preflight: cheap /models call, never spends completion tokens ----
+    pre_provider = OpenAICompatibleProvider(api_key=settings.ai_api_key, model=settings.ai_model, base_url=settings.ai_base_url)
+    preflight = _preflight(pre_provider, settings.ai_model)
+    if preflight["zai_connection"] != "PASS" or not preflight["model_available"]:
+        print(json.dumps({"status": "ZAI_PREFLIGHT_FAIL", "preflight": preflight}, indent=2))
+        sys.exit(3)
+    model_name = preflight["model"]
+    factory = lambda: OpenAICompatibleProvider(api_key=settings.ai_api_key, model=model_name, base_url=settings.ai_base_url)
+    gold = [sample for sample in samples if sample["quality"] == "GOLD"]
+    provider = factory()
+    rows, errors, provider_used, batch_size, cache_hits = run_phase(lambda: provider, "gold", gold, corpus, index, lexicon, session, dataset.sha256, valid_ncm)
+    m = metrics(rows)
+    prices_configured = settings.ai_price_in_per_m is not None and settings.ai_price_out_per_m is not None
+    usage = provider.usage if provider is not None else {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
+    cost_total = round(usage["input_tokens"] / 1e6 * settings.ai_price_in_per_m + usage["output_tokens"] / 1e6 * settings.ai_price_out_per_m, 6) if prices_configured else "UNKNOWN"
+    report = {
+        "artifact": "REAL_LLM_NCM_EVALUATION_V1", "phase": "gold",
+        "provider": "z.ai", "base_url": settings.ai_base_url, "model": model_name, "prompt_version": PROMPT_VERSION_LLM,
+        "preflight": preflight,
+        "metrics": m, "attribution": attribution(rows), "confidence_buckets": confidence_buckets(rows),
+        "batch_size_selected": batch_size,
+        "gate": {"selection_accuracy_min": GATE_SELECTION_ACCURACY, "high_conf_wrong_max": GATE_HIGH_CONF_WRONG,
+                 "passed": (m["SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE"] or 0) >= GATE_SELECTION_ACCURACY and (m["HIGH_CONFIDENCE_WRONG_RATE"] or 0) <= GATE_HIGH_CONF_WRONG},
+        "cache_hits_first_run": cache_hits, "ai_write_allowed": False,
+        "usage": {**usage, "http_calls": provider.http_calls, "retries": provider.retries,
+                  "average_items_per_call": round(len(rows) / max(provider.http_calls, 1), 2),
+                  "tokens_per_item": round((usage["input_tokens"] + usage["output_tokens"]) / max(len(rows), 1), 1),
+                  "cost_total": cost_total, "cost_per_item": (round(cost_total / max(len(rows), 1), 6) if prices_configured else "UNKNOWN"),
+                  "prices_configured": prices_configured},
+    }
+    Path(f"{output_dir}/REAL_LLM_NCM_EVALUATION_V1.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    Path(f"{output_dir}/REAL_LLM_NCM_ERRORS_V1.json").write_text(json.dumps(errors, indent=2, ensure_ascii=False), encoding="utf-8")
+    # cache proof: identical re-execution must make zero new calls
+    provider2 = factory()
+    rows2, _, _, _, cache_hits2 = run_phase(lambda: provider2, "gold", gold, corpus, index, lexicon, session, dataset.sha256, valid_ncm)
+    second_calls = provider2.http_calls
+    report["cache_proof"] = {"second_run_calls": second_calls, "second_run_cache_hits": cache_hits2,
+                             "cache_hit_reexecution": second_calls == 0, "identical_metrics": metrics(rows2)["CORRECT"] == m["CORRECT"]}
+    Path(f"{output_dir}/REAL_LLM_NCM_EVALUATION_V1.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    Path(f"{output_dir}/REAL_LLM_COST_V1.json").write_text(json.dumps({"artifact": "REAL_LLM_COST_V1", "usage": report["usage"], "previous_estimate_usd": 0.44, "cache_proof": report["cache_proof"]}, indent=2), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output-dir", default="/tmp")
+    parser.add_argument("--per-segment", type=int, default=40)
+    args = parser.parse_args()
+    print(json.dumps(main(args.output_dir, args.per_segment), indent=2))

central/scripts/seed_product_kb.py

@@ -0,0 +1,158 @@
+"""Seed the Product Knowledge Base deterministically from the 5 Firebird databases.
+
+Two phases (Firebird is only reachable from Windows; Postgres only inside the compose network):
+
+  export (Windows, READ-ONLY SELECT against Firebird):
+      python scripts/seed_product_kb.py export --output docs/runs/ai_kb_seed_payload.json
+  load (inside the fiscal-hub container, against Postgres):
+      python -m central.scripts.seed_product_kb load --input docs/runs/ai_kb_seed_payload.json
+
+Export computes, per physical product row: the stable global fingerprint
+(description|unit; GTIN validated by checksum), and emits observations. Load upserts the
+GLOBAL product, records per-tenant observations, seeds knowledge already proven by the
+gate (validated_ncm when the declared NCM is active-official; ERP-declared values stay
+ERP_OBSERVATION evidence), and links TENANT_PRODUCT_CONTEXT rows. Tenant context is
+never promoted to global knowledge.
+"""
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+DATABASES = [
+    ("TESTESOFICIAIS", None),  # path from FIREBIRD_DATABASE_PATH
+    ("CONSTRUCAO", "Banco/CONSTRUCAO.FDB"),
+    ("MMMODAS", "Banco/MMMODAS.FDB"),
+    ("RINCAO", "Banco/RINCAO.FDB"),
+    ("UNIAO", "Banco/UNIAO.FDB"),
+]
+
+QUERY = "SELECT p.CODIGO, p.PRODUTO, p.UNIDADE, p.CODBARRA, p.GTIN, p.CLASSIFICACAO_FISCAL, p.CEST FROM C000025 p"
+
+
+def _canonical_ncm(value) -> str:
+    digits = "".join(char for char in str(value or "") if char.isdigit())
+    return digits if len(digits) == 8 else ""
+
+
+def export_payload(limit_per_db: int = 0, output: str = "docs/runs/ai_kb_seed_payload.json") -> dict:
+    import fdb
+    from dotenv import dotenv_values
+    from central.app.ai.fingerprint import normalize_description, product_fingerprint, validate_gtin
+    env = dotenv_values(".env")
+    stats = {"databases": [], "rows_read": 0, "firebird_writes": 0}
+    products = {}
+    for label, relative in DATABASES:
+        path = env["FIREBIRD_DATABASE_PATH"] if relative is None else str(Path(relative).resolve())
+        connection = fdb.connect(database=path, user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"))
+        try:
+            cursor = connection.cursor(); cursor.execute(QUERY)
+            columns = [column[0].strip().upper() for column in cursor.description]
+            rows = cursor.fetchmany(limit_per_db) if limit_per_db else cursor.fetchall()
+        finally:
+            connection.close()
+        # collapse the physical duplicate CODIGO rows into observations of the same product
+        seen_codes = set()
+        for row in rows:
+            record = dict(zip(columns, row))
+            code = str(record.get("CODIGO") or "").strip()
+            dedupe = f"{label}:{code}"
+            if dedupe in seen_codes:  # duplicate physical key (e.g. MMMODAS 017421) -> one observation
+                continue
+            seen_codes.add(dedupe)
+            unit = normalize_description(record.get("UNIDADE")) or None
+            fingerprint = product_fingerprint(record.get("PRODUTO"), None, unit)
+            gtin = validate_gtin(record.get("GTIN")) or validate_gtin(record.get("CODBARRA"))
+            entry = products.setdefault(fingerprint, {
+                "fingerprint": fingerprint,
+                "gtin": gtin,
+                "normalized_description": normalize_description(record.get("PRODUTO")),
+                "unit": unit,
+                "ncm": _canonical_ncm(record.get("CLASSIFICACAO_FISCAL")),
+                "cest": str(record.get("CEST") or "").strip() or None,
+                "observations": [],
+            })
+            if gtin and not entry["gtin"]:
+                entry["gtin"] = gtin
+            entry["observations"].append({"database": label, "company_key": label, "erp_code": code})
+        stats["databases"].append({"database": label, "rows": len(rows)})
+        stats["rows_read"] += len(rows)
+    payload = {"artifact": "AI_KB_SEED_PAYLOAD", "exported_at": datetime.now(timezone.utc).isoformat(), "stats": stats, "products": list(products.values())}
+    Path(output).parent.mkdir(parents=True, exist_ok=True)
+    Path(output).write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
+    stats["unique_products"] = len(products)
+    stats["payload_path"] = output
+    return stats
+
+
+def load_payload(input_path: str) -> dict:
+    from sqlalchemy import select
+    from central.app.ai.kb import KnowledgeBase
+    from central.app.db import SessionLocal
+    from central.app.models import Dataset, NcmEntry, Tenant, TenantProductContext
+    payload = json.loads(Path(input_path).read_text(encoding="utf8"))
+    session = SessionLocal()
+    kb = KnowledgeBase(session)
+    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+    active_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all()) if dataset else set()
+    tenant_ids = {}
+    stats = {"products": 0, "observations": 0, "validated_ncm": 0, "erp_ncm_observations": 0, "erp_cest_observations": 0, "tenant_contexts": 0, "firebird_writes": 0}
+    for product in payload["products"]:
+        row = kb.upsert_product(fingerprint=product["fingerprint"], gtin=product.get("gtin"), normalized_description=product.get("normalized_description", ""), unit=product.get("unit"))
+        stats["products"] += 1
+        for observation in product["observations"]:
+            label = observation["database"]
+            if label not in tenant_ids:
+                tenant = session.scalar(select(Tenant).where(Tenant.name == label))
+                if tenant is None:
+                    tenant = Tenant(name=label); session.add(tenant); session.flush()
+                tenant_ids[label] = tenant.id
+            kb.record_observation(row.id, tenant_id=tenant_ids[label], company_key=label, erp_code=observation["erp_code"], database_label=label, observed_fields={"ncm": product.get("ncm") or None, "cest": product.get("cest")})
+            stats["observations"] += 1
+            existing = session.scalar(select(TenantProductContext).where(TenantProductContext.tenant_id == tenant_ids[label], TenantProductContext.company_key == label, TenantProductContext.erp_code == observation["erp_code"]))
+            if existing is None:
+                session.add(TenantProductContext(tenant_id=tenant_ids[label], company_key=label, erp_code=observation["erp_code"], kb_product_id=row.id))
+                stats["tenant_contexts"] += 1
+        ncm = product.get("ncm") or ""
+        if ncm and ncm in active_ncm:
+            kb.add_knowledge(row.id, "validated_ncm", ncm, source="OFFICIAL_DATASET", confidence=1.0, dataset_versions={"ncm": dataset.sha256})
+            stats["validated_ncm"] += 1
+        elif ncm:
+            kb.add_knowledge(row.id, "candidate_ncm", ncm, source="ERP_OBSERVATION", confidence=0.3, evidence={"declared_in": [observation["database"] for observation in product["observations"]]})
+            stats["erp_ncm_observations"] += 1
+        if product.get("cest"):
+            kb.add_knowledge(row.id, "candidate_cest", product["cest"], source="ERP_OBSERVATION", confidence=0.3, evidence={"declared_in": [observation["database"] for observation in product["observations"]]})
+            stats["erp_cest_observations"] += 1
+        if stats["products"] % 2000 == 0:
+            session.commit()
+    session.commit()
+    stats.update(kb.stats())
+    session.close()
+    return stats
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser()
+    sub = parser.add_subparsers(dest="phase", required=True)
+    export_parser = sub.add_parser("export")
+    export_parser.add_argument("--limit", type=int, default=0)
+    export_parser.add_argument("--output", default="docs/runs/ai_kb_seed_payload.json")
+    load_parser = sub.add_parser("load")
+    load_parser.add_argument("--input", default="docs/runs/ai_kb_seed_payload.json")
+    load_parser.add_argument("--output", default="docs/runs/ai_kb_seed.json")
+    args = parser.parse_args(argv)
+    if args.phase == "export":
+        result = export_payload(args.limit, args.output)
+    else:
+        result = load_payload(args.input)
+        result["loaded_at"] = datetime.now(timezone.utc).isoformat()
+        Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.output).write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
+    print(json.dumps(result, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()