chore: capture semantic pipeline baseline

2026-09-21 15:52:17 -03:00
parent 9dcc263217
commit 6731eeca08
47 changed files with 3724 additions and 15 deletions
@@ -0,0 +1,112 @@
+"""Reprocess the 750 AI work items with V2 retrieval (section 16).
+
+Classification per item (INVALID_NCM): identity resolve (GTIN -> fingerprint) against the
+KB; validated non-conflicting knowledge resolves without AI. Otherwise hybrid retrieval:
+a HIGH_CONFIDENCE_RETRIEVAL requires hybrid score >= 0.75, an officially valid NCM and
+cross-tenant agreement >= 2 — it is NOT an automatic fiscal resolution (validator still
+required). Everything else with candidates -> LLM_SELECTION_REQUIRED; without candidates
+-> UNRESOLVED.
+
+CEST/EX_TIPI items never call retrieval semantics beyond the official candidate lists:
+exactly one official candidate resolves deterministically; none is UNRESOLVED
+(NO_OFFICIAL_MAPPING); more than one requires (future) LLM selection.
+PRODUCT_ROLE stays LLM/manual by contract.
+"""
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+
+from central.app.ai.candidates import cest_candidates_for_ncm, ex_tipi_candidates_for_ncm
+from central.app.ai.kb import EXACT, FINGERPRINT, KnowledgeBase
+from central.app.ai.retrieval import (EnrichedNcmCorpus, KbNeighborIndex, derive_abbreviation_lexicon, hybrid_candidates)
+from central.app.db import SessionLocal
+from central.app.models import AiWorkItem, Dataset, KbObservation, KbProduct, NcmEntry
+
+HIGH_CONFIDENCE_THRESHOLD = 0.75
+
+
+def _resolve_product(session, dedup_key: str):
+    """invalid_ncm:kb:{id} -> KB product by id; invalid_ncm:{db}:{code} -> via observation."""
+    if dedup_key.startswith("invalid_ncm:kb:"):
+        return session.get(KbProduct, int(dedup_key.rsplit(":", 1)[1]))
+    parts = dedup_key.split(":")
+    if len(parts) == 3:
+        observation = session.scalar(select(KbObservation).where(KbObservation.database_label == parts[1], KbObservation.erp_code == parts[2]))
+        return session.get(KbProduct, observation.kb_product_id) if observation else None
+    return None
+
+
+def main(output="docs/AI_RESIDUAL_PLAN_V2.json") -> dict:
+    session = SessionLocal()
+    kb = KnowledgeBase(session)
+    corpus = EnrichedNcmCorpus.build(session)
+    index = KbNeighborIndex.build(session)
+    lexicon = derive_abbreviation_lexicon(index.corpus(), min_support=3)
+    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+    valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all()) if dataset else set()
+    items = session.scalars(select(AiWorkItem)).all()
+    counts = {key: 0 for key in ("KB_EXACT_HIT", "KB_FINGERPRINT_HIT", "HIGH_CONFIDENCE_RETRIEVAL", "SINGLE_OFFICIAL_CANDIDATE", "LLM_SELECTION_REQUIRED", "UNRESOLVED")}
+    llm_cells = 0
+    for item in items:
+        if item.task_type == "INVALID_NCM":
+            product = None
+            product = _resolve_product(session, item.dedup_key)
+            if product is not None and product.gtin:
+                resolved, hit = kb.resolve(gtin=product.gtin)
+                if resolved and hit == EXACT:
+                    knowledge = kb.current_knowledge(resolved.id).get("validated_ncm", {})
+                    if knowledge and not knowledge.get("conflict"):
+                        item.status = "KB_EXACT_HIT"; counts["KB_EXACT_HIT"] += 1; continue
+            if product is not None:
+                resolved, hit = kb.resolve(fingerprint=product.fingerprint)
+                if resolved and hit == FINGERPRINT:
+                    knowledge = kb.current_knowledge(resolved.id).get("validated_ncm", {})
+                    if knowledge and not knowledge.get("conflict"):
+                        item.status = "KB_FINGERPRINT_HIT"; counts["KB_FINGERPRINT_HIT"] += 1; continue
+            description = product.normalized_description if product is not None else (item.payload or {}).get("description", "")
+            ranked = hybrid_candidates(corpus, index, description, k=10, lexicon=lexicon, exclude_product_id=(product.id if product else None))
+            if ranked and ranked[0][2] >= HIGH_CONFIDENCE_THRESHOLD and ranked[0][0] in valid_ncm and ranked[0][5] >= 2:
+                item.status = "HIGH_CONFIDENCE_RETRIEVAL"; counts["HIGH_CONFIDENCE_RETRIEVAL"] += 1
+                item.payload = {**item.payload, "retrieval_top1": ranked[0][0], "retrieval_score": ranked[0][2], "tenant_agreement": ranked[0][5]}
+            elif ranked:
+                item.status = "LLM_SELECTION_REQUIRED"; counts["LLM_SELECTION_REQUIRED"] += 1; llm_cells += item.cells_covered
+            else:
+                item.status = "UNRESOLVED"; counts["UNRESOLVED"] += 1
+        elif item.task_type in ("CEST", "EX_TIPI"):
+            ncm = (item.payload or {}).get("ncm", "")
+            candidates = cest_candidates_for_ncm(session, ncm) if item.task_type == "CEST" else ex_tipi_candidates_for_ncm(session, ncm)
+            if len(candidates) == 1:
+                item.status = "SINGLE_OFFICIAL_CANDIDATE"; counts["SINGLE_OFFICIAL_CANDIDATE"] += 1
+            elif not candidates:
+                item.status = "UNRESOLVED_NO_OFFICIAL_MAPPING"; counts["UNRESOLVED"] += 1
+            else:
+                item.status = "LLM_SELECTION_REQUIRED"; counts["LLM_SELECTION_REQUIRED"] += 1; llm_cells += item.cells_covered
+        else:  # PRODUCT_ROLE
+            item.status = "LLM_SELECTION_REQUIRED"; counts["LLM_SELECTION_REQUIRED"] += 1; llm_cells += item.cells_covered
+    session.commit()
+    llm_items = counts["LLM_SELECTION_REQUIRED"]
+    report = {
+        "artifact": "AI_RESIDUAL_PLAN_V2",
+        "work_items": len(items),
+        "classification": counts,
+        "llm_selection_required_items": llm_items,
+        "llm_selection_required_cells": llm_cells,
+        "high_confidence_threshold": HIGH_CONFIDENCE_THRESHOLD,
+        "note": "HIGH_CONFIDENCE_RETRIEVAL is a candidate, not a fiscal resolution: the official validator + operator gate still apply. AI_WRITE_ALLOWED=false.",
+        "dataset_versions": {"ncm": dataset.sha256} if dataset else {},
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", default="/tmp/plan_v2.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.output), indent=2))