chore: capture semantic pipeline baseline

2026-09-21 15:52:17 -03:00
parent 9dcc263217
commit 6731eeca08
47 changed files with 3724 additions and 15 deletions
@@ -0,0 +1,64 @@
+"""Residual-question dedup: group pending cells by the KNOWLEDGE needed, not per cell.
+
+- INVALID_NCM  -> one work item per product fingerprint (same product in N databases collapses)
+- CEST/EX_TIPI -> one work item per (doubt, NCM): all databases sharing that NCM collapse
+- PRODUCT_ROLE -> tenant-specific by contract: one work item per (company, group)
+- Everything else (origin, ST retention, company profile, duplicate key) never becomes
+  an AI work item.
+"""
+from dataclasses import dataclass, field
+
+_INVALID_NCM, _ROLE = "ncm", "product_role"
+
+
+@dataclass
+class WorkItem:
+    task_type: str
+    dedup_key: str
+    question_refs: list = field(default_factory=list)
+    cells_covered: int = 0
+    payload: dict = field(default_factory=dict)
+    kb_product_id: int | None = None
+
+
+def build_work_items(questions, product_fingerprints=None) -> list[WorkItem]:
+    """`product_fingerprints` maps "{DATABASE}:{erp_code}" -> KB fingerprint (from the seed pass)."""
+    product_fingerprints = product_fingerprints or {}
+    items: dict = {}
+
+    def _get(task_type, dedup_key, ref, cells):
+        item = items.get(dedup_key)
+        if item is None:
+            item = WorkItem(task_type=task_type, dedup_key=dedup_key)
+            items[dedup_key] = item
+        item.question_refs.append(ref)
+        item.cells_covered += cells
+        return item
+
+    for question in questions:
+        key = question.get("KEY")
+        scope = question.get("SCOPE")
+        cells = int(question.get("CELLS") or question.get("PRODUCTS_AFFECTED") or 0)
+        ref = question.get("QUESTION_ID") or f"{question.get('DATABASE')}:{scope}:{question.get('NCM') or question.get('GROUP')}:{key}"
+        if key in ("cest", "ex_tipi") and scope == "NCM":
+            ncm = str(question.get("NCM") or "").strip()
+            if ncm:
+                item = _get(key.upper(), f"{key}:{ncm}", ref, cells)
+                item.payload.setdefault("ncm", ncm)
+                item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
+        elif key == _INVALID_NCM and scope == "PRODUCT":
+            database = question.get("DATABASE", "")
+            erp_code = str(question.get("DATABASE_CODE") or "").strip()
+            fingerprint = product_fingerprints.get(f"{database}:{erp_code}")
+            dedup = f"invalid_ncm:{fingerprint}" if fingerprint else f"invalid_ncm:{database}:{erp_code}"
+            item = _get("INVALID_NCM", dedup, ref, cells)
+            item.payload.setdefault("erp_code", erp_code)
+            item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
+        elif key == _ROLE and scope == "GROUP":
+            company = question.get("COMPANY", "")
+            group = str(question.get("GROUP") or "").strip()
+            if group:
+                item = _get("PRODUCT_ROLE", f"product_role:{company}:{group}", ref, cells)
+                item.payload.setdefault("company_key", company)
+                item.payload["group"] = group
+    return list(items.values())