feat: complete pre-ai local completion v3

central/app/ai/kb_consolidation.py

@@ -0,0 +1,87 @@
+"""Conservative, explainable product identity and conflict classification."""
+
+import hashlib
+import re
+from collections import defaultdict
+
+from .fingerprint import normalize_description, token_stems, validate_gtin
+
+GTIN_CONFLICT = "GTIN_CONFLICT"
+NCM_CONFLICT = "NCM_CONFLICT"
+DESCRIPTION_CONFLICT = "DESCRIPTION_CONFLICT"
+PACKAGING_VARIATION = "PACKAGING_VARIATION"
+FALSE_MATCH = "FALSE_MATCH"
+TENANT_CONTEXT_ONLY = "TENANT_CONTEXT_ONLY"
+
+
+def canonical_product_id(observation: dict) -> str:
+    """Stable global ID; fiscal and tenant fields are intentionally excluded."""
+    gtin = validate_gtin(observation.get("gtin"))
+    if gtin:
+        return "GTIN:" + gtin
+    values = "|".join((
+        normalize_description(observation.get("normalized_description") or observation.get("description")),
+        normalize_description(observation.get("brand")),
+        normalize_description(observation.get("unit")),
+        normalize_description(observation.get("package_quantity")),
+    ))
+    return "FP:" + hashlib.sha256(values.encode()).hexdigest()
+
+
+def _description_equivalent(values):
+    stems = [token_stems(value) for value in values if value]
+    if not stems:
+        return True
+    return all(current == stems[0] or current.issubset(stems[0]) or stems[0].issubset(current) for current in stems[1:])
+
+
+def classify_conflict(observations: list[dict]) -> str:
+    """Classify only from structural evidence; ambiguous cases remain conflicts."""
+    gtins = {validate_gtin(row.get("gtin")) for row in observations if validate_gtin(row.get("gtin"))}
+    ncms = {str(row.get("ncm")) for row in observations if row.get("ncm")}
+    descriptions = [row.get("normalized_description") or row.get("description") for row in observations]
+    packages = {str(row.get("package_quantity")) for row in observations if row.get("package_quantity")}
+    tenants = {str(obs.get("database") or obs.get("company_key")) for row in observations for obs in row.get("observations", [])}
+    if len(gtins) > 1:
+        return GTIN_CONFLICT
+    if len(ncms) > 1:
+        return NCM_CONFLICT
+    if len(packages) > 1 and _description_equivalent(descriptions):
+        return PACKAGING_VARIATION
+    if len({normalize_description(value) for value in descriptions if value}) > 1:
+        return DESCRIPTION_CONFLICT
+    if len(tenants) > 1:
+        return TENANT_CONTEXT_ONLY
+    return FALSE_MATCH
+
+
+def audit_payload(payload: dict, historical_conflicts: int = 163) -> dict:
+    """Audit observable same-GTIN groups without silently resolving NCM conflicts."""
+    by_gtin = defaultdict(list)
+    for product in payload.get("products", []):
+        gtin = validate_gtin(product.get("gtin"))
+        if gtin:
+            by_gtin[gtin].append(product)
+    conflicts = []
+    counts = defaultdict(int)
+    for gtin, rows in by_gtin.items():
+        if len({row.get("ncm") for row in rows if row.get("ncm")}) <= 1:
+            continue
+        kind = classify_conflict(rows)
+        counts[kind] += 1
+        conflicts.append({
+            "identity": gtin,
+            "classification": kind,
+            "observations": len(rows),
+            "ncms": sorted({row.get("ncm") for row in rows if row.get("ncm")}),
+            "auto_resolution": "KEEP_KNOWLEDGE_CONFLICT",
+        })
+    return {
+        "artifact": "KB_CONFLICT_AUDIT_V1",
+        "historical_conflicts_reported": historical_conflicts,
+        "observable_conflicts_audited": len(conflicts),
+        "unobservable_historical_conflicts": max(0, historical_conflicts - len(conflicts)),
+        "classification_counts": dict(sorted(counts.items())),
+        "conflicts": conflicts,
+        "policy": "only structurally equivalent identities may share a canonical product; tenant fiscal context is never global knowledge",
+    }