"""Retrieval V3 evidence layer. It ranks candidates only; it never decides fiscal truth.""" import hashlib import json from difflib import SequenceMatcher from .fingerprint import normalize_description, token_stems from .retrieval import commercial_tokens, hybrid_candidates def _char_similarity(left, right): return SequenceMatcher(None, normalize_description(left), normalize_description(right)).ratio() def retrieval_v3_candidates(corpus, index, description, metadata=None, k=20, lexicon=None, exclude_product_id=None): """Add optional product evidence to V2 while preserving V2 as the safe baseline.""" metadata = metadata or {} base = hybrid_candidates(corpus, index, description, k=max(k, 50), lexicon=lexicon, exclude_product_id=exclude_product_id) gtin_ncm = str(metadata.get("gtin_ncm") or "") group_ncms = {str(value) for value in (metadata.get("group_ncms") or [])} rarity = metadata.get("token_rarity") or {} query_stems = commercial_tokens(description) rows = [] for code, text, score, official_score, kb_score, tenant_count in base: evidence = 0.0 if gtin_ncm and code == gtin_ncm: evidence += 1.0 if code in group_ncms: evidence += 0.05 matched_rarity = sum(float(rarity.get(token, 1.0)) for token in query_stems if token in token_stems(text)) evidence += min(0.05, matched_rarity / 100.0) rows.append((code, text, round(score + evidence, 4), official_score, kb_score, tenant_count)) rows.sort(key=lambda row: (-row[2], row[0])) return rows[:k] def candidate_fingerprint(payload): return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest() def prepare_candidate_set(work_item, candidates, *, dataset_versions=None, gtin_evidence=None, kb_neighbors=None, hierarchical_context=None): """Build the final LLM-ready evidence package without selecting a candidate.""" payload = { "task_type": work_item.task_type, "dedup_key": work_item.dedup_key, "normalized_product": normalize_description(work_item.payload.get("description", "")), "top20_candidate_ncms": [ {"ncm": row[0], "official_description": row[1], "score": row[2], "official_score": row[3], "kb_score": row[4], "tenant_count": row[5]} for row in candidates[:20] ], "official_descriptions": {row[0]: row[1] for row in candidates[:20]}, "hierarchical_context": hierarchical_context or {}, "kb_neighbors": kb_neighbors or [], "gtin_evidence": gtin_evidence or {}, "dataset_versions": dict(dataset_versions or {}), } payload["input_fingerprint"] = candidate_fingerprint(payload) return payload def compact_product_role_context(rows, limit=12): """Keep representative role evidence bounded; no role is inferred.""" unique = [] seen = set() for row in rows: description = normalize_description(row.get("description")) if not description or description in seen: continue seen.add(description) unique.append({"description": description, "group": row.get("group"), "subgroup": row.get("subgroup"), "operations": row.get("operations", [])}) if len(unique) >= limit: break return {"examples": unique, "evidence_only": True, "decision": None}