74 lines
3.3 KiB
Python
74 lines
3.3 KiB
Python
"""Retrieval V3 evidence layer. It ranks candidates only; it never decides fiscal truth."""
|
|
|
|
import hashlib
|
|
import json
|
|
from difflib import SequenceMatcher
|
|
|
|
from .fingerprint import normalize_description, token_stems
|
|
from .retrieval import commercial_tokens, hybrid_candidates
|
|
|
|
|
|
def _char_similarity(left, right):
|
|
return SequenceMatcher(None, normalize_description(left), normalize_description(right)).ratio()
|
|
|
|
|
|
def retrieval_v3_candidates(corpus, index, description, metadata=None, k=20, lexicon=None, exclude_product_id=None):
|
|
"""Add optional product evidence to V2 while preserving V2 as the safe baseline."""
|
|
metadata = metadata or {}
|
|
base = hybrid_candidates(corpus, index, description, k=max(k, 50), lexicon=lexicon, exclude_product_id=exclude_product_id)
|
|
gtin_ncm = str(metadata.get("gtin_ncm") or "")
|
|
group_ncms = {str(value) for value in (metadata.get("group_ncms") or [])}
|
|
rarity = metadata.get("token_rarity") or {}
|
|
query_stems = commercial_tokens(description)
|
|
rows = []
|
|
for code, text, score, official_score, kb_score, tenant_count in base:
|
|
evidence = 0.0
|
|
if gtin_ncm and code == gtin_ncm:
|
|
evidence += 1.0
|
|
if code in group_ncms:
|
|
evidence += 0.05
|
|
matched_rarity = sum(float(rarity.get(token, 1.0)) for token in query_stems if token in token_stems(text))
|
|
evidence += min(0.05, matched_rarity / 100.0)
|
|
rows.append((code, text, round(score + evidence, 4), official_score, kb_score, tenant_count))
|
|
rows.sort(key=lambda row: (-row[2], row[0]))
|
|
return rows[:k]
|
|
|
|
|
|
def candidate_fingerprint(payload):
|
|
return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest()
|
|
|
|
|
|
def prepare_candidate_set(work_item, candidates, *, dataset_versions=None, gtin_evidence=None, kb_neighbors=None, hierarchical_context=None):
|
|
"""Build the final LLM-ready evidence package without selecting a candidate."""
|
|
payload = {
|
|
"task_type": work_item.task_type,
|
|
"dedup_key": work_item.dedup_key,
|
|
"normalized_product": normalize_description(work_item.payload.get("description", "")),
|
|
"top20_candidate_ncms": [
|
|
{"ncm": row[0], "official_description": row[1], "score": row[2], "official_score": row[3], "kb_score": row[4], "tenant_count": row[5]}
|
|
for row in candidates[:20]
|
|
],
|
|
"official_descriptions": {row[0]: row[1] for row in candidates[:20]},
|
|
"hierarchical_context": hierarchical_context or {},
|
|
"kb_neighbors": kb_neighbors or [],
|
|
"gtin_evidence": gtin_evidence or {},
|
|
"dataset_versions": dict(dataset_versions or {}),
|
|
}
|
|
payload["input_fingerprint"] = candidate_fingerprint(payload)
|
|
return payload
|
|
|
|
|
|
def compact_product_role_context(rows, limit=12):
|
|
"""Keep representative role evidence bounded; no role is inferred."""
|
|
unique = []
|
|
seen = set()
|
|
for row in rows:
|
|
description = normalize_description(row.get("description"))
|
|
if not description or description in seen:
|
|
continue
|
|
seen.add(description)
|
|
unique.append({"description": description, "group": row.get("group"), "subgroup": row.get("subgroup"), "operations": row.get("operations", [])})
|
|
if len(unique) >= limit:
|
|
break
|
|
return {"examples": unique, "evidence_only": True, "decision": None}
|