feat: complete pre-ai local completion v3
This commit is contained in:
OpenCode
@@ -0,0 +1,55 @@
|
||||
"""Offline-safe, ID-correlated provider batch execution."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .provider import ProviderTransportError
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchRunResult:
|
||||
results_by_work_item_id: dict[str, dict] = field(default_factory=dict)
|
||||
errors: dict[str, str] = field(default_factory=dict)
|
||||
provider_calls: int = 0
|
||||
retries: int = 0
|
||||
usage: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
def run_batches(items, provider, batch_size: int, max_retries: int = 2, *, scenario: str | None = None) -> BatchRunResult:
|
||||
if batch_size <= 0:
|
||||
raise ValueError("batch_size must be positive")
|
||||
if max_retries < 0:
|
||||
raise ValueError("max_retries must not be negative")
|
||||
|
||||
result = BatchRunResult()
|
||||
for start in range(0, len(items), batch_size):
|
||||
batch = list(items[start:start + batch_size])
|
||||
expected_ids = [str(item.work_item_id) for item in batch]
|
||||
answers = None
|
||||
for attempt in range(max_retries + 1):
|
||||
result.provider_calls += 1
|
||||
try:
|
||||
kwargs = {} if scenario is None else {"scenario": scenario}
|
||||
answers = provider.select_batch(batch, **kwargs)
|
||||
break
|
||||
except ProviderTransportError as error:
|
||||
if attempt == max_retries:
|
||||
for work_item_id in expected_ids:
|
||||
result.errors[work_item_id] = str(error)
|
||||
else:
|
||||
result.retries += 1
|
||||
except Exception as error:
|
||||
for work_item_id in expected_ids:
|
||||
result.errors[work_item_id] = str(error)
|
||||
break
|
||||
if answers is None:
|
||||
continue
|
||||
answer_ids = [str(answer.get("work_item_id")) for answer in answers]
|
||||
if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids):
|
||||
error = "provider response did not contain exactly one answer per work_item_id"
|
||||
for work_item_id in expected_ids:
|
||||
result.errors[work_item_id] = error
|
||||
continue
|
||||
result.results_by_work_item_id.update({str(answer["work_item_id"]): answer for answer in answers})
|
||||
|
||||
result.usage = dict(getattr(provider, "usage", {}) or {})
|
||||
return result
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Pure cache and dataset-freshness policies for offline and real evaluations."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
|
||||
VALID, NEEDS_REVALIDATION, CONFLICT = "VALID", "NEEDS_REVALIDATION", "CONFLICT"
|
||||
|
||||
|
||||
def cache_key(input_fingerprint, candidate_set_hash, model, prompt_version, dataset_versions):
|
||||
"""Return a stable key covering every input that can change an answer."""
|
||||
payload = {
|
||||
"input_fingerprint": str(input_fingerprint),
|
||||
"candidate_set_hash": str(candidate_set_hash),
|
||||
"model": str(model),
|
||||
"prompt_version": str(prompt_version),
|
||||
"dataset_versions": dict(sorted((dataset_versions or {}).items())),
|
||||
}
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def classify_knowledge_freshness(record_versions, active_versions):
|
||||
"""Classify versioned knowledge without deleting or rewriting history."""
|
||||
recorded = dict(record_versions or {})
|
||||
active = dict(active_versions or {})
|
||||
if any(key in recorded and key in active and recorded[key] != active[key] for key in recorded):
|
||||
return CONFLICT
|
||||
if recorded != active:
|
||||
return NEEDS_REVALIDATION
|
||||
return VALID
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Deterministic provider responses for offline pipeline tests."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from .llm_provider import LlmAnswerError, LlmEvalItem, validate_answer
|
||||
from .provider import ProviderContractError, ProviderTransportError
|
||||
|
||||
|
||||
SCENARIOS = frozenset({
|
||||
"CORRECT", "WRONG", "ABSTAIN", "INVALID_JSON", "MISSING_ITEM",
|
||||
"DUPLICATE_ITEM", "OUT_OF_CANDIDATE", "TIMEOUT", "429", "500",
|
||||
})
|
||||
|
||||
|
||||
class DeterministicFakeSemanticProvider:
|
||||
"""Return repeatable answers without persistence or network access."""
|
||||
|
||||
name = "deterministic-fake"
|
||||
model = "deterministic-fake-v1"
|
||||
prompt_version = "fake-provider-v1"
|
||||
|
||||
def __init__(self, *, shuffle: bool = False, fail_work_item_ids: Iterable[str] = (), failure_scenario: str = "TIMEOUT"):
|
||||
self.shuffle = shuffle
|
||||
self.fail_work_item_ids = {str(item_id) for item_id in fail_work_item_ids}
|
||||
self.failure_scenario = failure_scenario
|
||||
self.calls = 0
|
||||
self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
|
||||
|
||||
def select_batch(self, items: list[LlmEvalItem], scenario: str | None = None) -> list[dict]:
|
||||
scenario = scenario or "CORRECT"
|
||||
if scenario not in SCENARIOS:
|
||||
raise ValueError(f"unknown fake provider scenario: {scenario}")
|
||||
self.calls += 1
|
||||
if self.fail_work_item_ids and len(items) == 1 and str(items[0].work_item_id) in self.fail_work_item_ids:
|
||||
self._raise_transport(self.failure_scenario)
|
||||
if scenario in {"TIMEOUT", "429", "500"}:
|
||||
self._raise_transport(scenario)
|
||||
if scenario == "INVALID_JSON":
|
||||
raise LlmAnswerError("fake provider returned invalid JSON")
|
||||
|
||||
answers = [self._answer(item, scenario) for item in items]
|
||||
if scenario == "MISSING_ITEM" and answers:
|
||||
answers.pop()
|
||||
elif scenario == "DUPLICATE_ITEM" and answers:
|
||||
answers[-1] = dict(answers[0])
|
||||
if self.shuffle:
|
||||
answers.reverse()
|
||||
expected_ids = [str(item.work_item_id) for item in items]
|
||||
answer_ids = [str(answer.get("work_item_id")) for answer in answers]
|
||||
if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids):
|
||||
raise ProviderContractError("fake provider response did not contain exactly one answer per work_item_id")
|
||||
items_by_id = {str(item.work_item_id): item for item in items}
|
||||
try:
|
||||
return [validate_answer(answer, items_by_id[str(answer["work_item_id"])]) for answer in answers]
|
||||
except (LlmAnswerError, KeyError, TypeError) as error:
|
||||
raise ProviderContractError(str(error)) from error
|
||||
|
||||
@staticmethod
|
||||
def _answer(item: LlmEvalItem, scenario: str) -> dict:
|
||||
candidates = [str(candidate[0]) for candidate in item.candidates]
|
||||
selected = None
|
||||
abstain = scenario == "ABSTAIN"
|
||||
if not abstain and candidates:
|
||||
selected = "99999999" if scenario == "OUT_OF_CANDIDATE" else candidates[min(1, len(candidates) - 1) if scenario == "WRONG" else 0]
|
||||
return {
|
||||
"work_item_id": item.work_item_id,
|
||||
"selected_ncm": selected,
|
||||
"confidence": 0.0 if abstain else 0.75,
|
||||
"abstain": abstain or selected is None,
|
||||
"reason_short": f"fake scenario {scenario}",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _raise_transport(scenario: str) -> None:
|
||||
if scenario not in {"TIMEOUT", "429", "500"}:
|
||||
raise ValueError(f"unknown fake transport scenario: {scenario}")
|
||||
raise ProviderTransportError(f"fake provider transport failure: {scenario}")
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Conservative, explainable product identity and conflict classification."""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
from .fingerprint import normalize_description, token_stems, validate_gtin
|
||||
|
||||
GTIN_CONFLICT = "GTIN_CONFLICT"
|
||||
NCM_CONFLICT = "NCM_CONFLICT"
|
||||
DESCRIPTION_CONFLICT = "DESCRIPTION_CONFLICT"
|
||||
PACKAGING_VARIATION = "PACKAGING_VARIATION"
|
||||
FALSE_MATCH = "FALSE_MATCH"
|
||||
TENANT_CONTEXT_ONLY = "TENANT_CONTEXT_ONLY"
|
||||
|
||||
|
||||
def canonical_product_id(observation: dict) -> str:
|
||||
"""Stable global ID; fiscal and tenant fields are intentionally excluded."""
|
||||
gtin = validate_gtin(observation.get("gtin"))
|
||||
if gtin:
|
||||
return "GTIN:" + gtin
|
||||
values = "|".join((
|
||||
normalize_description(observation.get("normalized_description") or observation.get("description")),
|
||||
normalize_description(observation.get("brand")),
|
||||
normalize_description(observation.get("unit")),
|
||||
normalize_description(observation.get("package_quantity")),
|
||||
))
|
||||
return "FP:" + hashlib.sha256(values.encode()).hexdigest()
|
||||
|
||||
|
||||
def _description_equivalent(values):
|
||||
stems = [token_stems(value) for value in values if value]
|
||||
if not stems:
|
||||
return True
|
||||
return all(current == stems[0] or current.issubset(stems[0]) or stems[0].issubset(current) for current in stems[1:])
|
||||
|
||||
|
||||
def classify_conflict(observations: list[dict]) -> str:
|
||||
"""Classify only from structural evidence; ambiguous cases remain conflicts."""
|
||||
gtins = {validate_gtin(row.get("gtin")) for row in observations if validate_gtin(row.get("gtin"))}
|
||||
ncms = {str(row.get("ncm")) for row in observations if row.get("ncm")}
|
||||
descriptions = [row.get("normalized_description") or row.get("description") for row in observations]
|
||||
packages = {str(row.get("package_quantity")) for row in observations if row.get("package_quantity")}
|
||||
tenants = {str(obs.get("database") or obs.get("company_key")) for row in observations for obs in row.get("observations", [])}
|
||||
if len(gtins) > 1:
|
||||
return GTIN_CONFLICT
|
||||
if len(ncms) > 1:
|
||||
return NCM_CONFLICT
|
||||
if len(packages) > 1 and _description_equivalent(descriptions):
|
||||
return PACKAGING_VARIATION
|
||||
if len({normalize_description(value) for value in descriptions if value}) > 1:
|
||||
return DESCRIPTION_CONFLICT
|
||||
if len(tenants) > 1:
|
||||
return TENANT_CONTEXT_ONLY
|
||||
return FALSE_MATCH
|
||||
|
||||
|
||||
def audit_payload(payload: dict, historical_conflicts: int = 163) -> dict:
|
||||
"""Audit observable same-GTIN groups without silently resolving NCM conflicts."""
|
||||
by_gtin = defaultdict(list)
|
||||
for product in payload.get("products", []):
|
||||
gtin = validate_gtin(product.get("gtin"))
|
||||
if gtin:
|
||||
by_gtin[gtin].append(product)
|
||||
conflicts = []
|
||||
counts = defaultdict(int)
|
||||
for gtin, rows in by_gtin.items():
|
||||
if len({row.get("ncm") for row in rows if row.get("ncm")}) <= 1:
|
||||
continue
|
||||
kind = classify_conflict(rows)
|
||||
counts[kind] += 1
|
||||
conflicts.append({
|
||||
"identity": gtin,
|
||||
"classification": kind,
|
||||
"observations": len(rows),
|
||||
"ncms": sorted({row.get("ncm") for row in rows if row.get("ncm")}),
|
||||
"auto_resolution": "KEEP_KNOWLEDGE_CONFLICT",
|
||||
})
|
||||
return {
|
||||
"artifact": "KB_CONFLICT_AUDIT_V1",
|
||||
"historical_conflicts_reported": historical_conflicts,
|
||||
"observable_conflicts_audited": len(conflicts),
|
||||
"unobservable_historical_conflicts": max(0, historical_conflicts - len(conflicts)),
|
||||
"classification_counts": dict(sorted(counts.items())),
|
||||
"conflicts": conflicts,
|
||||
"policy": "only structurally equivalent identities may share a canonical product; tenant fiscal context is never global knowledge",
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Deterministic prompt compaction that preserves candidate evidence."""
|
||||
|
||||
import json
|
||||
|
||||
|
||||
def estimate_tokens(payload) -> int:
|
||||
return max(1, len(json.dumps(payload, ensure_ascii=False, sort_keys=True)) // 4)
|
||||
|
||||
|
||||
def compact_payload(payload):
|
||||
"""Remove repeated metadata only; candidate codes/descriptions are untouched."""
|
||||
compact = dict(payload)
|
||||
compact.pop("debug", None)
|
||||
compact.pop("duplicate_examples", None)
|
||||
candidates = compact.get("official_candidates")
|
||||
if isinstance(candidates, list):
|
||||
seen = set()
|
||||
compact["official_candidates"] = [
|
||||
row for row in candidates
|
||||
if not (row.get("ncm") in seen or seen.add(row.get("ncm")))
|
||||
]
|
||||
return compact
|
||||
|
||||
|
||||
def measure_payloads(payloads):
|
||||
before = sum(estimate_tokens(payload) for payload in payloads)
|
||||
after = sum(estimate_tokens(compact_payload(payload)) for payload in payloads)
|
||||
return {"items": len(payloads), "tokens_before": before, "tokens_after": after, "reduction_percent": round((1 - after / before) * 100, 2) if before else 0.0}
|
||||
@@ -18,6 +18,14 @@ class SemanticOutputError(ValueError):
|
||||
"""Raised when a provider answer violates the strict structured-output schema."""
|
||||
|
||||
|
||||
class ProviderTransportError(RuntimeError):
|
||||
"""A retryable failure while contacting a semantic provider."""
|
||||
|
||||
|
||||
class ProviderContractError(ValueError):
|
||||
"""A non-retryable provider response or protocol violation."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class StructuredResult:
|
||||
task_type: str
|
||||
@@ -137,3 +145,11 @@ class BudgetGuard:
|
||||
if plan.get("estimated_cost") is None:
|
||||
return allow_unknown_cost
|
||||
return plan["estimated_cost"] <= self.max_run_cost_usd
|
||||
|
||||
def authorize_usage(self, input_tokens: int, output_tokens: int, *, prior_cost_usd: float = 0.0,
|
||||
allow_unknown_cost: bool = False) -> bool:
|
||||
"""Apply the same guard to actual usage before accepting a provider result."""
|
||||
plan = self.estimate(input_tokens, output_tokens)
|
||||
if plan.get("estimated_cost") is None:
|
||||
return allow_unknown_cost
|
||||
return float(prior_cost_usd) + plan["estimated_cost"] <= self.max_run_cost_usd
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Pure, review-first questionnaire helpers for local fiscal context."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
|
||||
from central.app.context_keys import validate_entry
|
||||
|
||||
|
||||
def _options(row):
|
||||
value = row.get("VALID_OPTIONS", [])
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
value = [x.strip() for x in value.split("|") if x.strip()]
|
||||
return list(value) if isinstance(value, (list, tuple, set)) else [value]
|
||||
|
||||
|
||||
def build_questionnaire(rows):
|
||||
result = []
|
||||
for row in rows:
|
||||
scope = str(row.get("SCOPE", row.get("scope", ""))).strip().upper()
|
||||
key = str(row.get("KEY", row.get("key", ""))).strip()
|
||||
scope_key = str(row.get("GROUP") or row.get("NCM") or row.get("scope_key") or "").strip()
|
||||
result.append({
|
||||
"question_id": str(row.get("QUESTION_ID", row.get("question_id", ""))).strip(),
|
||||
"database": str(row.get("DATABASE", row.get("database", ""))).strip(),
|
||||
"scope": scope,
|
||||
"scope_key": scope_key if scope != "COMPANY" else "",
|
||||
"group": str(row.get("GROUP") or "").strip(),
|
||||
"key": key,
|
||||
"question": row.get("QUESTION", row.get("question", "")),
|
||||
"options": _options(row),
|
||||
"affected": {"products": int(row.get("PRODUCTS_AFFECTED") or row.get("products_affected") or 0), "cells": int(row.get("CELLS") or row.get("cells") or 0)},
|
||||
"examples": row.get("EXAMPLES", row.get("examples", "")),
|
||||
"consequence": row.get("WHAT_IT_WILL_RESOLVE", row.get("consequence", "")),
|
||||
})
|
||||
return sorted(result, key=lambda q: (q["database"], q["scope"], q["scope_key"], q["key"], q["question_id"]))
|
||||
|
||||
|
||||
def group_questions(questions):
|
||||
"""Group equivalent human decisions while retaining every affected question ID."""
|
||||
groups = {}
|
||||
for question in questions:
|
||||
key = (question["database"], question["group"], question["scope_key"], question["key"], question["question"], tuple(question["options"]))
|
||||
group = groups.setdefault(key, {
|
||||
"group_id": "GROUP-" + str(len(groups) + 1),
|
||||
"database": question["database"],
|
||||
"group": question["group"],
|
||||
"scope": question["scope"],
|
||||
"scope_key": question["scope_key"],
|
||||
"key": question["key"],
|
||||
"question": question["question"],
|
||||
"options": question["options"],
|
||||
"question_ids": [],
|
||||
"affected": {"products": 0, "cells": 0},
|
||||
"examples": [],
|
||||
"consequence": question["consequence"],
|
||||
})
|
||||
group["question_ids"].append(question["question_id"])
|
||||
group["affected"]["products"] += question["affected"]["products"]
|
||||
group["affected"]["cells"] += question["affected"]["cells"]
|
||||
if question["examples"] and len(group["examples"]) < 5:
|
||||
group["examples"].append(str(question["examples"]))
|
||||
return list(groups.values())
|
||||
|
||||
|
||||
def validate_answers(questions, answers):
|
||||
entries, effects, rollback_plan = [], [], []
|
||||
by_id = {q["question_id"]: q for q in questions}
|
||||
for question_id, value in answers.items():
|
||||
if question_id not in by_id:
|
||||
raise ValueError(f"unknown question {question_id!r}; nothing was persisted")
|
||||
q = by_id[question_id]
|
||||
value = str(value).strip()
|
||||
if q["options"] and value not in q["options"]:
|
||||
raise ValueError(f"{question_id}: value must be one of {q['options']}; nothing was persisted")
|
||||
entry, error = validate_entry({"scope": q["scope"], "scope_key": q["scope_key"], "key": q["key"], "value": value})
|
||||
if error:
|
||||
raise ValueError(f"{question_id}: {error}; nothing was persisted")
|
||||
entry["company_key"] = q["database"]
|
||||
entries.append(entry)
|
||||
effect = {"question_id": question_id, "database": q["database"], "key": q["key"], "affected": q["affected"], "behavior": "context_entry"}
|
||||
if q["key"] == "entry_scope" and value == "MIXED_OR_DYNAMIC":
|
||||
effect["behavior"] = "no_static_cfop_entrada"
|
||||
effects.append(effect)
|
||||
if q["key"] == "store_ipi_profile" and value == "LEAVE_NOT_APPLICABLE":
|
||||
rollback_plan.append({"database": q["database"], "field": "IPI", "affected_cells": q["affected"]["cells"]})
|
||||
return {"entries": entries, "effects": effects, "rollback_plan": rollback_plan}
|
||||
|
||||
|
||||
def render_json(result):
|
||||
return json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
|
||||
|
||||
def render_csv(result):
|
||||
output = io.StringIO()
|
||||
writer = csv.DictWriter(output, fieldnames=["company_key", "scope", "scope_key", "key", "value", "reason"])
|
||||
writer.writeheader()
|
||||
writer.writerows({k: entry.get(k, "") for k in writer.fieldnames} for entry in result["entries"])
|
||||
return output.getvalue()
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Retrieval V3 evidence layer. It ranks candidates only; it never decides fiscal truth."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
from .fingerprint import normalize_description, token_stems
|
||||
from .retrieval import commercial_tokens, hybrid_candidates
|
||||
|
||||
|
||||
def _char_similarity(left, right):
|
||||
return SequenceMatcher(None, normalize_description(left), normalize_description(right)).ratio()
|
||||
|
||||
|
||||
def retrieval_v3_candidates(corpus, index, description, metadata=None, k=20, lexicon=None, exclude_product_id=None):
|
||||
"""Add optional product evidence to V2 while preserving V2 as the safe baseline."""
|
||||
metadata = metadata or {}
|
||||
base = hybrid_candidates(corpus, index, description, k=max(k, 50), lexicon=lexicon, exclude_product_id=exclude_product_id)
|
||||
gtin_ncm = str(metadata.get("gtin_ncm") or "")
|
||||
group_ncms = {str(value) for value in (metadata.get("group_ncms") or [])}
|
||||
rarity = metadata.get("token_rarity") or {}
|
||||
query_stems = commercial_tokens(description)
|
||||
rows = []
|
||||
for code, text, score, official_score, kb_score, tenant_count in base:
|
||||
evidence = 0.0
|
||||
if gtin_ncm and code == gtin_ncm:
|
||||
evidence += 1.0
|
||||
if code in group_ncms:
|
||||
evidence += 0.05
|
||||
matched_rarity = sum(float(rarity.get(token, 1.0)) for token in query_stems if token in token_stems(text))
|
||||
evidence += min(0.05, matched_rarity / 100.0)
|
||||
rows.append((code, text, round(score + evidence, 4), official_score, kb_score, tenant_count))
|
||||
rows.sort(key=lambda row: (-row[2], row[0]))
|
||||
return rows[:k]
|
||||
|
||||
|
||||
def candidate_fingerprint(payload):
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def prepare_candidate_set(work_item, candidates, *, dataset_versions=None, gtin_evidence=None, kb_neighbors=None, hierarchical_context=None):
|
||||
"""Build the final LLM-ready evidence package without selecting a candidate."""
|
||||
payload = {
|
||||
"task_type": work_item.task_type,
|
||||
"dedup_key": work_item.dedup_key,
|
||||
"normalized_product": normalize_description(work_item.payload.get("description", "")),
|
||||
"top20_candidate_ncms": [
|
||||
{"ncm": row[0], "official_description": row[1], "score": row[2], "official_score": row[3], "kb_score": row[4], "tenant_count": row[5]}
|
||||
for row in candidates[:20]
|
||||
],
|
||||
"official_descriptions": {row[0]: row[1] for row in candidates[:20]},
|
||||
"hierarchical_context": hierarchical_context or {},
|
||||
"kb_neighbors": kb_neighbors or [],
|
||||
"gtin_evidence": gtin_evidence or {},
|
||||
"dataset_versions": dict(dataset_versions or {}),
|
||||
}
|
||||
payload["input_fingerprint"] = candidate_fingerprint(payload)
|
||||
return payload
|
||||
|
||||
|
||||
def compact_product_role_context(rows, limit=12):
|
||||
"""Keep representative role evidence bounded; no role is inferred."""
|
||||
unique = []
|
||||
seen = set()
|
||||
for row in rows:
|
||||
description = normalize_description(row.get("description"))
|
||||
if not description or description in seen:
|
||||
continue
|
||||
seen.add(description)
|
||||
unique.append({"description": description, "group": row.get("group"), "subgroup": row.get("subgroup"), "operations": row.get("operations", [])})
|
||||
if len(unique) >= limit:
|
||||
break
|
||||
return {"examples": unique, "evidence_only": True, "decision": None}
|
||||
@@ -21,9 +21,10 @@ class WorkItem:
|
||||
kb_product_id: int | None = None
|
||||
|
||||
|
||||
def build_work_items(questions, product_fingerprints=None) -> list[WorkItem]:
|
||||
def build_work_items(questions, product_fingerprints=None, canonical_products=None) -> list[WorkItem]:
|
||||
"""`product_fingerprints` maps "{DATABASE}:{erp_code}" -> KB fingerprint (from the seed pass)."""
|
||||
product_fingerprints = product_fingerprints or {}
|
||||
canonical_products = canonical_products or {}
|
||||
items: dict = {}
|
||||
|
||||
def _get(task_type, dedup_key, ref, cells):
|
||||
@@ -50,6 +51,7 @@ def build_work_items(questions, product_fingerprints=None) -> list[WorkItem]:
|
||||
database = question.get("DATABASE", "")
|
||||
erp_code = str(question.get("DATABASE_CODE") or "").strip()
|
||||
fingerprint = product_fingerprints.get(f"{database}:{erp_code}")
|
||||
fingerprint = canonical_products.get(f"{database}:{erp_code}", fingerprint)
|
||||
dedup = f"invalid_ncm:{fingerprint}" if fingerprint else f"invalid_ncm:{database}:{erp_code}"
|
||||
item = _get("INVALID_NCM", dedup, ref, cells)
|
||||
item.payload.setdefault("erp_code", erp_code)
|
||||
@@ -58,7 +60,8 @@ def build_work_items(questions, product_fingerprints=None) -> list[WorkItem]:
|
||||
company = question.get("COMPANY", "")
|
||||
group = str(question.get("GROUP") or "").strip()
|
||||
if group:
|
||||
item = _get("PRODUCT_ROLE", f"product_role:{company}:{group}", ref, cells)
|
||||
normalized_group = " ".join(str(group).upper().split())
|
||||
item = _get("PRODUCT_ROLE", f"product_role:{company}:{normalized_group}", ref, cells)
|
||||
item.payload.setdefault("company_key", company)
|
||||
item.payload["group"] = group
|
||||
return list(items.values())
|
||||
|
||||
Reference in New Issue
Block a user