Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
This commit is contained in:
gti
@@ -0,0 +1,8 @@
|
||||
"""AI semantic layer V1 — NOT a fiscal authority.
|
||||
|
||||
Rules frozen by the operator contract:
|
||||
- AI only selects among official candidates; it never invents codes.
|
||||
- Cache first: KB exact/fingerprint hits never trigger AI calls.
|
||||
- No AI for origin, ST retention, or company-level decisions.
|
||||
- AI_WRITE_ALLOWED=false in V1: results stay AI_CANDIDATE_VALIDATED.
|
||||
"""
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Offline-safe, ID-correlated provider batch execution."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .provider import ProviderTransportError
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchRunResult:
|
||||
results_by_work_item_id: dict[str, dict] = field(default_factory=dict)
|
||||
errors: dict[str, str] = field(default_factory=dict)
|
||||
provider_calls: int = 0
|
||||
retries: int = 0
|
||||
usage: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
def run_batches(items, provider, batch_size: int, max_retries: int = 2, *, scenario: str | None = None) -> BatchRunResult:
|
||||
if batch_size <= 0:
|
||||
raise ValueError("batch_size must be positive")
|
||||
if max_retries < 0:
|
||||
raise ValueError("max_retries must not be negative")
|
||||
|
||||
result = BatchRunResult()
|
||||
for start in range(0, len(items), batch_size):
|
||||
batch = list(items[start:start + batch_size])
|
||||
expected_ids = [str(item.work_item_id) for item in batch]
|
||||
answers = None
|
||||
for attempt in range(max_retries + 1):
|
||||
result.provider_calls += 1
|
||||
try:
|
||||
kwargs = {} if scenario is None else {"scenario": scenario}
|
||||
answers = provider.select_batch(batch, **kwargs)
|
||||
break
|
||||
except ProviderTransportError as error:
|
||||
if attempt == max_retries:
|
||||
for work_item_id in expected_ids:
|
||||
result.errors[work_item_id] = str(error)
|
||||
else:
|
||||
result.retries += 1
|
||||
except Exception as error:
|
||||
for work_item_id in expected_ids:
|
||||
result.errors[work_item_id] = str(error)
|
||||
break
|
||||
if answers is None:
|
||||
continue
|
||||
answer_ids = [str(answer.get("work_item_id")) for answer in answers]
|
||||
if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids):
|
||||
error = "provider response did not contain exactly one answer per work_item_id"
|
||||
for work_item_id in expected_ids:
|
||||
result.errors[work_item_id] = error
|
||||
continue
|
||||
result.results_by_work_item_id.update({str(answer["work_item_id"]): answer for answer in answers})
|
||||
|
||||
result.usage = dict(getattr(provider, "usage", {}) or {})
|
||||
return result
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Persistent-boundary-compatible budget guard; provider invocation happens after this check."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from threading import Lock
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BudgetLimits:
|
||||
max_run_cost_usd: float = 0.54
|
||||
max_daily_cost_usd: float = 1.00
|
||||
max_monthly_cost_usd: float = 10.00
|
||||
max_requests_per_day: int = 100
|
||||
max_new_products_per_run: int = 750
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BudgetDecision:
|
||||
allowed: bool
|
||||
reason: str
|
||||
reserved_cost_usd: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UsageEntry:
|
||||
run_id: str
|
||||
tenant: str
|
||||
cost_usd: float
|
||||
requests: int
|
||||
new_products: int
|
||||
timestamp: str
|
||||
|
||||
|
||||
class BudgetLedger:
|
||||
"""Atomic in-process ledger with an explicit persistence boundary for Hub storage."""
|
||||
|
||||
def __init__(self, limits: BudgetLimits):
|
||||
self.limits = limits
|
||||
self.entries: list[UsageEntry] = []
|
||||
self._lock = Lock()
|
||||
|
||||
def check_and_reserve(self, run_id: str, tenant: str, cost_usd: float, requests: int, new_products: int) -> BudgetDecision:
|
||||
with self._lock:
|
||||
run_cost = sum(e.cost_usd for e in self.entries if e.run_id == run_id)
|
||||
day = datetime.now(timezone.utc).date()
|
||||
day_entries = [e for e in self.entries if e.timestamp[:10] == day.isoformat()]
|
||||
month_entries = [e for e in self.entries if e.timestamp[:7] == day.strftime("%Y-%m")]
|
||||
if run_cost + cost_usd > self.limits.max_run_cost_usd:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
if sum(e.cost_usd for e in day_entries) + cost_usd > self.limits.max_daily_cost_usd:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
if sum(e.cost_usd for e in month_entries) + cost_usd > self.limits.max_monthly_cost_usd:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
if sum(e.requests for e in day_entries) + requests > self.limits.max_requests_per_day:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
if sum(e.new_products for e in self.entries if e.run_id == run_id) + new_products > self.limits.max_new_products_per_run:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
self.entries.append(UsageEntry(run_id, tenant, cost_usd, requests, new_products, datetime.now(timezone.utc).isoformat()))
|
||||
return BudgetDecision(True, "RESERVED", cost_usd)
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Pure cache and dataset-freshness policies for offline and real evaluations."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
|
||||
VALID, NEEDS_REVALIDATION, CONFLICT = "VALID", "NEEDS_REVALIDATION", "CONFLICT"
|
||||
|
||||
|
||||
def cache_key(input_fingerprint, candidate_set_hash, model, prompt_version, dataset_versions):
|
||||
"""Return a stable key covering every input that can change an answer."""
|
||||
payload = {
|
||||
"input_fingerprint": str(input_fingerprint),
|
||||
"candidate_set_hash": str(candidate_set_hash),
|
||||
"model": str(model),
|
||||
"prompt_version": str(prompt_version),
|
||||
"dataset_versions": dict(sorted((dataset_versions or {}).items())),
|
||||
}
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def classify_knowledge_freshness(record_versions, active_versions):
|
||||
"""Classify versioned knowledge without deleting or rewriting history."""
|
||||
recorded = dict(record_versions or {})
|
||||
active = dict(active_versions or {})
|
||||
if any(key in recorded and key in active and recorded[key] != active[key] for key in recorded):
|
||||
return CONFLICT
|
||||
if recorded != active:
|
||||
return NEEDS_REVALIDATION
|
||||
return VALID
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Deterministic candidate generation from the ACTIVE official datasets.
|
||||
|
||||
The AI never receives an open question: for every task type the Hub first builds
|
||||
the official candidate list (NCM top-K by description, CEST links for the NCM,
|
||||
TIPI EX rows for the NCM). Providers may only pick from this list.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import CestEntry, CestNcmEntry, Dataset, NcmEntry, TipiEntry
|
||||
from .fingerprint import token_stems
|
||||
|
||||
|
||||
def _active_dataset(session: Session, kind: str):
|
||||
return session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE"))
|
||||
|
||||
|
||||
def ncm_top_k(session: Session, description, k: int = 10):
|
||||
"""Top-K official NCM codes by (stemmed) token overlap with the product description."""
|
||||
dataset = _active_dataset(session, "NCM")
|
||||
if dataset is None:
|
||||
return []
|
||||
stems = token_stems(description)
|
||||
if not stems:
|
||||
return []
|
||||
scored = []
|
||||
for entry in session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all():
|
||||
score = len(stems & token_stems(entry.description)) / len(stems)
|
||||
if score > 0:
|
||||
scored.append((entry.code, entry.description, round(score, 4)))
|
||||
scored.sort(key=lambda row: (-row[2], row[0]))
|
||||
return scored[:k]
|
||||
|
||||
|
||||
def cest_candidates_for_ncm(session: Session, ncm: str):
|
||||
dataset = _active_dataset(session, "CEST")
|
||||
if dataset is None:
|
||||
return []
|
||||
rows = session.execute(
|
||||
select(CestEntry, CestNcmEntry.ncm_code)
|
||||
.join(CestNcmEntry, CestNcmEntry.cest_entry_id == CestEntry.id)
|
||||
.where(CestEntry.dataset_id == dataset.id)
|
||||
).all()
|
||||
return list(dict.fromkeys(entry for entry, ncm_code in rows if ncm.startswith(ncm_code)))
|
||||
|
||||
|
||||
def ex_tipi_candidates_for_ncm(session: Session, ncm: str):
|
||||
dataset = _active_dataset(session, "TIPI_IPI")
|
||||
if dataset is None:
|
||||
return []
|
||||
return list(dict.fromkeys(
|
||||
entry for entry in session.scalars(select(TipiEntry).where(TipiEntry.dataset_id == dataset.id, TipiEntry.ncm == ncm, TipiEntry.ex.is_not(None))).all()
|
||||
))
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Metrics and error attribution for the real LLM NCM evaluation (sections 8-9)."""
|
||||
|
||||
|
||||
def metrics(rows) -> dict:
|
||||
"""rows: [{gold_in_candidates, selected, gold, confidence, invalid}]"""
|
||||
total = len(rows)
|
||||
correct = sum(1 for row in rows if not row["invalid"] and row["selected"] == row["gold"])
|
||||
wrong = sum(1 for row in rows if not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"])
|
||||
abstain = sum(1 for row in rows if not row["invalid"] and row["selected"] is None)
|
||||
invalid = sum(1 for row in rows if row["invalid"])
|
||||
gold_in = sum(1 for row in rows if row["gold_in_candidates"])
|
||||
answered = correct + wrong
|
||||
high_conf = [row for row in rows if not row["invalid"] and row["confidence"] >= 0.95 and row["selected"] is not None]
|
||||
high_conf_wrong = sum(1 for row in high_conf if row["selected"] != row["gold"])
|
||||
return {
|
||||
"TOTAL": total,
|
||||
"CORRECT": correct,
|
||||
"WRONG": wrong,
|
||||
"ABSTAIN": abstain,
|
||||
"INVALID": invalid,
|
||||
"GOLD_IN_CANDIDATES": gold_in,
|
||||
"ACCURACY_ALL": round(correct / total, 4) if total else None,
|
||||
"SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE": round(correct / answered, 4) if answered else None,
|
||||
"PRECISION_WHEN_HIGH_CONFIDENCE": round(sum(1 for row in high_conf if row["selected"] == row["gold"]) / len(high_conf), 4) if high_conf else None,
|
||||
"HIGH_CONFIDENCE_WRONG_RATE": round(high_conf_wrong / len(high_conf), 4) if high_conf else None,
|
||||
"ABSTENTION_RATE": round(abstain / total, 4) if total else None,
|
||||
}
|
||||
|
||||
|
||||
def attribution(rows) -> dict:
|
||||
return {
|
||||
"GOLD_NOT_IN_CANDIDATES": sum(1 for row in rows if not row["gold_in_candidates"]),
|
||||
"GOLD_IN_CANDIDATES_BUT_WRONG": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"]),
|
||||
"GOLD_IN_CANDIDATES_BUT_ABSTAIN": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is None),
|
||||
"INVALID_OUTPUT": sum(1 for row in rows if row["invalid"]),
|
||||
}
|
||||
|
||||
|
||||
_BUCKETS = ((">=0.99", 0.99, 1.01), ("0.95-0.9899", 0.95, 0.99), ("0.90-0.9499", 0.90, 0.95), ("0.80-0.8999", 0.80, 0.90), ("<0.80", -0.01, 0.80))
|
||||
|
||||
|
||||
def confidence_buckets(rows) -> dict:
|
||||
out = {name: {"total": 0, "correct": 0} for name, _, _ in _BUCKETS}
|
||||
for row in rows:
|
||||
if row["invalid"] or row["selected"] is None:
|
||||
continue
|
||||
for name, low, high in _BUCKETS:
|
||||
if low <= row["confidence"] < high:
|
||||
out[name]["total"] += 1
|
||||
if row["selected"] == row["gold"]:
|
||||
out[name]["correct"] += 1
|
||||
break
|
||||
return out
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Deterministic provider responses for offline pipeline tests."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from .llm_provider import LlmAnswerError, LlmEvalItem, validate_answer
|
||||
from .provider import ProviderContractError, ProviderTransportError
|
||||
|
||||
|
||||
SCENARIOS = frozenset({
|
||||
"CORRECT", "WRONG", "ABSTAIN", "INVALID_JSON", "MISSING_ITEM",
|
||||
"DUPLICATE_ITEM", "OUT_OF_CANDIDATE", "TIMEOUT", "429", "500",
|
||||
})
|
||||
|
||||
|
||||
class DeterministicFakeSemanticProvider:
|
||||
"""Return repeatable answers without persistence or network access."""
|
||||
|
||||
name = "deterministic-fake"
|
||||
model = "deterministic-fake-v1"
|
||||
prompt_version = "fake-provider-v1"
|
||||
|
||||
def __init__(self, *, shuffle: bool = False, fail_work_item_ids: Iterable[str] = (), failure_scenario: str = "TIMEOUT"):
|
||||
self.shuffle = shuffle
|
||||
self.fail_work_item_ids = {str(item_id) for item_id in fail_work_item_ids}
|
||||
self.failure_scenario = failure_scenario
|
||||
self.calls = 0
|
||||
self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
|
||||
|
||||
def select_batch(self, items: list[LlmEvalItem], scenario: str | None = None) -> list[dict]:
|
||||
scenario = scenario or "CORRECT"
|
||||
if scenario not in SCENARIOS:
|
||||
raise ValueError(f"unknown fake provider scenario: {scenario}")
|
||||
self.calls += 1
|
||||
if self.fail_work_item_ids and len(items) == 1 and str(items[0].work_item_id) in self.fail_work_item_ids:
|
||||
self._raise_transport(self.failure_scenario)
|
||||
if scenario in {"TIMEOUT", "429", "500"}:
|
||||
self._raise_transport(scenario)
|
||||
if scenario == "INVALID_JSON":
|
||||
raise LlmAnswerError("fake provider returned invalid JSON")
|
||||
|
||||
answers = [self._answer(item, scenario) for item in items]
|
||||
if scenario == "MISSING_ITEM" and answers:
|
||||
answers.pop()
|
||||
elif scenario == "DUPLICATE_ITEM" and answers:
|
||||
answers[-1] = dict(answers[0])
|
||||
if self.shuffle:
|
||||
answers.reverse()
|
||||
expected_ids = [str(item.work_item_id) for item in items]
|
||||
answer_ids = [str(answer.get("work_item_id")) for answer in answers]
|
||||
if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids):
|
||||
raise ProviderContractError("fake provider response did not contain exactly one answer per work_item_id")
|
||||
items_by_id = {str(item.work_item_id): item for item in items}
|
||||
try:
|
||||
return [validate_answer(answer, items_by_id[str(answer["work_item_id"])]) for answer in answers]
|
||||
except (LlmAnswerError, KeyError, TypeError) as error:
|
||||
raise ProviderContractError(str(error)) from error
|
||||
|
||||
@staticmethod
|
||||
def _answer(item: LlmEvalItem, scenario: str) -> dict:
|
||||
candidates = [str(candidate[0]) for candidate in item.candidates]
|
||||
selected = None
|
||||
abstain = scenario == "ABSTAIN"
|
||||
if not abstain and candidates:
|
||||
selected = "99999999" if scenario == "OUT_OF_CANDIDATE" else candidates[min(1, len(candidates) - 1) if scenario == "WRONG" else 0]
|
||||
return {
|
||||
"work_item_id": item.work_item_id,
|
||||
"selected_ncm": selected,
|
||||
"confidence": 0.0 if abstain else 0.75,
|
||||
"abstain": abstain or selected is None,
|
||||
"reason_short": f"fake scenario {scenario}",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _raise_transport(scenario: str) -> None:
|
||||
if scenario not in {"TIMEOUT", "429", "500"}:
|
||||
raise ValueError(f"unknown fake transport scenario: {scenario}")
|
||||
raise ProviderTransportError(f"fake provider transport failure: {scenario}")
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Product identity: stable fingerprints and GTIN validation.
|
||||
|
||||
Global identity uses ONLY stable attributes (normalized description, brand, unit,
|
||||
package quantity, GTIN). Fiscal/company context (CFOP, CSOSN, CST, rate, ST, tenant)
|
||||
never participates: those vary per context and must not merge distinct knowledge.
|
||||
"""
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from ..classification import ENGINE_VERSION # noqa: F401 (re-exported for the AI layer)
|
||||
|
||||
|
||||
def normalize_description(value) -> str:
|
||||
"""NFKD -> ascii -> A-Z0-9 tokens, space-joined (same canonical form as classification)."""
|
||||
text = unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper()
|
||||
return " ".join(re.sub(r"[^A-Z0-9]+", " ", text).split())
|
||||
|
||||
|
||||
def _normalize_token(value) -> str:
|
||||
return normalize_description(value)
|
||||
|
||||
|
||||
def _normalize_quantity(value) -> str:
|
||||
digits = re.sub(r"[^0-9.,]", "", str(value or ""))
|
||||
if not digits:
|
||||
return ""
|
||||
try:
|
||||
number = float(digits.replace(",", "."))
|
||||
except ValueError:
|
||||
return digits
|
||||
return f"{number:g}"
|
||||
|
||||
|
||||
def product_fingerprint(description, brand=None, unit=None, package_quantity=None) -> str:
|
||||
parts = [_normalize_token(description), _normalize_token(brand), _normalize_token(unit), _normalize_quantity(package_quantity)]
|
||||
return hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _stem(token: str) -> str:
|
||||
"""Minimal Portuguese plural stemming so AGUAS matches AGUA / MINERAIS matches MINERAL."""
|
||||
if len(token) > 4 and token.endswith("AIS"):
|
||||
return token[:-3] + "AL"
|
||||
if len(token) > 4 and token.endswith("ES"):
|
||||
return token[:-2]
|
||||
if len(token) > 3 and token.endswith("S"):
|
||||
return token[:-1]
|
||||
return token
|
||||
|
||||
|
||||
def token_stems(value) -> set:
|
||||
return {_stem(token) for token in normalize_description(value).split()}
|
||||
|
||||
|
||||
def overlap_score(query, official) -> float:
|
||||
"""Fraction of query description stems present in the official description stems."""
|
||||
stems = token_stems(query)
|
||||
if not stems:
|
||||
return 0.0
|
||||
return len(stems & token_stems(official)) / len(stems)
|
||||
|
||||
|
||||
def validate_gtin(value) -> str | None:
|
||||
"""Returns the canonical GTIN when the checksum is valid, else None.
|
||||
Accepts GTIN-8/12/13/14 (EAN-8/UPC-12/EAN-13/GTIN-14)."""
|
||||
digits = re.sub(r"\D", "", str(value or ""))
|
||||
if len(digits) not in (8, 12, 13, 14):
|
||||
return None
|
||||
total = 0
|
||||
for index, char in enumerate(reversed(digits)):
|
||||
digit = int(char)
|
||||
total += digit * (1 if index % 2 == 0 else 3)
|
||||
if total % 10 != 0:
|
||||
return None
|
||||
return digits
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Evidence reconstruction for historical KB conflicts.
|
||||
|
||||
The seed payload contains the current 90 observable conflicts but not the
|
||||
identity/evidence rows for the remaining 73 historical conflict records. This
|
||||
module makes that limitation explicit instead of fabricating identities.
|
||||
"""
|
||||
|
||||
from collections import Counter
|
||||
|
||||
RESOLVED_EQUIVALENT_PRODUCT = "RESOLVED_EQUIVALENT_PRODUCT"
|
||||
FALSE_MATCH = "FALSE_MATCH"
|
||||
VALID_NCM_CONFLICT = "VALID_NCM_CONFLICT"
|
||||
STALE_OBSERVATION = "STALE_OBSERVATION"
|
||||
INSUFFICIENT_EVIDENCE = "INSUFFICIENT_EVIDENCE"
|
||||
|
||||
|
||||
def reconstruct_historical_conflicts(payload: dict, historical_count: int = 73) -> dict:
|
||||
products = payload.get("products", [])
|
||||
gtin_count = sum(1 for product in products if product.get("gtin"))
|
||||
fingerprint_count = sum(1 for product in products if product.get("fingerprint"))
|
||||
evidence_summary = {
|
||||
"payload_products": len(products),
|
||||
"payload_gtin_products": gtin_count,
|
||||
"payload_fingerprint_products": fingerprint_count,
|
||||
"fields_available": ["gtin", "fingerprint", "normalized_description", "unit", "ncm", "observations"],
|
||||
"historical_identity_records_found": 0,
|
||||
}
|
||||
rows = []
|
||||
for index in range(1, historical_count + 1):
|
||||
rows.append({
|
||||
"historical_conflict_id": f"HISTORICAL-{index:03d}",
|
||||
"classification": INSUFFICIENT_EVIDENCE,
|
||||
"evidence_search": evidence_summary,
|
||||
"resolution": "PRESERVE_CONFLICT",
|
||||
"reason": "original conflict identity/evidence row is absent from available artifacts; no NCM selected",
|
||||
})
|
||||
return {
|
||||
"artifact": "KB_HISTORICAL_CONFLICT_AUDIT_V1",
|
||||
"historical_conflicts_before": historical_count,
|
||||
"historical_conflicts_resolved": 0,
|
||||
"historical_conflicts_remaining": historical_count,
|
||||
"classification_counts": dict(Counter(row["classification"] for row in rows)),
|
||||
"conflicts": rows,
|
||||
"current_observable_conflicts_preserved": 90,
|
||||
"current_conflict_policy": "NCM_CONFLICT rows remain blocked; no majority resolution",
|
||||
"evidence_complete": False,
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Product Knowledge Base service: identity resolution, versioned knowledge, conflicts.
|
||||
|
||||
Identity order (operator contract): 1. exact GTIN; 2. strong fingerprint; 3. semantic
|
||||
fingerprint candidate; 4. unknown. Conflicting knowledge for the same key is kept
|
||||
side by side and surfaced as KNOWLEDGE_CONFLICT — never picked by majority.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func as sa_func, select
|
||||
from ..models import KbKnowledge, KbObservation, KbProduct
|
||||
|
||||
EXACT, FINGERPRINT = "EXACT", "FINGERPRINT"
|
||||
_AUTHORITATIVE = {"OFFICIAL_DATASET", "OPERATOR", "AI_PLUS_OFFICIAL_VALIDATION"}
|
||||
|
||||
|
||||
class KnowledgeBase:
|
||||
def __init__(self, session):
|
||||
self.session = session
|
||||
|
||||
# ----- identity -----
|
||||
|
||||
def upsert_product(self, fingerprint, gtin=None, normalized_description="", brand=None, unit=None, package_quantity=None, category=None):
|
||||
gtin = gtin or None
|
||||
product = None
|
||||
if gtin:
|
||||
product = self.session.scalar(select(KbProduct).where(KbProduct.gtin == gtin))
|
||||
if product is None:
|
||||
product = self.session.scalar(select(KbProduct).where(KbProduct.fingerprint == fingerprint))
|
||||
if product is None:
|
||||
product = KbProduct(fingerprint=fingerprint, normalized_description=normalized_description or "")
|
||||
self.session.add(product)
|
||||
# identity upgrades (never downgrades)
|
||||
if gtin and not product.gtin:
|
||||
product.gtin = gtin
|
||||
product.identity_kind = "GTIN"
|
||||
if normalized_description and not product.normalized_description:
|
||||
product.normalized_description = normalized_description
|
||||
for field, value in (("brand", brand), ("unit", unit), ("package_quantity", package_quantity), ("category", category)):
|
||||
if value and not getattr(product, field):
|
||||
setattr(product, field, value)
|
||||
self.session.flush()
|
||||
return product
|
||||
|
||||
def resolve(self, gtin=None, fingerprint=None):
|
||||
"""Returns (product|None, hit_kind). Cache-first: EXACT (GTIN) beats FINGERPRINT."""
|
||||
if gtin:
|
||||
product = self.session.scalar(select(KbProduct).where(KbProduct.gtin == gtin))
|
||||
if product:
|
||||
return product, EXACT
|
||||
if fingerprint:
|
||||
product = self.session.scalar(select(KbProduct).where(KbProduct.fingerprint == fingerprint))
|
||||
if product:
|
||||
return product, FINGERPRINT
|
||||
return None, None
|
||||
|
||||
# ----- knowledge -----
|
||||
|
||||
def add_knowledge(self, kb_product_id, knowledge_key, value, source, confidence=0.0, evidence=None, model=None, prompt_version=None, dataset_versions=None):
|
||||
value = str(value)
|
||||
row = self.session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == kb_product_id, KbKnowledge.knowledge_key == knowledge_key, KbKnowledge.value == value))
|
||||
if row is None:
|
||||
row = KbKnowledge(kb_product_id=kb_product_id, knowledge_key=knowledge_key, value=value, source=source, confidence=confidence, evidence=evidence or {}, model=model, prompt_version=prompt_version, dataset_versions=dataset_versions or {})
|
||||
self.session.add(row)
|
||||
else:
|
||||
# authoritative sources upgrade provenance/confidence of the same value; history preserved
|
||||
if source in _AUTHORITATIVE:
|
||||
row.source = source
|
||||
row.confidence = max(row.confidence, confidence)
|
||||
row.evidence = {**row.evidence, **(evidence or {})}
|
||||
if dataset_versions:
|
||||
row.dataset_versions = {**row.dataset_versions, **dataset_versions}
|
||||
if source == "OFFICIAL_DATASET":
|
||||
row.validated_at = datetime.now(timezone.utc)
|
||||
self.session.flush()
|
||||
return row
|
||||
|
||||
def current_knowledge(self, kb_product_id):
|
||||
rows = self.session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == kb_product_id)).all()
|
||||
by_key: dict = {}
|
||||
for row in rows:
|
||||
by_key.setdefault(row.knowledge_key, []).append(row)
|
||||
result = {}
|
||||
for key, entries in by_key.items():
|
||||
values = {entry.value for entry in entries}
|
||||
if len(values) > 1:
|
||||
result[key] = {"conflict": True, "values": values}
|
||||
else:
|
||||
best = max(entries, key=lambda entry: entry.confidence)
|
||||
result[key] = {"conflict": False, "value": best.value, "source": best.source, "confidence": best.confidence}
|
||||
return result
|
||||
|
||||
# ----- observations -----
|
||||
|
||||
def record_observation(self, kb_product_id, tenant_id, company_key, erp_code, database_label, observed_fields, source="ERP_OBSERVATION"):
|
||||
row = KbObservation(kb_product_id=kb_product_id, tenant_id=tenant_id, company_key=company_key, erp_code=erp_code, database_label=database_label, observed_fields=observed_fields or {}, source=source)
|
||||
self.session.add(row); self.session.flush()
|
||||
return row
|
||||
|
||||
def observation_summary(self, kb_product_id):
|
||||
rows = self.session.scalars(select(KbObservation).where(KbObservation.kb_product_id == kb_product_id)).all()
|
||||
agreement: dict = {}
|
||||
for row in rows:
|
||||
for field, value in (row.observed_fields or {}).items():
|
||||
if value in (None, ""):
|
||||
continue
|
||||
bucket = agreement.setdefault(field, {})
|
||||
bucket[str(value)] = bucket.get(str(value), 0) + 1
|
||||
conflicts = {field: values for field, values in agreement.items() if len(values) > 1}
|
||||
return {"observation_count": len(rows), "tenant_count": len({row.tenant_id for row in rows}), "agreement": agreement, "conflicts": conflicts}
|
||||
|
||||
# ----- stats -----
|
||||
|
||||
def stats(self):
|
||||
products = self.session.scalar(select(sa_func.count()).select_from(KbProduct)) or 0
|
||||
gtin_products = self.session.scalar(select(sa_func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
|
||||
conflicts = 0
|
||||
for product_id in self.session.scalars(select(KbProduct.id)).all():
|
||||
knowledge = self.current_knowledge(product_id)
|
||||
conflicts += sum(1 for entry in knowledge.values() if entry.get("conflict"))
|
||||
fingerprint_products = products - gtin_products
|
||||
return {"products": products, "gtin_products": gtin_products, "fingerprint_products": fingerprint_products, "conflicts": conflicts}
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Conservative, explainable product identity and conflict classification."""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
from .fingerprint import normalize_description, token_stems, validate_gtin
|
||||
|
||||
GTIN_CONFLICT = "GTIN_CONFLICT"
|
||||
NCM_CONFLICT = "NCM_CONFLICT"
|
||||
DESCRIPTION_CONFLICT = "DESCRIPTION_CONFLICT"
|
||||
PACKAGING_VARIATION = "PACKAGING_VARIATION"
|
||||
FALSE_MATCH = "FALSE_MATCH"
|
||||
TENANT_CONTEXT_ONLY = "TENANT_CONTEXT_ONLY"
|
||||
|
||||
|
||||
def canonical_product_id(observation: dict) -> str:
|
||||
"""Stable global ID; fiscal and tenant fields are intentionally excluded."""
|
||||
gtin = validate_gtin(observation.get("gtin"))
|
||||
if gtin:
|
||||
return "GTIN:" + gtin
|
||||
values = "|".join((
|
||||
normalize_description(observation.get("normalized_description") or observation.get("description")),
|
||||
normalize_description(observation.get("brand")),
|
||||
normalize_description(observation.get("unit")),
|
||||
normalize_description(observation.get("package_quantity")),
|
||||
))
|
||||
return "FP:" + hashlib.sha256(values.encode()).hexdigest()
|
||||
|
||||
|
||||
def _description_equivalent(values):
|
||||
stems = [token_stems(value) for value in values if value]
|
||||
if not stems:
|
||||
return True
|
||||
return all(current == stems[0] or current.issubset(stems[0]) or stems[0].issubset(current) for current in stems[1:])
|
||||
|
||||
|
||||
def classify_conflict(observations: list[dict]) -> str:
|
||||
"""Classify only from structural evidence; ambiguous cases remain conflicts."""
|
||||
gtins = {validate_gtin(row.get("gtin")) for row in observations if validate_gtin(row.get("gtin"))}
|
||||
ncms = {str(row.get("ncm")) for row in observations if row.get("ncm")}
|
||||
descriptions = [row.get("normalized_description") or row.get("description") for row in observations]
|
||||
packages = {str(row.get("package_quantity")) for row in observations if row.get("package_quantity")}
|
||||
tenants = {str(obs.get("database") or obs.get("company_key")) for row in observations for obs in row.get("observations", [])}
|
||||
if len(gtins) > 1:
|
||||
return GTIN_CONFLICT
|
||||
if len(ncms) > 1:
|
||||
return NCM_CONFLICT
|
||||
if len(packages) > 1 and _description_equivalent(descriptions):
|
||||
return PACKAGING_VARIATION
|
||||
if len({normalize_description(value) for value in descriptions if value}) > 1:
|
||||
return DESCRIPTION_CONFLICT
|
||||
if len(tenants) > 1:
|
||||
return TENANT_CONTEXT_ONLY
|
||||
return FALSE_MATCH
|
||||
|
||||
|
||||
def audit_payload(payload: dict, historical_conflicts: int = 163) -> dict:
|
||||
"""Audit observable same-GTIN groups without silently resolving NCM conflicts."""
|
||||
by_gtin = defaultdict(list)
|
||||
for product in payload.get("products", []):
|
||||
gtin = validate_gtin(product.get("gtin"))
|
||||
if gtin:
|
||||
by_gtin[gtin].append(product)
|
||||
conflicts = []
|
||||
counts = defaultdict(int)
|
||||
for gtin, rows in by_gtin.items():
|
||||
if len({row.get("ncm") for row in rows if row.get("ncm")}) <= 1:
|
||||
continue
|
||||
kind = classify_conflict(rows)
|
||||
counts[kind] += 1
|
||||
conflicts.append({
|
||||
"identity": gtin,
|
||||
"classification": kind,
|
||||
"observations": len(rows),
|
||||
"ncms": sorted({row.get("ncm") for row in rows if row.get("ncm")}),
|
||||
"auto_resolution": "KEEP_KNOWLEDGE_CONFLICT",
|
||||
})
|
||||
return {
|
||||
"artifact": "KB_CONFLICT_AUDIT_V1",
|
||||
"historical_conflicts_reported": historical_conflicts,
|
||||
"observable_conflicts_audited": len(conflicts),
|
||||
"unobservable_historical_conflicts": max(0, historical_conflicts - len(conflicts)),
|
||||
"classification_counts": dict(sorted(counts.items())),
|
||||
"conflicts": conflicts,
|
||||
"policy": "only structurally equivalent identities may share a canonical product; tenant fiscal context is never global knowledge",
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Real LLM provider (OpenAI-compatible) for the NCM semantic evaluation.
|
||||
|
||||
Contract (mission REAL LLM EVALUATION V1):
|
||||
- The API key lives ONLY in the Hub environment (AI_API_KEY). Never committed, logged,
|
||||
written to artifacts, sent to the Agent or Firebird.
|
||||
- The gold answer is never part of the prompt: items carry only description/brand/unit/
|
||||
segment-weak-context + the official top-K hybrid candidates with hierarchical context.
|
||||
- Strict structured output: per item {work_item_id, selected_ncm, confidence, abstain,
|
||||
reason_short}; selected_ncm must be in the candidate list or null (abstain).
|
||||
- Batching is safe only when the response is valid JSON with 1:1 work_item_id matching,
|
||||
no omitted items, no cross-product mixing. Verified progressively (5/10/20).
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
PROMPT_VERSION_LLM = "llm-ncm-eval-v1"
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"You are a Brazilian customs classification assistant. For each product below, choose the "
|
||||
"most likely official NCM code from the provided candidate list, or abstain. "
|
||||
"Rules: selected_ncm MUST be one of the candidate codes for that exact item, or null with "
|
||||
"abstain=true when evidence is insufficient. Never invent a code. confidence is your calibrated "
|
||||
"probability in [0,1]. Reply with ONLY a JSON array, one object per item, in the same order, "
|
||||
'each: {"work_item_id": str, "selected_ncm": str|null, "confidence": float, "abstain": bool, "reason_short": str}.'
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmEvalItem:
|
||||
work_item_id: str
|
||||
description: str
|
||||
normalized_description: str
|
||||
brand: str | None
|
||||
unit: str | None
|
||||
segment: str
|
||||
candidates: list # [(code, official_description, hybrid_score, official_score, kb_score, tenant_count)]
|
||||
|
||||
|
||||
class LlmAnswerError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def item_prompt_payload(item: LlmEvalItem) -> dict:
|
||||
return {
|
||||
"work_item_id": item.work_item_id,
|
||||
"description": item.description,
|
||||
"normalized_description": item.normalized_description,
|
||||
"brand": item.brand,
|
||||
"unit": item.unit,
|
||||
"segment_weak_context": item.segment,
|
||||
"official_candidates": [
|
||||
{"ncm": code, "official_description": text}
|
||||
for code, text, *_ in item.candidates
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def candidate_set_hash(item: LlmEvalItem) -> str:
|
||||
codes = [code for code, *_ in item.candidates]
|
||||
return hashlib.sha256("|".join(codes).encode()).hexdigest()
|
||||
|
||||
|
||||
def input_fingerprint(item: LlmEvalItem) -> str:
|
||||
payload = {**item_prompt_payload(item), "system": _SYSTEM_PROMPT}
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
|
||||
|
||||
def validate_answer(answer: dict, item: LlmEvalItem) -> dict:
|
||||
"""Fail-closed validator: schema, work_item_id, candidate-list confinement."""
|
||||
if not isinstance(answer, dict):
|
||||
raise LlmAnswerError("answer must be an object")
|
||||
if str(answer.get("work_item_id")) != str(item.work_item_id):
|
||||
raise LlmAnswerError(f"work_item_id mismatch: {answer.get('work_item_id')!r} != {item.work_item_id!r}")
|
||||
abstain = bool(answer.get("abstain"))
|
||||
selected = answer.get("selected_ncm")
|
||||
selected = None if selected in (None, "") else str(selected)
|
||||
try:
|
||||
confidence = float(answer.get("confidence"))
|
||||
except (TypeError, ValueError):
|
||||
raise LlmAnswerError("confidence must be numeric")
|
||||
if not 0.0 <= confidence <= 1.0:
|
||||
raise LlmAnswerError("confidence out of [0,1]")
|
||||
codes = {code for code, *_ in item.candidates}
|
||||
if abstain:
|
||||
if selected is not None:
|
||||
raise LlmAnswerError("abstain must carry selected_ncm=null")
|
||||
else:
|
||||
if selected is None:
|
||||
raise LlmAnswerError("non-abstain requires selected_ncm")
|
||||
if selected not in codes:
|
||||
raise LlmAnswerError(f"selected_ncm {selected!r} is outside the candidate list")
|
||||
return {"work_item_id": item.work_item_id, "selected_ncm": selected, "confidence": confidence, "abstain": abstain, "reason_short": str(answer.get("reason_short") or "")[:200]}
|
||||
|
||||
|
||||
class OpenAICompatibleProvider:
|
||||
"""Talks to any OpenAI-compatible /chat/completions endpoint. `transport` is
|
||||
injectable for tests; production default uses httpx with the key from settings only."""
|
||||
|
||||
name = "openai-compatible"
|
||||
prompt_version = PROMPT_VERSION_LLM
|
||||
|
||||
def __init__(self, api_key: str, model: str, base_url: str = "https://api.openai.com/v1", transport=None, timeout: float = 120.0):
|
||||
self.api_key = api_key
|
||||
self.model = model
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout = timeout
|
||||
self._transport = transport or self._httpx_transport
|
||||
self.http_calls = 0
|
||||
self.retries = 0
|
||||
self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
|
||||
|
||||
def _httpx_transport(self, payload: dict) -> dict:
|
||||
import httpx
|
||||
response = httpx.post(f"{self.base_url}/chat/completions", json=payload, headers={"Authorization": f"Bearer {self.api_key}"}, timeout=self.timeout)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def list_models(self) -> list:
|
||||
"""GET /models with the account key — validates auth/base URL/model availability
|
||||
without spending completion tokens. Never raises with the key attached."""
|
||||
import httpx
|
||||
response = httpx.get(f"{self.base_url}/models", headers={"Authorization": f"Bearer {self.api_key}"}, timeout=self.timeout)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return sorted(str(item.get("id")) for item in data.get("data", []) if item.get("id"))
|
||||
|
||||
def _request(self, messages: list, max_retries: int = 2) -> dict:
|
||||
payload = {"model": self.model, "messages": messages, "temperature": 0}
|
||||
last_error = None
|
||||
for attempt in range(max_retries + 1):
|
||||
self.http_calls += 1
|
||||
if attempt:
|
||||
self.retries += 1
|
||||
try:
|
||||
data = self._transport(payload)
|
||||
usage = data.get("usage") or {}
|
||||
self.usage["input_tokens"] += int(usage.get("prompt_tokens") or 0)
|
||||
self.usage["output_tokens"] += int(usage.get("completion_tokens") or 0)
|
||||
cached = usage.get("prompt_tokens_details") or {}
|
||||
self.usage["cached_tokens"] += int(cached.get("cached_tokens") or 0)
|
||||
return data
|
||||
except Exception as error: # transport-level failure only; JSON contract errors are not retried
|
||||
last_error = error
|
||||
raise RuntimeError(f"LLM transport failed after {max_retries + 1} attempts: {last_error}")
|
||||
|
||||
def select_batch(self, items: list[LlmEvalItem]) -> list[dict]:
|
||||
"""One HTTP call for the whole batch. Raises LlmAnswerError on any 1:1/schema violation
|
||||
so the harness can shrink the batch size."""
|
||||
user_content = json.dumps([item_prompt_payload(item) for item in items], ensure_ascii=False)
|
||||
data = self._request([{"role": "system", "content": _SYSTEM_PROMPT}, {"role": "user", "content": user_content}])
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
content = re.sub(r"^```(?:json)?|```$", "", content.strip(), flags=re.MULTILINE).strip()
|
||||
answers = json.loads(content)
|
||||
if not isinstance(answers, list) or len(answers) != len(items):
|
||||
raise LlmAnswerError(f"expected {len(items)} answers, got {len(answers) if isinstance(answers, list) else type(answers).__name__}")
|
||||
return [validate_answer(answer, item) for answer, item in zip(answers, items)]
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Deterministic prompt compaction that preserves candidate evidence."""
|
||||
|
||||
import json
|
||||
|
||||
|
||||
def estimate_tokens(payload) -> int:
|
||||
return max(1, len(json.dumps(payload, ensure_ascii=False, sort_keys=True)) // 4)
|
||||
|
||||
|
||||
def compact_payload(payload):
|
||||
"""Remove repeated metadata only; candidate codes/descriptions are untouched."""
|
||||
compact = dict(payload)
|
||||
compact.pop("debug", None)
|
||||
compact.pop("duplicate_examples", None)
|
||||
candidates = compact.get("official_candidates")
|
||||
if isinstance(candidates, list):
|
||||
seen = set()
|
||||
compact["official_candidates"] = [
|
||||
row for row in candidates
|
||||
if not (row.get("ncm") in seen or seen.add(row.get("ncm")))
|
||||
]
|
||||
return compact
|
||||
|
||||
|
||||
def measure_payloads(payloads):
|
||||
before = sum(estimate_tokens(payload) for payload in payloads)
|
||||
after = sum(estimate_tokens(compact_payload(payload)) for payload in payloads)
|
||||
return {"items": len(payloads), "tokens_before": before, "tokens_after": after, "reduction_percent": round((1 - after / before) * 100, 2) if before else 0.0}
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Provider abstraction with strict structured output and a hard budget guard.
|
||||
|
||||
The fiscal engine is never coupled to a vendor: it talks to AISemanticProvider.
|
||||
V1 ships BaselineProvider (deterministic, zero-cost) used by the evaluation
|
||||
harness and as the no-credential fallback; real providers subclass the same
|
||||
contract and are only invoked with AI_ENABLED=true and credentials in the Hub.
|
||||
"""
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from ..classification import ENGINE_VERSION
|
||||
from .fingerprint import overlap_score
|
||||
|
||||
PROMPT_VERSION = "ai-semantic-v1"
|
||||
|
||||
|
||||
class SemanticOutputError(ValueError):
|
||||
"""Raised when a provider answer violates the strict structured-output schema."""
|
||||
|
||||
|
||||
class ProviderTransportError(RuntimeError):
|
||||
"""A retryable failure while contacting a semantic provider."""
|
||||
|
||||
|
||||
class ProviderContractError(ValueError):
|
||||
"""A non-retryable provider response or protocol violation."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class StructuredResult:
|
||||
task_type: str
|
||||
input_fingerprint: str
|
||||
selected_candidate: str # candidate code, or "NONE"
|
||||
confidence: float
|
||||
reason: str
|
||||
candidate_ids_seen: list[str]
|
||||
model: str
|
||||
prompt_version: str
|
||||
created_at: str
|
||||
dataset_versions: dict = field(default_factory=dict)
|
||||
engine_version: str = ENGINE_VERSION
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"task_type": self.task_type,
|
||||
"input_fingerprint": self.input_fingerprint,
|
||||
"selected_candidate": self.selected_candidate,
|
||||
"confidence": self.confidence,
|
||||
"reason": self.reason,
|
||||
"candidate_ids_seen": self.candidate_ids_seen,
|
||||
"model": self.model,
|
||||
"prompt_version": self.prompt_version,
|
||||
"created_at": self.created_at,
|
||||
"dataset_versions": self.dataset_versions,
|
||||
"engine_version": self.engine_version,
|
||||
}
|
||||
|
||||
|
||||
def result_from_dict(data) -> StructuredResult:
|
||||
"""Fail-closed schema check: the selection must be NONE or one of the seen candidates."""
|
||||
result = StructuredResult(
|
||||
task_type=str(data["task_type"]),
|
||||
input_fingerprint=str(data["input_fingerprint"]),
|
||||
selected_candidate=str(data["selected_candidate"]),
|
||||
confidence=float(data["confidence"]),
|
||||
reason=str(data["reason"]),
|
||||
candidate_ids_seen=[str(item) for item in data["candidate_ids_seen"]],
|
||||
model=str(data["model"]),
|
||||
prompt_version=str(data["prompt_version"]),
|
||||
created_at=str(data["created_at"]),
|
||||
dataset_versions=dict(data.get("dataset_versions") or {}),
|
||||
)
|
||||
if result.selected_candidate != "NONE" and result.selected_candidate not in result.candidate_ids_seen:
|
||||
raise SemanticOutputError(f"selected_candidate {result.selected_candidate!r} is not among the official candidates")
|
||||
if not 0.0 <= result.confidence <= 1.0:
|
||||
raise SemanticOutputError("confidence must be within [0,1]")
|
||||
return result
|
||||
|
||||
|
||||
class AISemanticProvider:
|
||||
"""Contract every provider must honour. `select` receives ONLY official candidates."""
|
||||
name = "abstract"
|
||||
model = "abstract"
|
||||
|
||||
def select(self, work_item, candidates) -> StructuredResult:
|
||||
raise NotImplementedError
|
||||
|
||||
def estimate(self, work_items) -> dict:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _score(text: str, official: str) -> float:
|
||||
return overlap_score(text, official)
|
||||
|
||||
|
||||
class BaselineProvider(AISemanticProvider):
|
||||
"""Deterministic zero-cost baseline: highest token-overlap candidate, or NONE.
|
||||
Used for the evaluation pipeline and as the mandatory no-credential fallback."""
|
||||
|
||||
name = "baseline"
|
||||
model = "baseline-deterministic-v1"
|
||||
|
||||
def select(self, work_item, candidates, dataset_versions=None) -> StructuredResult:
|
||||
description = work_item.payload.get("description", "") if hasattr(work_item, "payload") else ""
|
||||
scored = [(code, official, _score(description, official)) for code, official in [(str(c[0]), c[1]) for c in candidates]]
|
||||
best = max(scored, key=lambda row: row[2]) if scored else None
|
||||
selected, _, score = best if best else ("NONE", "", 0.0)
|
||||
if not scored or score <= 0.0:
|
||||
selected = "NONE"
|
||||
return StructuredResult(
|
||||
task_type=work_item.task_type,
|
||||
input_fingerprint=work_item.dedup_key,
|
||||
selected_candidate=selected,
|
||||
confidence=round(score, 4),
|
||||
reason="baseline token-overlap top-1" if selected != "NONE" else "no token overlap with any official candidate",
|
||||
candidate_ids_seen=[str(c[0]) for c in candidates],
|
||||
model=self.model,
|
||||
prompt_version=PROMPT_VERSION,
|
||||
created_at=datetime.now(timezone.utc).isoformat(),
|
||||
dataset_versions=dict(dataset_versions or {}),
|
||||
)
|
||||
|
||||
def estimate(self, work_items) -> dict:
|
||||
return {"input_tokens": 0, "output_tokens": 0, "estimated_cost": 0.0}
|
||||
|
||||
|
||||
class BudgetGuard:
|
||||
"""Refuses execution when the estimated cost exceeds AI_MAX_RUN_COST_USD.
|
||||
Never falls back to a more expensive model. Prices are operator-configured — when
|
||||
absent, cost is UNKNOWN: the residual run is refused, but an explicitly limited
|
||||
benchmark (e.g. the 65-item GOLD eval) may opt in with allow_unknown_cost."""
|
||||
|
||||
def __init__(self, max_run_cost_usd: float, price_in_per_m: float | None, price_out_per_m: float | None):
|
||||
self.max_run_cost_usd = float(max_run_cost_usd)
|
||||
self.price_in_per_m = None if price_in_per_m is None else float(price_in_per_m)
|
||||
self.price_out_per_m = None if price_out_per_m is None else float(price_out_per_m)
|
||||
|
||||
def estimate(self, items_in_tokens: int, items_out_tokens: int) -> dict:
|
||||
if self.price_in_per_m is None or self.price_out_per_m is None:
|
||||
return {"input_tokens": items_in_tokens, "output_tokens": items_out_tokens, "estimated_cost": None, "cost_unknown": True}
|
||||
cost = (items_in_tokens / 1_000_000) * self.price_in_per_m + (items_out_tokens / 1_000_000) * self.price_out_per_m
|
||||
return {"input_tokens": items_in_tokens, "output_tokens": items_out_tokens, "estimated_cost": round(cost, 6)}
|
||||
|
||||
def authorize(self, plan: dict, allow_unknown_cost: bool = False) -> bool:
|
||||
if plan.get("estimated_cost") is None:
|
||||
return allow_unknown_cost
|
||||
return plan["estimated_cost"] <= self.max_run_cost_usd
|
||||
|
||||
def authorize_usage(self, input_tokens: int, output_tokens: int, *, prior_cost_usd: float = 0.0,
|
||||
allow_unknown_cost: bool = False) -> bool:
|
||||
"""Apply the same guard to actual usage before accepting a provider result."""
|
||||
plan = self.estimate(input_tokens, output_tokens)
|
||||
if plan.get("estimated_cost") is None:
|
||||
return allow_unknown_cost
|
||||
return float(prior_cost_usd) + plan["estimated_cost"] <= self.max_run_cost_usd
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Pure, review-first questionnaire helpers for local fiscal context."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
|
||||
from central.app.context_keys import validate_entry
|
||||
|
||||
|
||||
def _options(row):
|
||||
value = row.get("VALID_OPTIONS", [])
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
value = [x.strip() for x in value.split("|") if x.strip()]
|
||||
return list(value) if isinstance(value, (list, tuple, set)) else [value]
|
||||
|
||||
|
||||
def build_questionnaire(rows):
|
||||
result = []
|
||||
for row in rows:
|
||||
scope = str(row.get("SCOPE", row.get("scope", ""))).strip().upper()
|
||||
key = str(row.get("KEY", row.get("key", ""))).strip()
|
||||
scope_key = str(row.get("GROUP") or row.get("NCM") or row.get("scope_key") or "").strip()
|
||||
result.append({
|
||||
"question_id": str(row.get("QUESTION_ID", row.get("question_id", ""))).strip(),
|
||||
"database": str(row.get("DATABASE", row.get("database", ""))).strip(),
|
||||
"scope": scope,
|
||||
"scope_key": scope_key if scope != "COMPANY" else "",
|
||||
"group": str(row.get("GROUP") or "").strip(),
|
||||
"key": key,
|
||||
"question": row.get("QUESTION", row.get("question", "")),
|
||||
"options": _options(row),
|
||||
"affected": {"products": int(row.get("PRODUCTS_AFFECTED") or row.get("products_affected") or 0), "cells": int(row.get("CELLS") or row.get("cells") or 0)},
|
||||
"examples": row.get("EXAMPLES", row.get("examples", "")),
|
||||
"consequence": row.get("WHAT_IT_WILL_RESOLVE", row.get("consequence", "")),
|
||||
})
|
||||
return sorted(result, key=lambda q: (q["database"], q["scope"], q["scope_key"], q["key"], q["question_id"]))
|
||||
|
||||
|
||||
def group_questions(questions):
|
||||
"""Group equivalent human decisions while retaining every affected question ID."""
|
||||
groups = {}
|
||||
for question in questions:
|
||||
key = (question["database"], question["group"], question["scope_key"], question["key"], question["question"], tuple(question["options"]))
|
||||
group = groups.setdefault(key, {
|
||||
"group_id": "GROUP-" + str(len(groups) + 1),
|
||||
"database": question["database"],
|
||||
"group": question["group"],
|
||||
"scope": question["scope"],
|
||||
"scope_key": question["scope_key"],
|
||||
"key": question["key"],
|
||||
"question": question["question"],
|
||||
"options": question["options"],
|
||||
"question_ids": [],
|
||||
"affected": {"products": 0, "cells": 0},
|
||||
"examples": [],
|
||||
"consequence": question["consequence"],
|
||||
})
|
||||
group["question_ids"].append(question["question_id"])
|
||||
group["affected"]["products"] += question["affected"]["products"]
|
||||
group["affected"]["cells"] += question["affected"]["cells"]
|
||||
if question["examples"] and len(group["examples"]) < 5:
|
||||
group["examples"].append(str(question["examples"]))
|
||||
return list(groups.values())
|
||||
|
||||
|
||||
def validate_answers(questions, answers):
|
||||
entries, effects, rollback_plan = [], [], []
|
||||
by_id = {q["question_id"]: q for q in questions}
|
||||
for question_id, value in answers.items():
|
||||
if question_id not in by_id:
|
||||
raise ValueError(f"unknown question {question_id!r}; nothing was persisted")
|
||||
q = by_id[question_id]
|
||||
value = str(value).strip()
|
||||
if q["options"] and value not in q["options"]:
|
||||
raise ValueError(f"{question_id}: value must be one of {q['options']}; nothing was persisted")
|
||||
entry, error = validate_entry({"scope": q["scope"], "scope_key": q["scope_key"], "key": q["key"], "value": value})
|
||||
if error:
|
||||
raise ValueError(f"{question_id}: {error}; nothing was persisted")
|
||||
entry["company_key"] = q["database"]
|
||||
entries.append(entry)
|
||||
effect = {"question_id": question_id, "database": q["database"], "key": q["key"], "affected": q["affected"], "behavior": "context_entry"}
|
||||
if q["key"] == "entry_scope" and value == "MIXED_OR_DYNAMIC":
|
||||
effect["behavior"] = "no_static_cfop_entrada"
|
||||
effects.append(effect)
|
||||
if q["key"] == "store_ipi_profile" and value == "LEAVE_NOT_APPLICABLE":
|
||||
rollback_plan.append({"database": q["database"], "field": "IPI", "affected_cells": q["affected"]["cells"]})
|
||||
return {"entries": entries, "effects": effects, "rollback_plan": rollback_plan}
|
||||
|
||||
|
||||
def render_json(result):
|
||||
return json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
|
||||
|
||||
def render_csv(result):
|
||||
output = io.StringIO()
|
||||
writer = csv.DictWriter(output, fieldnames=["company_key", "scope", "scope_key", "key", "value", "reason"])
|
||||
writer.writeheader()
|
||||
writer.writerows({k: entry.get(k, "") for k in writer.fieldnames} for entry in result["entries"])
|
||||
return output.getvalue()
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Semantic candidate retrieval V2 — deterministic, zero LLM.
|
||||
|
||||
Three retrieval sources merged into one ranking:
|
||||
A. OFFICIAL: hierarchical search over the official NCM tree. Every 8-digit code is
|
||||
enriched with its ancestors' descriptions (chapter/position/subposition), so the
|
||||
very short legal text carries the vocabulary of its path (section 6).
|
||||
B. KB NEIGHBORS: the Knowledge Base acts as a commercial->official bridge — products
|
||||
whose commercial description resembles the query contribute their VALIDATED NCMs,
|
||||
weighted by similarity, tenant_count, observation_count and agreement (sections 2-3).
|
||||
C. HYBRID: single final ranking; the KB is only evidence for CANDIDATES, never a
|
||||
majority-vote fiscal decision.
|
||||
|
||||
Abbreviation lexicon (section 8) is derived statistically from the KB itself; commercial
|
||||
normalization (section 7) strips units/sizes/colors/refs deterministically.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import Dataset, KbKnowledge, KbObservation, KbProduct, NcmEntry, Tenant
|
||||
from .fingerprint import normalize_description, token_stems
|
||||
|
||||
# ---------------- commercial normalization (section 7) ----------------
|
||||
|
||||
_UNITS = {"UN", "UNID", "UNIDADE", "CX", "CAIXA", "FD", "FDO", "FARDO", "PCT", "PACOTE", "PKG",
|
||||
"KG", "G", "GR", "GRAMA", "MG", "ML", "L", "LT", "LITRO", "LATA", "GARRAFA", "PET",
|
||||
"PC", "PECA", "DZ", "DUZIA", "PAR", "KIT", "JG", "PARA", "COM", "SEM", "C", "S"}
|
||||
_DISCARD = _UNITS | {"MASC", "MASCULINO", "FEM", "FEMININO", "INF", "INFANTIL", "TAM", "TAMANHO",
|
||||
"COR", "REF", "COD", "CODIGO", "MOD", "MODELO", "NOVA", "NOVO", "TIPO"}
|
||||
_LEXEME_STRIP_PREFIXES = ("1", "2", "3", "4", "5", "6", "7", "8", "9", "0", ".")
|
||||
|
||||
|
||||
def _is_size_or_code(token: str) -> bool:
|
||||
if token.isdigit():
|
||||
return True
|
||||
digits = "".join(char for char in token if char.isdigit())
|
||||
letters = "".join(char for char in token if char.isalpha())
|
||||
return bool(digits) and letters in _UNITS # "1KG", "500ML", "2LT"
|
||||
|
||||
|
||||
def commercial_tokens(description) -> set:
|
||||
"""Stable commercial lexemes: units/sizes/colors/refs removed, stems kept."""
|
||||
tokens = []
|
||||
skip_next = False
|
||||
for token in normalize_description(description).split():
|
||||
if skip_next:
|
||||
skip_next = False
|
||||
continue
|
||||
if token in ("TAM", "COR", "REF", "COD", "MOD"):
|
||||
skip_next = True
|
||||
continue
|
||||
if token in _DISCARD or _is_size_or_code(token):
|
||||
continue
|
||||
tokens.append(token)
|
||||
return {_stem_safe(token) for token in tokens}
|
||||
|
||||
|
||||
def _stem_safe(token: str) -> str:
|
||||
from .fingerprint import _stem
|
||||
return _stem(token)
|
||||
|
||||
|
||||
def expand_stems(stems: set, lexicon: dict) -> set:
|
||||
expanded = set(stems)
|
||||
for stem in stems:
|
||||
target = lexicon.get(stem)
|
||||
if target:
|
||||
expanded.add(target)
|
||||
return expanded
|
||||
|
||||
|
||||
def derive_abbreviation_lexicon(corpus: dict, min_support: int = 3) -> dict:
|
||||
"""corpus: {commercial_description: ncm}. Statistical abbreviations only: a short token
|
||||
(3-5 chars) that is a strict prefix of a longer token, where BOTH forms occur across
|
||||
distinct products that share the same NCM. Support = (#products with the short form) +
|
||||
(#products with the long form); accepted when support >= min_support and both >= 1."""
|
||||
by_ncm: dict = {}
|
||||
for description, ncm in corpus.items():
|
||||
tokens = set(normalize_description(description).split())
|
||||
by_ncm.setdefault(ncm, []).append(tokens)
|
||||
best: dict = {}
|
||||
for descriptions in by_ncm.values():
|
||||
if len(descriptions) < 2:
|
||||
continue
|
||||
all_tokens = set().union(*descriptions)
|
||||
for short in all_tokens:
|
||||
if not (3 <= len(short) <= 5):
|
||||
continue
|
||||
short_count = sum(1 for tokens in descriptions if short in tokens)
|
||||
if short_count < 1:
|
||||
continue
|
||||
for long_token in all_tokens:
|
||||
if len(long_token) > len(short) and long_token.startswith(short):
|
||||
long_count = sum(1 for tokens in descriptions if long_token in tokens)
|
||||
support = short_count + long_count
|
||||
if support >= min_support and long_count >= 1 and short_count >= 1:
|
||||
current = best.get(short)
|
||||
if current is None or support > current[1]:
|
||||
best[short] = (long_token, support)
|
||||
return {short: target for short, (target, _) in best.items()}
|
||||
|
||||
|
||||
# ---------------- enriched official corpus (sections 5-6) ----------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnrichedNcmCorpus:
|
||||
documents: dict # code -> enriched stems set
|
||||
descriptions: dict # code -> official description
|
||||
positions: dict # 4-digit position -> enriched stems (chapter + position)
|
||||
children: dict # 4-digit position -> [8-digit codes]
|
||||
|
||||
@staticmethod
|
||||
def build(session: Session) -> "EnrichedNcmCorpus":
|
||||
dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
if dataset is None:
|
||||
return EnrichedNcmCorpus({}, {}, {}, {})
|
||||
entries = session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all()
|
||||
by_code = {entry.code: entry.description for entry in entries}
|
||||
documents, positions, children = {}, {}, {}
|
||||
for code, description in by_code.items():
|
||||
if len(code) != 8:
|
||||
continue
|
||||
ancestors = [text for text in (by_code.get(code[:2]), by_code.get(code[:4]), by_code.get(code[:6])) if text]
|
||||
documents[code] = token_stems(description) | set().union(*[token_stems(text) for text in ancestors]) if ancestors else token_stems(description)
|
||||
positions.setdefault(code[:4], token_stems(by_code.get(code[:4], "")) | token_stems(by_code.get(code[:2], "")))
|
||||
children.setdefault(code[:4], []).append(code)
|
||||
return EnrichedNcmCorpus(documents, by_code, positions, children)
|
||||
|
||||
def document(self, code: str) -> str:
|
||||
return " ".join(sorted(self.documents.get(code, set())))
|
||||
|
||||
def items(self):
|
||||
return list(self.documents.items())
|
||||
|
||||
|
||||
def _score(stems: set, document_stems: set, idf: dict | None = None) -> float:
|
||||
if not stems:
|
||||
return 0.0
|
||||
if idf:
|
||||
matched = sum(idf.get(stem, 1.0) for stem in stems if stem in document_stems)
|
||||
total = sum(idf.get(stem, 1.0) for stem in stems)
|
||||
return matched / total
|
||||
return len(stems & document_stems) / len(stems)
|
||||
|
||||
|
||||
def hierarchical_candidates(corpus: EnrichedNcmCorpus, description, k: int = 20, max_positions: int = 15):
|
||||
"""Two-stage: rank 4-digit positions first, then their 8-digit descendants (section 5)."""
|
||||
stems = commercial_tokens(description)
|
||||
if not stems:
|
||||
return []
|
||||
position_scores = sorted(((_score(stems, doc), position) for position, doc in corpus.positions.items()), reverse=True)
|
||||
ranked = []
|
||||
for position_score, position in position_scores[:max_positions]:
|
||||
if position_score <= 0:
|
||||
continue
|
||||
for code in corpus.children.get(position, []):
|
||||
code_score = _score(stems, corpus.documents[code])
|
||||
if code_score > 0:
|
||||
ranked.append((code, corpus.descriptions.get(code, ""), round(code_score, 4)))
|
||||
ranked.sort(key=lambda row: (-row[2], row[0]))
|
||||
return ranked[:k]
|
||||
|
||||
|
||||
# ---------------- KB neighbor bridge (sections 2-3) ----------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class KbNeighborIndex:
|
||||
entries: list # (product_id, stems, ncm, tenant_ids:set, observation_count, has_conflict)
|
||||
|
||||
@staticmethod
|
||||
def build(session: Session) -> "KbNeighborIndex":
|
||||
validated = {}
|
||||
for knowledge in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "validated_ncm")).all():
|
||||
validated.setdefault(knowledge.kb_product_id, set()).add(knowledge.value)
|
||||
conflicts = {product_id: len(values) > 1 for product_id, values in validated.items()}
|
||||
products = {product.id: product for product in session.scalars(select(KbProduct)).all()}
|
||||
observations = session.scalars(select(KbObservation)).all()
|
||||
per_product_tenants: dict = {}
|
||||
for observation in observations:
|
||||
per_product_tenants.setdefault(observation.kb_product_id, set()).add(observation.tenant_id)
|
||||
entries = []
|
||||
for product_id, values in validated.items():
|
||||
if conflicts.get(product_id):
|
||||
continue # conflicting knowledge never feeds neighbors
|
||||
for value in values:
|
||||
product = products.get(product_id)
|
||||
if product is None or not product.normalized_description:
|
||||
continue
|
||||
tenants = per_product_tenants.get(product_id, set())
|
||||
entries.append((product_id, commercial_tokens(product.normalized_description), value, tenants, len(tenants), False))
|
||||
return KbNeighborIndex(entries)
|
||||
|
||||
def corpus(self) -> dict:
|
||||
return {" ".join(sorted(stems)): ncm for _, stems, ncm, _, _, _ in self.entries}
|
||||
|
||||
|
||||
def kb_neighbor_candidates(index: KbNeighborIndex, description, k: int = 10, exclude_product_id: int | None = None):
|
||||
"""Aggregate NCMs of lexically similar known products. `exclude_product_id` implements
|
||||
leave-one-out evaluation: the queried product never retrieves itself. Returns
|
||||
(ncm, neighbor_score, observation_count, tenant_count) — agreement is the union of
|
||||
distinct tenants behind the contributing neighbors, reported as evidence, never used
|
||||
as an automatic majority decision."""
|
||||
stems = commercial_tokens(description)
|
||||
if not stems:
|
||||
return []
|
||||
best: dict = {}
|
||||
for product_id, neighbor_stems, ncm, tenant_ids, observation_count, _ in index.entries:
|
||||
if exclude_product_id is not None and product_id == exclude_product_id:
|
||||
continue
|
||||
similarity = _score(stems, neighbor_stems)
|
||||
if similarity <= 0:
|
||||
continue
|
||||
current = best.get(ncm)
|
||||
if current is None:
|
||||
best[ncm] = [ncm, round(similarity, 4), observation_count, set(tenant_ids)]
|
||||
else:
|
||||
current[1] = max(current[1], round(similarity, 4))
|
||||
current[3] |= set(tenant_ids)
|
||||
ranked = sorted(((row[0], row[1], row[2], len(row[3])) for row in best.values()), key=lambda row: (-row[1], -row[3], row[0]))
|
||||
return ranked[:k]
|
||||
|
||||
|
||||
# ---------------- hybrid ranking (section 4) ----------------
|
||||
|
||||
OFFICIAL_WEIGHT, KB_WEIGHT = 0.3, 0.7 # tuned on the V2 benchmark (KB bridge carries the signal;
|
||||
# the official component breaks ties and covers products absent from the KB)
|
||||
|
||||
|
||||
def hybrid_candidates(corpus: EnrichedNcmCorpus, index: KbNeighborIndex, description, k: int = 20, lexicon: dict | None = None, exclude_product_id: int | None = None):
|
||||
stems = expand_stems(commercial_tokens(description), lexicon or {})
|
||||
official = hierarchical_candidates(corpus, " ".join(sorted(stems)) or description, k=k)
|
||||
neighbors = kb_neighbor_candidates(index, description, k=k, exclude_product_id=exclude_product_id)
|
||||
combined: dict = {}
|
||||
for code, text, score in official:
|
||||
combined[code] = [code, text, OFFICIAL_WEIGHT * score, score, 0.0, 0]
|
||||
for code, similarity, observation_count, tenant_count in neighbors:
|
||||
row = combined.setdefault(code, [code, "", 0.0, 0.0, 0.0, 0])
|
||||
row[2] += KB_WEIGHT * similarity
|
||||
row[4] = similarity
|
||||
row[5] = tenant_count
|
||||
ranked = sorted(combined.values(), key=lambda row: (-round(row[2], 4), row[0]))
|
||||
return [(row[0], row[1], round(row[2], 4), row[3], row[4], row[5]) for row in ranked[:k]]
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Retrieval V3 evidence layer. It ranks candidates only; it never decides fiscal truth."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
from .fingerprint import normalize_description, token_stems
|
||||
from .retrieval import commercial_tokens, hybrid_candidates
|
||||
|
||||
|
||||
def _char_similarity(left, right):
|
||||
return SequenceMatcher(None, normalize_description(left), normalize_description(right)).ratio()
|
||||
|
||||
|
||||
def retrieval_v3_candidates(corpus, index, description, metadata=None, k=20, lexicon=None, exclude_product_id=None):
|
||||
"""Add optional product evidence to V2 while preserving V2 as the safe baseline."""
|
||||
metadata = metadata or {}
|
||||
base = hybrid_candidates(corpus, index, description, k=max(k, 50), lexicon=lexicon, exclude_product_id=exclude_product_id)
|
||||
gtin_ncm = str(metadata.get("gtin_ncm") or "")
|
||||
group_ncms = {str(value) for value in (metadata.get("group_ncms") or [])}
|
||||
rarity = metadata.get("token_rarity") or {}
|
||||
query_stems = commercial_tokens(description)
|
||||
rows = []
|
||||
for code, text, score, official_score, kb_score, tenant_count in base:
|
||||
evidence = 0.0
|
||||
if gtin_ncm and code == gtin_ncm:
|
||||
evidence += 1.0
|
||||
if code in group_ncms:
|
||||
evidence += 0.05
|
||||
matched_rarity = sum(float(rarity.get(token, 1.0)) for token in query_stems if token in token_stems(text))
|
||||
evidence += min(0.05, matched_rarity / 100.0)
|
||||
rows.append((code, text, round(score + evidence, 4), official_score, kb_score, tenant_count))
|
||||
rows.sort(key=lambda row: (-row[2], row[0]))
|
||||
return rows[:k]
|
||||
|
||||
|
||||
def candidate_fingerprint(payload):
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def prepare_candidate_set(work_item, candidates, *, dataset_versions=None, gtin_evidence=None, kb_neighbors=None, hierarchical_context=None):
|
||||
"""Build the final LLM-ready evidence package without selecting a candidate."""
|
||||
payload = {
|
||||
"task_type": work_item.task_type,
|
||||
"dedup_key": work_item.dedup_key,
|
||||
"normalized_product": normalize_description(work_item.payload.get("description", "")),
|
||||
"top20_candidate_ncms": [
|
||||
{"ncm": row[0], "official_description": row[1], "score": row[2], "official_score": row[3], "kb_score": row[4], "tenant_count": row[5]}
|
||||
for row in candidates[:20]
|
||||
],
|
||||
"official_descriptions": {row[0]: row[1] for row in candidates[:20]},
|
||||
"hierarchical_context": hierarchical_context or {},
|
||||
"kb_neighbors": kb_neighbors or [],
|
||||
"gtin_evidence": gtin_evidence or {},
|
||||
"dataset_versions": dict(dataset_versions or {}),
|
||||
}
|
||||
payload["input_fingerprint"] = candidate_fingerprint(payload)
|
||||
return payload
|
||||
|
||||
|
||||
def compact_product_role_context(rows, limit=12):
|
||||
"""Keep representative role evidence bounded; no role is inferred."""
|
||||
unique = []
|
||||
seen = set()
|
||||
for row in rows:
|
||||
description = normalize_description(row.get("description"))
|
||||
if not description or description in seen:
|
||||
continue
|
||||
seen.add(description)
|
||||
unique.append({"description": description, "group": row.get("group"), "subgroup": row.get("subgroup"), "operations": row.get("operations", [])})
|
||||
if len(unique) >= limit:
|
||||
break
|
||||
return {"examples": unique, "evidence_only": True, "decision": None}
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Residual-question dedup: group pending cells by the KNOWLEDGE needed, not per cell.
|
||||
|
||||
- INVALID_NCM -> one work item per product fingerprint (same product in N databases collapses)
|
||||
- CEST/EX_TIPI -> one work item per (doubt, NCM): all databases sharing that NCM collapse
|
||||
- PRODUCT_ROLE -> tenant-specific by contract: one work item per (company, group)
|
||||
- Everything else (origin, ST retention, company profile, duplicate key) never becomes
|
||||
an AI work item.
|
||||
"""
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
_INVALID_NCM, _ROLE = "ncm", "product_role"
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkItem:
|
||||
task_type: str
|
||||
dedup_key: str
|
||||
question_refs: list = field(default_factory=list)
|
||||
cells_covered: int = 0
|
||||
payload: dict = field(default_factory=dict)
|
||||
kb_product_id: int | None = None
|
||||
|
||||
|
||||
def build_work_items(questions, product_fingerprints=None, canonical_products=None) -> list[WorkItem]:
|
||||
"""`product_fingerprints` maps "{DATABASE}:{erp_code}" -> KB fingerprint (from the seed pass)."""
|
||||
product_fingerprints = product_fingerprints or {}
|
||||
canonical_products = canonical_products or {}
|
||||
items: dict = {}
|
||||
|
||||
def _get(task_type, dedup_key, ref, cells):
|
||||
item = items.get(dedup_key)
|
||||
if item is None:
|
||||
item = WorkItem(task_type=task_type, dedup_key=dedup_key)
|
||||
items[dedup_key] = item
|
||||
item.question_refs.append(ref)
|
||||
item.cells_covered += cells
|
||||
return item
|
||||
|
||||
for question in questions:
|
||||
key = question.get("KEY")
|
||||
scope = question.get("SCOPE")
|
||||
cells = int(question.get("CELLS") or question.get("PRODUCTS_AFFECTED") or 0)
|
||||
ref = question.get("QUESTION_ID") or f"{question.get('DATABASE')}:{scope}:{question.get('NCM') or question.get('GROUP')}:{key}"
|
||||
if key in ("cest", "ex_tipi") and scope == "NCM":
|
||||
ncm = str(question.get("NCM") or "").strip()
|
||||
if ncm:
|
||||
item = _get(key.upper(), f"{key}:{ncm}", ref, cells)
|
||||
item.payload.setdefault("ncm", ncm)
|
||||
item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
|
||||
elif key == _INVALID_NCM and scope == "PRODUCT":
|
||||
database = question.get("DATABASE", "")
|
||||
erp_code = str(question.get("DATABASE_CODE") or "").strip()
|
||||
fingerprint = product_fingerprints.get(f"{database}:{erp_code}")
|
||||
fingerprint = canonical_products.get(f"{database}:{erp_code}", fingerprint)
|
||||
dedup = f"invalid_ncm:{fingerprint}" if fingerprint else f"invalid_ncm:{database}:{erp_code}"
|
||||
item = _get("INVALID_NCM", dedup, ref, cells)
|
||||
item.payload.setdefault("erp_code", erp_code)
|
||||
item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
|
||||
elif key == _ROLE and scope == "GROUP":
|
||||
company = question.get("COMPANY", "")
|
||||
group = str(question.get("GROUP") or "").strip()
|
||||
if group:
|
||||
normalized_group = " ".join(str(group).upper().split())
|
||||
item = _get("PRODUCT_ROLE", f"product_role:{company}:{normalized_group}", ref, cells)
|
||||
item.payload.setdefault("company_key", company)
|
||||
item.payload["group"] = group
|
||||
return list(items.values())
|
||||
Reference in New Issue
Block a user