chore: capture semantic pipeline baseline
This commit is contained in:
OpenCode
@@ -0,0 +1,37 @@
|
||||
"""Migration 0012: AI semantic layer V1 (KB, tenant context, work items, runs).
|
||||
|
||||
Revision ID: 0012
|
||||
Revises: 0011
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, DateTime, Float, ForeignKey, Integer, JSON, String, Text, UniqueConstraint, func
|
||||
|
||||
revision = "0012"
|
||||
down_revision = "0011"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table("kb_products", Column("id", Integer, primary_key=True), Column("fingerprint", String(64), nullable=False, unique=True), Column("gtin", String(16), unique=True), Column("identity_kind", String(16), nullable=False, server_default="FINGERPRINT"), Column("normalized_description", Text, nullable=False, server_default=""), Column("brand", Text), Column("unit", String(32)), Column("package_quantity", String(32)), Column("category", Text), Column("semantic_attributes", JSON, nullable=False), Column("created_at", DateTime(timezone=True), server_default=func.now()), Column("updated_at", DateTime(timezone=True), server_default=func.now()))
|
||||
op.create_index("ix_kb_products_gtin", "kb_products", ["gtin"])
|
||||
op.create_table("kb_knowledge", Column("id", Integer, primary_key=True), Column("kb_product_id", Integer, ForeignKey("kb_products.id"), nullable=False), Column("knowledge_key", String(48), nullable=False), Column("value", Text, nullable=False), Column("source", String(32), nullable=False), Column("confidence", Float, nullable=False, server_default="0"), Column("evidence", JSON, nullable=False), Column("model", String(64)), Column("prompt_version", String(32)), Column("dataset_versions", JSON, nullable=False), Column("created_at", DateTime(timezone=True), server_default=func.now()), Column("validated_at", DateTime(timezone=True)), UniqueConstraint("kb_product_id", "knowledge_key", "value", name="uq_kb_knowledge_value"))
|
||||
op.create_index("ix_kb_knowledge_kb_product_id", "kb_knowledge", ["kb_product_id"])
|
||||
op.create_index("ix_kb_knowledge_knowledge_key", "kb_knowledge", ["knowledge_key"])
|
||||
op.create_table("kb_observations", Column("id", Integer, primary_key=True), Column("kb_product_id", Integer, ForeignKey("kb_products.id"), nullable=False), Column("tenant_id", Integer, ForeignKey("tenants.id"), nullable=False), Column("company_key", String(64), nullable=False), Column("erp_code", String(32), nullable=False), Column("database_label", String(64), nullable=False), Column("observed_fields", JSON, nullable=False), Column("source", String(32), nullable=False, server_default="ERP_OBSERVATION"), Column("observed_at", DateTime(timezone=True), server_default=func.now()))
|
||||
op.create_index("ix_kb_observations_kb_product_id", "kb_observations", ["kb_product_id"])
|
||||
op.create_table("tenant_product_context", Column("id", Integer, primary_key=True), Column("tenant_id", Integer, ForeignKey("tenants.id"), nullable=False), Column("company_key", String(64), nullable=False), Column("erp_code", String(32), nullable=False), Column("kb_product_id", Integer, ForeignKey("kb_products.id")), Column("product_role", String(16)), Column("st_withheld", String(4)), Column("operation_profile", JSON, nullable=False), Column("source", String(32), nullable=False, server_default="ERP_OBSERVATION"), Column("confidence", Float, nullable=False, server_default="0"), Column("created_at", DateTime(timezone=True), server_default=func.now()), Column("updated_at", DateTime(timezone=True), server_default=func.now()), UniqueConstraint("tenant_id", "company_key", "erp_code", name="uq_tenant_product_context"))
|
||||
op.create_index("ix_tenant_product_context_tenant_id", "tenant_product_context", ["tenant_id"])
|
||||
op.create_table("ai_work_items", Column("id", Integer, primary_key=True), Column("task_type", String(24), nullable=False), Column("dedup_key", String(160), nullable=False, unique=True), Column("question_refs", JSON, nullable=False), Column("payload", JSON, nullable=False), Column("status", String(24), nullable=False, server_default="PENDING"), Column("kb_product_id", Integer, ForeignKey("kb_products.id")), Column("cells_covered", Integer, nullable=False, server_default="0"), Column("created_at", DateTime(timezone=True), server_default=func.now()), Column("updated_at", DateTime(timezone=True), server_default=func.now()))
|
||||
op.create_index("ix_ai_work_items_task_type", "ai_work_items", ["task_type"])
|
||||
op.create_index("ix_ai_work_items_status", "ai_work_items", ["status"])
|
||||
op.create_table("ai_runs", Column("id", Integer, primary_key=True), Column("provider", String(64), nullable=False), Column("model", String(64), nullable=False), Column("prompt_version", String(32), nullable=False), Column("work_item_ids", JSON, nullable=False), Column("input_tokens", Integer, nullable=False, server_default="0"), Column("output_tokens", Integer, nullable=False, server_default="0"), Column("cost_usd", Float, nullable=False, server_default="0"), Column("status", String(16), nullable=False, server_default="RUN"), Column("results", JSON, nullable=False), Column("budget_guard", JSON, nullable=False), Column("started_at", DateTime(timezone=True), server_default=func.now()), Column("finished_at", DateTime(timezone=True)))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("ai_runs")
|
||||
op.drop_table("ai_work_items")
|
||||
op.drop_table("tenant_product_context")
|
||||
op.drop_table("kb_observations")
|
||||
op.drop_table("kb_knowledge")
|
||||
op.drop_index("ix_kb_products_gtin", table_name="kb_products")
|
||||
op.drop_table("kb_products")
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Migration 0013: widen ai_work_items.status for V2 classification labels.
|
||||
|
||||
Revision ID: 0013
|
||||
Revises: 0012
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "0013"
|
||||
down_revision = "0012"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.alter_column("ai_work_items", "status", existing_type=sa.String(24), type_=sa.String(32), nullable=False, existing_server_default="PENDING")
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.alter_column("ai_work_items", "status", existing_type=sa.String(32), type_=sa.String(24), nullable=False, existing_server_default="PENDING")
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Migration 0014: persistent LLM eval result cache.
|
||||
|
||||
Revision ID: 0014
|
||||
Revises: 0013
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, DateTime, ForeignKey, Integer, JSON, String, UniqueConstraint, func
|
||||
|
||||
revision = "0014"
|
||||
down_revision = "0013"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table("ai_eval_cache", Column("id", Integer, primary_key=True), Column("input_fingerprint", String(64), nullable=False), Column("candidate_set_hash", String(64), nullable=False), Column("prompt_version", String(32), nullable=False), Column("model", String(64), nullable=False), Column("dataset_version_ncm", String(64), nullable=False), Column("phase", String(16), nullable=False), Column("answer", JSON, nullable=False), Column("gold_ncm", String(8)), Column("created_at", DateTime(timezone=True), server_default=func.now()), UniqueConstraint("input_fingerprint", "candidate_set_hash", "prompt_version", "model", "dataset_version_ncm", name="uq_ai_eval_cache"))
|
||||
op.create_index("ix_ai_eval_cache_input_fingerprint", "ai_eval_cache", ["input_fingerprint"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("ai_eval_cache")
|
||||
@@ -0,0 +1,8 @@
|
||||
"""AI semantic layer V1 — NOT a fiscal authority.
|
||||
|
||||
Rules frozen by the operator contract:
|
||||
- AI only selects among official candidates; it never invents codes.
|
||||
- Cache first: KB exact/fingerprint hits never trigger AI calls.
|
||||
- No AI for origin, ST retention, or company-level decisions.
|
||||
- AI_WRITE_ALLOWED=false in V1: results stay AI_CANDIDATE_VALIDATED.
|
||||
"""
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Deterministic candidate generation from the ACTIVE official datasets.
|
||||
|
||||
The AI never receives an open question: for every task type the Hub first builds
|
||||
the official candidate list (NCM top-K by description, CEST links for the NCM,
|
||||
TIPI EX rows for the NCM). Providers may only pick from this list.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import CestEntry, CestNcmEntry, Dataset, NcmEntry, TipiEntry
|
||||
from .fingerprint import token_stems
|
||||
|
||||
|
||||
def _active_dataset(session: Session, kind: str):
|
||||
return session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE"))
|
||||
|
||||
|
||||
def ncm_top_k(session: Session, description, k: int = 10):
|
||||
"""Top-K official NCM codes by (stemmed) token overlap with the product description."""
|
||||
dataset = _active_dataset(session, "NCM")
|
||||
if dataset is None:
|
||||
return []
|
||||
stems = token_stems(description)
|
||||
if not stems:
|
||||
return []
|
||||
scored = []
|
||||
for entry in session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all():
|
||||
score = len(stems & token_stems(entry.description)) / len(stems)
|
||||
if score > 0:
|
||||
scored.append((entry.code, entry.description, round(score, 4)))
|
||||
scored.sort(key=lambda row: (-row[2], row[0]))
|
||||
return scored[:k]
|
||||
|
||||
|
||||
def cest_candidates_for_ncm(session: Session, ncm: str):
|
||||
dataset = _active_dataset(session, "CEST")
|
||||
if dataset is None:
|
||||
return []
|
||||
rows = session.execute(
|
||||
select(CestEntry, CestNcmEntry.ncm_code)
|
||||
.join(CestNcmEntry, CestNcmEntry.cest_entry_id == CestEntry.id)
|
||||
.where(CestEntry.dataset_id == dataset.id)
|
||||
).all()
|
||||
return list(dict.fromkeys(entry for entry, ncm_code in rows if ncm.startswith(ncm_code)))
|
||||
|
||||
|
||||
def ex_tipi_candidates_for_ncm(session: Session, ncm: str):
|
||||
dataset = _active_dataset(session, "TIPI_IPI")
|
||||
if dataset is None:
|
||||
return []
|
||||
return list(dict.fromkeys(
|
||||
entry for entry in session.scalars(select(TipiEntry).where(TipiEntry.dataset_id == dataset.id, TipiEntry.ncm == ncm, TipiEntry.ex.is_not(None))).all()
|
||||
))
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Metrics and error attribution for the real LLM NCM evaluation (sections 8-9)."""
|
||||
|
||||
|
||||
def metrics(rows) -> dict:
|
||||
"""rows: [{gold_in_candidates, selected, gold, confidence, invalid}]"""
|
||||
total = len(rows)
|
||||
correct = sum(1 for row in rows if not row["invalid"] and row["selected"] == row["gold"])
|
||||
wrong = sum(1 for row in rows if not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"])
|
||||
abstain = sum(1 for row in rows if not row["invalid"] and row["selected"] is None)
|
||||
invalid = sum(1 for row in rows if row["invalid"])
|
||||
gold_in = sum(1 for row in rows if row["gold_in_candidates"])
|
||||
answered = correct + wrong
|
||||
high_conf = [row for row in rows if not row["invalid"] and row["confidence"] >= 0.95 and row["selected"] is not None]
|
||||
high_conf_wrong = sum(1 for row in high_conf if row["selected"] != row["gold"])
|
||||
return {
|
||||
"TOTAL": total,
|
||||
"CORRECT": correct,
|
||||
"WRONG": wrong,
|
||||
"ABSTAIN": abstain,
|
||||
"INVALID": invalid,
|
||||
"GOLD_IN_CANDIDATES": gold_in,
|
||||
"ACCURACY_ALL": round(correct / total, 4) if total else None,
|
||||
"SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE": round(correct / answered, 4) if answered else None,
|
||||
"PRECISION_WHEN_HIGH_CONFIDENCE": round(sum(1 for row in high_conf if row["selected"] == row["gold"]) / len(high_conf), 4) if high_conf else None,
|
||||
"HIGH_CONFIDENCE_WRONG_RATE": round(high_conf_wrong / len(high_conf), 4) if high_conf else None,
|
||||
"ABSTENTION_RATE": round(abstain / total, 4) if total else None,
|
||||
}
|
||||
|
||||
|
||||
def attribution(rows) -> dict:
|
||||
return {
|
||||
"GOLD_NOT_IN_CANDIDATES": sum(1 for row in rows if not row["gold_in_candidates"]),
|
||||
"GOLD_IN_CANDIDATES_BUT_WRONG": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"]),
|
||||
"GOLD_IN_CANDIDATES_BUT_ABSTAIN": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is None),
|
||||
"INVALID_OUTPUT": sum(1 for row in rows if row["invalid"]),
|
||||
}
|
||||
|
||||
|
||||
_BUCKETS = ((">=0.99", 0.99, 1.01), ("0.95-0.9899", 0.95, 0.99), ("0.90-0.9499", 0.90, 0.95), ("0.80-0.8999", 0.80, 0.90), ("<0.80", -0.01, 0.80))
|
||||
|
||||
|
||||
def confidence_buckets(rows) -> dict:
|
||||
out = {name: {"total": 0, "correct": 0} for name, _, _ in _BUCKETS}
|
||||
for row in rows:
|
||||
if row["invalid"] or row["selected"] is None:
|
||||
continue
|
||||
for name, low, high in _BUCKETS:
|
||||
if low <= row["confidence"] < high:
|
||||
out[name]["total"] += 1
|
||||
if row["selected"] == row["gold"]:
|
||||
out[name]["correct"] += 1
|
||||
break
|
||||
return out
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Product identity: stable fingerprints and GTIN validation.
|
||||
|
||||
Global identity uses ONLY stable attributes (normalized description, brand, unit,
|
||||
package quantity, GTIN). Fiscal/company context (CFOP, CSOSN, CST, rate, ST, tenant)
|
||||
never participates: those vary per context and must not merge distinct knowledge.
|
||||
"""
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from ..classification import ENGINE_VERSION # noqa: F401 (re-exported for the AI layer)
|
||||
|
||||
|
||||
def normalize_description(value) -> str:
|
||||
"""NFKD -> ascii -> A-Z0-9 tokens, space-joined (same canonical form as classification)."""
|
||||
text = unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper()
|
||||
return " ".join(re.sub(r"[^A-Z0-9]+", " ", text).split())
|
||||
|
||||
|
||||
def _normalize_token(value) -> str:
|
||||
return normalize_description(value)
|
||||
|
||||
|
||||
def _normalize_quantity(value) -> str:
|
||||
digits = re.sub(r"[^0-9.,]", "", str(value or ""))
|
||||
if not digits:
|
||||
return ""
|
||||
try:
|
||||
number = float(digits.replace(",", "."))
|
||||
except ValueError:
|
||||
return digits
|
||||
return f"{number:g}"
|
||||
|
||||
|
||||
def product_fingerprint(description, brand=None, unit=None, package_quantity=None) -> str:
|
||||
parts = [_normalize_token(description), _normalize_token(brand), _normalize_token(unit), _normalize_quantity(package_quantity)]
|
||||
return hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _stem(token: str) -> str:
|
||||
"""Minimal Portuguese plural stemming so AGUAS matches AGUA / MINERAIS matches MINERAL."""
|
||||
if len(token) > 4 and token.endswith("AIS"):
|
||||
return token[:-3] + "AL"
|
||||
if len(token) > 4 and token.endswith("ES"):
|
||||
return token[:-2]
|
||||
if len(token) > 3 and token.endswith("S"):
|
||||
return token[:-1]
|
||||
return token
|
||||
|
||||
|
||||
def token_stems(value) -> set:
|
||||
return {_stem(token) for token in normalize_description(value).split()}
|
||||
|
||||
|
||||
def overlap_score(query, official) -> float:
|
||||
"""Fraction of query description stems present in the official description stems."""
|
||||
stems = token_stems(query)
|
||||
if not stems:
|
||||
return 0.0
|
||||
return len(stems & token_stems(official)) / len(stems)
|
||||
|
||||
|
||||
def validate_gtin(value) -> str | None:
|
||||
"""Returns the canonical GTIN when the checksum is valid, else None.
|
||||
Accepts GTIN-8/12/13/14 (EAN-8/UPC-12/EAN-13/GTIN-14)."""
|
||||
digits = re.sub(r"\D", "", str(value or ""))
|
||||
if len(digits) not in (8, 12, 13, 14):
|
||||
return None
|
||||
total = 0
|
||||
for index, char in enumerate(reversed(digits)):
|
||||
digit = int(char)
|
||||
total += digit * (1 if index % 2 == 0 else 3)
|
||||
if total % 10 != 0:
|
||||
return None
|
||||
return digits
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Product Knowledge Base service: identity resolution, versioned knowledge, conflicts.
|
||||
|
||||
Identity order (operator contract): 1. exact GTIN; 2. strong fingerprint; 3. semantic
|
||||
fingerprint candidate; 4. unknown. Conflicting knowledge for the same key is kept
|
||||
side by side and surfaced as KNOWLEDGE_CONFLICT — never picked by majority.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func as sa_func, select
|
||||
from ..models import KbKnowledge, KbObservation, KbProduct
|
||||
|
||||
EXACT, FINGERPRINT = "EXACT", "FINGERPRINT"
|
||||
_AUTHORITATIVE = {"OFFICIAL_DATASET", "OPERATOR", "AI_PLUS_OFFICIAL_VALIDATION"}
|
||||
|
||||
|
||||
class KnowledgeBase:
|
||||
def __init__(self, session):
|
||||
self.session = session
|
||||
|
||||
# ----- identity -----
|
||||
|
||||
def upsert_product(self, fingerprint, gtin=None, normalized_description="", brand=None, unit=None, package_quantity=None, category=None):
|
||||
gtin = gtin or None
|
||||
product = None
|
||||
if gtin:
|
||||
product = self.session.scalar(select(KbProduct).where(KbProduct.gtin == gtin))
|
||||
if product is None:
|
||||
product = self.session.scalar(select(KbProduct).where(KbProduct.fingerprint == fingerprint))
|
||||
if product is None:
|
||||
product = KbProduct(fingerprint=fingerprint, normalized_description=normalized_description or "")
|
||||
self.session.add(product)
|
||||
# identity upgrades (never downgrades)
|
||||
if gtin and not product.gtin:
|
||||
product.gtin = gtin
|
||||
product.identity_kind = "GTIN"
|
||||
if normalized_description and not product.normalized_description:
|
||||
product.normalized_description = normalized_description
|
||||
for field, value in (("brand", brand), ("unit", unit), ("package_quantity", package_quantity), ("category", category)):
|
||||
if value and not getattr(product, field):
|
||||
setattr(product, field, value)
|
||||
self.session.flush()
|
||||
return product
|
||||
|
||||
def resolve(self, gtin=None, fingerprint=None):
|
||||
"""Returns (product|None, hit_kind). Cache-first: EXACT (GTIN) beats FINGERPRINT."""
|
||||
if gtin:
|
||||
product = self.session.scalar(select(KbProduct).where(KbProduct.gtin == gtin))
|
||||
if product:
|
||||
return product, EXACT
|
||||
if fingerprint:
|
||||
product = self.session.scalar(select(KbProduct).where(KbProduct.fingerprint == fingerprint))
|
||||
if product:
|
||||
return product, FINGERPRINT
|
||||
return None, None
|
||||
|
||||
# ----- knowledge -----
|
||||
|
||||
def add_knowledge(self, kb_product_id, knowledge_key, value, source, confidence=0.0, evidence=None, model=None, prompt_version=None, dataset_versions=None):
|
||||
value = str(value)
|
||||
row = self.session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == kb_product_id, KbKnowledge.knowledge_key == knowledge_key, KbKnowledge.value == value))
|
||||
if row is None:
|
||||
row = KbKnowledge(kb_product_id=kb_product_id, knowledge_key=knowledge_key, value=value, source=source, confidence=confidence, evidence=evidence or {}, model=model, prompt_version=prompt_version, dataset_versions=dataset_versions or {})
|
||||
self.session.add(row)
|
||||
else:
|
||||
# authoritative sources upgrade provenance/confidence of the same value; history preserved
|
||||
if source in _AUTHORITATIVE:
|
||||
row.source = source
|
||||
row.confidence = max(row.confidence, confidence)
|
||||
row.evidence = {**row.evidence, **(evidence or {})}
|
||||
if dataset_versions:
|
||||
row.dataset_versions = {**row.dataset_versions, **dataset_versions}
|
||||
if source == "OFFICIAL_DATASET":
|
||||
row.validated_at = datetime.now(timezone.utc)
|
||||
self.session.flush()
|
||||
return row
|
||||
|
||||
def current_knowledge(self, kb_product_id):
|
||||
rows = self.session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == kb_product_id)).all()
|
||||
by_key: dict = {}
|
||||
for row in rows:
|
||||
by_key.setdefault(row.knowledge_key, []).append(row)
|
||||
result = {}
|
||||
for key, entries in by_key.items():
|
||||
values = {entry.value for entry in entries}
|
||||
if len(values) > 1:
|
||||
result[key] = {"conflict": True, "values": values}
|
||||
else:
|
||||
best = max(entries, key=lambda entry: entry.confidence)
|
||||
result[key] = {"conflict": False, "value": best.value, "source": best.source, "confidence": best.confidence}
|
||||
return result
|
||||
|
||||
# ----- observations -----
|
||||
|
||||
def record_observation(self, kb_product_id, tenant_id, company_key, erp_code, database_label, observed_fields, source="ERP_OBSERVATION"):
|
||||
row = KbObservation(kb_product_id=kb_product_id, tenant_id=tenant_id, company_key=company_key, erp_code=erp_code, database_label=database_label, observed_fields=observed_fields or {}, source=source)
|
||||
self.session.add(row); self.session.flush()
|
||||
return row
|
||||
|
||||
def observation_summary(self, kb_product_id):
|
||||
rows = self.session.scalars(select(KbObservation).where(KbObservation.kb_product_id == kb_product_id)).all()
|
||||
agreement: dict = {}
|
||||
for row in rows:
|
||||
for field, value in (row.observed_fields or {}).items():
|
||||
if value in (None, ""):
|
||||
continue
|
||||
bucket = agreement.setdefault(field, {})
|
||||
bucket[str(value)] = bucket.get(str(value), 0) + 1
|
||||
conflicts = {field: values for field, values in agreement.items() if len(values) > 1}
|
||||
return {"observation_count": len(rows), "tenant_count": len({row.tenant_id for row in rows}), "agreement": agreement, "conflicts": conflicts}
|
||||
|
||||
# ----- stats -----
|
||||
|
||||
def stats(self):
|
||||
products = self.session.scalar(select(sa_func.count()).select_from(KbProduct)) or 0
|
||||
gtin_products = self.session.scalar(select(sa_func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
|
||||
conflicts = 0
|
||||
for product_id in self.session.scalars(select(KbProduct.id)).all():
|
||||
knowledge = self.current_knowledge(product_id)
|
||||
conflicts += sum(1 for entry in knowledge.values() if entry.get("conflict"))
|
||||
fingerprint_products = products - gtin_products
|
||||
return {"products": products, "gtin_products": gtin_products, "fingerprint_products": fingerprint_products, "conflicts": conflicts}
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Real LLM provider (OpenAI-compatible) for the NCM semantic evaluation.
|
||||
|
||||
Contract (mission REAL LLM EVALUATION V1):
|
||||
- The API key lives ONLY in the Hub environment (AI_API_KEY). Never committed, logged,
|
||||
written to artifacts, sent to the Agent or Firebird.
|
||||
- The gold answer is never part of the prompt: items carry only description/brand/unit/
|
||||
segment-weak-context + the official top-K hybrid candidates with hierarchical context.
|
||||
- Strict structured output: per item {work_item_id, selected_ncm, confidence, abstain,
|
||||
reason_short}; selected_ncm must be in the candidate list or null (abstain).
|
||||
- Batching is safe only when the response is valid JSON with 1:1 work_item_id matching,
|
||||
no omitted items, no cross-product mixing. Verified progressively (5/10/20).
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
PROMPT_VERSION_LLM = "llm-ncm-eval-v1"
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"You are a Brazilian customs classification assistant. For each product below, choose the "
|
||||
"most likely official NCM code from the provided candidate list, or abstain. "
|
||||
"Rules: selected_ncm MUST be one of the candidate codes for that exact item, or null with "
|
||||
"abstain=true when evidence is insufficient. Never invent a code. confidence is your calibrated "
|
||||
"probability in [0,1]. Reply with ONLY a JSON array, one object per item, in the same order, "
|
||||
'each: {"work_item_id": str, "selected_ncm": str|null, "confidence": float, "abstain": bool, "reason_short": str}.'
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmEvalItem:
|
||||
work_item_id: str
|
||||
description: str
|
||||
normalized_description: str
|
||||
brand: str | None
|
||||
unit: str | None
|
||||
segment: str
|
||||
candidates: list # [(code, official_description, hybrid_score, official_score, kb_score, tenant_count)]
|
||||
|
||||
|
||||
class LlmAnswerError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def item_prompt_payload(item: LlmEvalItem) -> dict:
|
||||
return {
|
||||
"work_item_id": item.work_item_id,
|
||||
"description": item.description,
|
||||
"normalized_description": item.normalized_description,
|
||||
"brand": item.brand,
|
||||
"unit": item.unit,
|
||||
"segment_weak_context": item.segment,
|
||||
"official_candidates": [
|
||||
{"ncm": code, "official_description": text}
|
||||
for code, text, *_ in item.candidates
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def candidate_set_hash(item: LlmEvalItem) -> str:
|
||||
codes = [code for code, *_ in item.candidates]
|
||||
return hashlib.sha256("|".join(codes).encode()).hexdigest()
|
||||
|
||||
|
||||
def input_fingerprint(item: LlmEvalItem) -> str:
|
||||
payload = {**item_prompt_payload(item), "system": _SYSTEM_PROMPT}
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
|
||||
|
||||
def validate_answer(answer: dict, item: LlmEvalItem) -> dict:
|
||||
"""Fail-closed validator: schema, work_item_id, candidate-list confinement."""
|
||||
if not isinstance(answer, dict):
|
||||
raise LlmAnswerError("answer must be an object")
|
||||
if str(answer.get("work_item_id")) != str(item.work_item_id):
|
||||
raise LlmAnswerError(f"work_item_id mismatch: {answer.get('work_item_id')!r} != {item.work_item_id!r}")
|
||||
abstain = bool(answer.get("abstain"))
|
||||
selected = answer.get("selected_ncm")
|
||||
selected = None if selected in (None, "") else str(selected)
|
||||
try:
|
||||
confidence = float(answer.get("confidence"))
|
||||
except (TypeError, ValueError):
|
||||
raise LlmAnswerError("confidence must be numeric")
|
||||
if not 0.0 <= confidence <= 1.0:
|
||||
raise LlmAnswerError("confidence out of [0,1]")
|
||||
codes = {code for code, *_ in item.candidates}
|
||||
if abstain:
|
||||
if selected is not None:
|
||||
raise LlmAnswerError("abstain must carry selected_ncm=null")
|
||||
else:
|
||||
if selected is None:
|
||||
raise LlmAnswerError("non-abstain requires selected_ncm")
|
||||
if selected not in codes:
|
||||
raise LlmAnswerError(f"selected_ncm {selected!r} is outside the candidate list")
|
||||
return {"work_item_id": item.work_item_id, "selected_ncm": selected, "confidence": confidence, "abstain": abstain, "reason_short": str(answer.get("reason_short") or "")[:200]}
|
||||
|
||||
|
||||
class OpenAICompatibleProvider:
|
||||
"""Talks to any OpenAI-compatible /chat/completions endpoint. `transport` is
|
||||
injectable for tests; production default uses httpx with the key from settings only."""
|
||||
|
||||
name = "openai-compatible"
|
||||
prompt_version = PROMPT_VERSION_LLM
|
||||
|
||||
def __init__(self, api_key: str, model: str, base_url: str = "https://api.openai.com/v1", transport=None, timeout: float = 120.0):
|
||||
self.api_key = api_key
|
||||
self.model = model
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout = timeout
|
||||
self._transport = transport or self._httpx_transport
|
||||
self.http_calls = 0
|
||||
self.retries = 0
|
||||
self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
|
||||
|
||||
def _httpx_transport(self, payload: dict) -> dict:
|
||||
import httpx
|
||||
response = httpx.post(f"{self.base_url}/chat/completions", json=payload, headers={"Authorization": f"Bearer {self.api_key}"}, timeout=self.timeout)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def list_models(self) -> list:
|
||||
"""GET /models with the account key — validates auth/base URL/model availability
|
||||
without spending completion tokens. Never raises with the key attached."""
|
||||
import httpx
|
||||
response = httpx.get(f"{self.base_url}/models", headers={"Authorization": f"Bearer {self.api_key}"}, timeout=self.timeout)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return sorted(str(item.get("id")) for item in data.get("data", []) if item.get("id"))
|
||||
|
||||
def _request(self, messages: list, max_retries: int = 2) -> dict:
|
||||
payload = {"model": self.model, "messages": messages, "temperature": 0}
|
||||
last_error = None
|
||||
for attempt in range(max_retries + 1):
|
||||
self.http_calls += 1
|
||||
if attempt:
|
||||
self.retries += 1
|
||||
try:
|
||||
data = self._transport(payload)
|
||||
usage = data.get("usage") or {}
|
||||
self.usage["input_tokens"] += int(usage.get("prompt_tokens") or 0)
|
||||
self.usage["output_tokens"] += int(usage.get("completion_tokens") or 0)
|
||||
cached = usage.get("prompt_tokens_details") or {}
|
||||
self.usage["cached_tokens"] += int(cached.get("cached_tokens") or 0)
|
||||
return data
|
||||
except Exception as error: # transport-level failure only; JSON contract errors are not retried
|
||||
last_error = error
|
||||
raise RuntimeError(f"LLM transport failed after {max_retries + 1} attempts: {last_error}")
|
||||
|
||||
def select_batch(self, items: list[LlmEvalItem]) -> list[dict]:
|
||||
"""One HTTP call for the whole batch. Raises LlmAnswerError on any 1:1/schema violation
|
||||
so the harness can shrink the batch size."""
|
||||
user_content = json.dumps([item_prompt_payload(item) for item in items], ensure_ascii=False)
|
||||
data = self._request([{"role": "system", "content": _SYSTEM_PROMPT}, {"role": "user", "content": user_content}])
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
content = re.sub(r"^```(?:json)?|```$", "", content.strip(), flags=re.MULTILINE).strip()
|
||||
answers = json.loads(content)
|
||||
if not isinstance(answers, list) or len(answers) != len(items):
|
||||
raise LlmAnswerError(f"expected {len(items)} answers, got {len(answers) if isinstance(answers, list) else type(answers).__name__}")
|
||||
return [validate_answer(answer, item) for answer, item in zip(answers, items)]
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Provider abstraction with strict structured output and a hard budget guard.
|
||||
|
||||
The fiscal engine is never coupled to a vendor: it talks to AISemanticProvider.
|
||||
V1 ships BaselineProvider (deterministic, zero-cost) used by the evaluation
|
||||
harness and as the no-credential fallback; real providers subclass the same
|
||||
contract and are only invoked with AI_ENABLED=true and credentials in the Hub.
|
||||
"""
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from ..classification import ENGINE_VERSION
|
||||
from .fingerprint import overlap_score
|
||||
|
||||
PROMPT_VERSION = "ai-semantic-v1"
|
||||
|
||||
|
||||
class SemanticOutputError(ValueError):
|
||||
"""Raised when a provider answer violates the strict structured-output schema."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class StructuredResult:
|
||||
task_type: str
|
||||
input_fingerprint: str
|
||||
selected_candidate: str # candidate code, or "NONE"
|
||||
confidence: float
|
||||
reason: str
|
||||
candidate_ids_seen: list[str]
|
||||
model: str
|
||||
prompt_version: str
|
||||
created_at: str
|
||||
dataset_versions: dict = field(default_factory=dict)
|
||||
engine_version: str = ENGINE_VERSION
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"task_type": self.task_type,
|
||||
"input_fingerprint": self.input_fingerprint,
|
||||
"selected_candidate": self.selected_candidate,
|
||||
"confidence": self.confidence,
|
||||
"reason": self.reason,
|
||||
"candidate_ids_seen": self.candidate_ids_seen,
|
||||
"model": self.model,
|
||||
"prompt_version": self.prompt_version,
|
||||
"created_at": self.created_at,
|
||||
"dataset_versions": self.dataset_versions,
|
||||
"engine_version": self.engine_version,
|
||||
}
|
||||
|
||||
|
||||
def result_from_dict(data) -> StructuredResult:
|
||||
"""Fail-closed schema check: the selection must be NONE or one of the seen candidates."""
|
||||
result = StructuredResult(
|
||||
task_type=str(data["task_type"]),
|
||||
input_fingerprint=str(data["input_fingerprint"]),
|
||||
selected_candidate=str(data["selected_candidate"]),
|
||||
confidence=float(data["confidence"]),
|
||||
reason=str(data["reason"]),
|
||||
candidate_ids_seen=[str(item) for item in data["candidate_ids_seen"]],
|
||||
model=str(data["model"]),
|
||||
prompt_version=str(data["prompt_version"]),
|
||||
created_at=str(data["created_at"]),
|
||||
dataset_versions=dict(data.get("dataset_versions") or {}),
|
||||
)
|
||||
if result.selected_candidate != "NONE" and result.selected_candidate not in result.candidate_ids_seen:
|
||||
raise SemanticOutputError(f"selected_candidate {result.selected_candidate!r} is not among the official candidates")
|
||||
if not 0.0 <= result.confidence <= 1.0:
|
||||
raise SemanticOutputError("confidence must be within [0,1]")
|
||||
return result
|
||||
|
||||
|
||||
class AISemanticProvider:
|
||||
"""Contract every provider must honour. `select` receives ONLY official candidates."""
|
||||
name = "abstract"
|
||||
model = "abstract"
|
||||
|
||||
def select(self, work_item, candidates) -> StructuredResult:
|
||||
raise NotImplementedError
|
||||
|
||||
def estimate(self, work_items) -> dict:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _score(text: str, official: str) -> float:
|
||||
return overlap_score(text, official)
|
||||
|
||||
|
||||
class BaselineProvider(AISemanticProvider):
|
||||
"""Deterministic zero-cost baseline: highest token-overlap candidate, or NONE.
|
||||
Used for the evaluation pipeline and as the mandatory no-credential fallback."""
|
||||
|
||||
name = "baseline"
|
||||
model = "baseline-deterministic-v1"
|
||||
|
||||
def select(self, work_item, candidates, dataset_versions=None) -> StructuredResult:
|
||||
description = work_item.payload.get("description", "") if hasattr(work_item, "payload") else ""
|
||||
scored = [(code, official, _score(description, official)) for code, official in [(str(c[0]), c[1]) for c in candidates]]
|
||||
best = max(scored, key=lambda row: row[2]) if scored else None
|
||||
selected, _, score = best if best else ("NONE", "", 0.0)
|
||||
if not scored or score <= 0.0:
|
||||
selected = "NONE"
|
||||
return StructuredResult(
|
||||
task_type=work_item.task_type,
|
||||
input_fingerprint=work_item.dedup_key,
|
||||
selected_candidate=selected,
|
||||
confidence=round(score, 4),
|
||||
reason="baseline token-overlap top-1" if selected != "NONE" else "no token overlap with any official candidate",
|
||||
candidate_ids_seen=[str(c[0]) for c in candidates],
|
||||
model=self.model,
|
||||
prompt_version=PROMPT_VERSION,
|
||||
created_at=datetime.now(timezone.utc).isoformat(),
|
||||
dataset_versions=dict(dataset_versions or {}),
|
||||
)
|
||||
|
||||
def estimate(self, work_items) -> dict:
|
||||
return {"input_tokens": 0, "output_tokens": 0, "estimated_cost": 0.0}
|
||||
|
||||
|
||||
class BudgetGuard:
|
||||
"""Refuses execution when the estimated cost exceeds AI_MAX_RUN_COST_USD.
|
||||
Never falls back to a more expensive model. Prices are operator-configured — when
|
||||
absent, cost is UNKNOWN: the residual run is refused, but an explicitly limited
|
||||
benchmark (e.g. the 65-item GOLD eval) may opt in with allow_unknown_cost."""
|
||||
|
||||
def __init__(self, max_run_cost_usd: float, price_in_per_m: float | None, price_out_per_m: float | None):
|
||||
self.max_run_cost_usd = float(max_run_cost_usd)
|
||||
self.price_in_per_m = None if price_in_per_m is None else float(price_in_per_m)
|
||||
self.price_out_per_m = None if price_out_per_m is None else float(price_out_per_m)
|
||||
|
||||
def estimate(self, items_in_tokens: int, items_out_tokens: int) -> dict:
|
||||
if self.price_in_per_m is None or self.price_out_per_m is None:
|
||||
return {"input_tokens": items_in_tokens, "output_tokens": items_out_tokens, "estimated_cost": None, "cost_unknown": True}
|
||||
cost = (items_in_tokens / 1_000_000) * self.price_in_per_m + (items_out_tokens / 1_000_000) * self.price_out_per_m
|
||||
return {"input_tokens": items_in_tokens, "output_tokens": items_out_tokens, "estimated_cost": round(cost, 6)}
|
||||
|
||||
def authorize(self, plan: dict, allow_unknown_cost: bool = False) -> bool:
|
||||
if plan.get("estimated_cost") is None:
|
||||
return allow_unknown_cost
|
||||
return plan["estimated_cost"] <= self.max_run_cost_usd
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Semantic candidate retrieval V2 — deterministic, zero LLM.
|
||||
|
||||
Three retrieval sources merged into one ranking:
|
||||
A. OFFICIAL: hierarchical search over the official NCM tree. Every 8-digit code is
|
||||
enriched with its ancestors' descriptions (chapter/position/subposition), so the
|
||||
very short legal text carries the vocabulary of its path (section 6).
|
||||
B. KB NEIGHBORS: the Knowledge Base acts as a commercial->official bridge — products
|
||||
whose commercial description resembles the query contribute their VALIDATED NCMs,
|
||||
weighted by similarity, tenant_count, observation_count and agreement (sections 2-3).
|
||||
C. HYBRID: single final ranking; the KB is only evidence for CANDIDATES, never a
|
||||
majority-vote fiscal decision.
|
||||
|
||||
Abbreviation lexicon (section 8) is derived statistically from the KB itself; commercial
|
||||
normalization (section 7) strips units/sizes/colors/refs deterministically.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import Dataset, KbKnowledge, KbObservation, KbProduct, NcmEntry, Tenant
|
||||
from .fingerprint import normalize_description, token_stems
|
||||
|
||||
# ---------------- commercial normalization (section 7) ----------------
|
||||
|
||||
_UNITS = {"UN", "UNID", "UNIDADE", "CX", "CAIXA", "FD", "FDO", "FARDO", "PCT", "PACOTE", "PKG",
|
||||
"KG", "G", "GR", "GRAMA", "MG", "ML", "L", "LT", "LITRO", "LATA", "GARRAFA", "PET",
|
||||
"PC", "PECA", "DZ", "DUZIA", "PAR", "KIT", "JG", "PARA", "COM", "SEM", "C", "S"}
|
||||
_DISCARD = _UNITS | {"MASC", "MASCULINO", "FEM", "FEMININO", "INF", "INFANTIL", "TAM", "TAMANHO",
|
||||
"COR", "REF", "COD", "CODIGO", "MOD", "MODELO", "NOVA", "NOVO", "TIPO"}
|
||||
_LEXEME_STRIP_PREFIXES = ("1", "2", "3", "4", "5", "6", "7", "8", "9", "0", ".")
|
||||
|
||||
|
||||
def _is_size_or_code(token: str) -> bool:
|
||||
if token.isdigit():
|
||||
return True
|
||||
digits = "".join(char for char in token if char.isdigit())
|
||||
letters = "".join(char for char in token if char.isalpha())
|
||||
return bool(digits) and letters in _UNITS # "1KG", "500ML", "2LT"
|
||||
|
||||
|
||||
def commercial_tokens(description) -> set:
|
||||
"""Stable commercial lexemes: units/sizes/colors/refs removed, stems kept."""
|
||||
tokens = []
|
||||
skip_next = False
|
||||
for token in normalize_description(description).split():
|
||||
if skip_next:
|
||||
skip_next = False
|
||||
continue
|
||||
if token in ("TAM", "COR", "REF", "COD", "MOD"):
|
||||
skip_next = True
|
||||
continue
|
||||
if token in _DISCARD or _is_size_or_code(token):
|
||||
continue
|
||||
tokens.append(token)
|
||||
return {_stem_safe(token) for token in tokens}
|
||||
|
||||
|
||||
def _stem_safe(token: str) -> str:
|
||||
from .fingerprint import _stem
|
||||
return _stem(token)
|
||||
|
||||
|
||||
def expand_stems(stems: set, lexicon: dict) -> set:
|
||||
expanded = set(stems)
|
||||
for stem in stems:
|
||||
target = lexicon.get(stem)
|
||||
if target:
|
||||
expanded.add(target)
|
||||
return expanded
|
||||
|
||||
|
||||
def derive_abbreviation_lexicon(corpus: dict, min_support: int = 3) -> dict:
|
||||
"""corpus: {commercial_description: ncm}. Statistical abbreviations only: a short token
|
||||
(3-5 chars) that is a strict prefix of a longer token, where BOTH forms occur across
|
||||
distinct products that share the same NCM. Support = (#products with the short form) +
|
||||
(#products with the long form); accepted when support >= min_support and both >= 1."""
|
||||
by_ncm: dict = {}
|
||||
for description, ncm in corpus.items():
|
||||
tokens = set(normalize_description(description).split())
|
||||
by_ncm.setdefault(ncm, []).append(tokens)
|
||||
best: dict = {}
|
||||
for descriptions in by_ncm.values():
|
||||
if len(descriptions) < 2:
|
||||
continue
|
||||
all_tokens = set().union(*descriptions)
|
||||
for short in all_tokens:
|
||||
if not (3 <= len(short) <= 5):
|
||||
continue
|
||||
short_count = sum(1 for tokens in descriptions if short in tokens)
|
||||
if short_count < 1:
|
||||
continue
|
||||
for long_token in all_tokens:
|
||||
if len(long_token) > len(short) and long_token.startswith(short):
|
||||
long_count = sum(1 for tokens in descriptions if long_token in tokens)
|
||||
support = short_count + long_count
|
||||
if support >= min_support and long_count >= 1 and short_count >= 1:
|
||||
current = best.get(short)
|
||||
if current is None or support > current[1]:
|
||||
best[short] = (long_token, support)
|
||||
return {short: target for short, (target, _) in best.items()}
|
||||
|
||||
|
||||
# ---------------- enriched official corpus (sections 5-6) ----------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnrichedNcmCorpus:
|
||||
documents: dict # code -> enriched stems set
|
||||
descriptions: dict # code -> official description
|
||||
positions: dict # 4-digit position -> enriched stems (chapter + position)
|
||||
children: dict # 4-digit position -> [8-digit codes]
|
||||
|
||||
@staticmethod
|
||||
def build(session: Session) -> "EnrichedNcmCorpus":
|
||||
dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
if dataset is None:
|
||||
return EnrichedNcmCorpus({}, {}, {}, {})
|
||||
entries = session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all()
|
||||
by_code = {entry.code: entry.description for entry in entries}
|
||||
documents, positions, children = {}, {}, {}
|
||||
for code, description in by_code.items():
|
||||
if len(code) != 8:
|
||||
continue
|
||||
ancestors = [text for text in (by_code.get(code[:2]), by_code.get(code[:4]), by_code.get(code[:6])) if text]
|
||||
documents[code] = token_stems(description) | set().union(*[token_stems(text) for text in ancestors]) if ancestors else token_stems(description)
|
||||
positions.setdefault(code[:4], token_stems(by_code.get(code[:4], "")) | token_stems(by_code.get(code[:2], "")))
|
||||
children.setdefault(code[:4], []).append(code)
|
||||
return EnrichedNcmCorpus(documents, by_code, positions, children)
|
||||
|
||||
def document(self, code: str) -> str:
|
||||
return " ".join(sorted(self.documents.get(code, set())))
|
||||
|
||||
def items(self):
|
||||
return list(self.documents.items())
|
||||
|
||||
|
||||
def _score(stems: set, document_stems: set, idf: dict | None = None) -> float:
|
||||
if not stems:
|
||||
return 0.0
|
||||
if idf:
|
||||
matched = sum(idf.get(stem, 1.0) for stem in stems if stem in document_stems)
|
||||
total = sum(idf.get(stem, 1.0) for stem in stems)
|
||||
return matched / total
|
||||
return len(stems & document_stems) / len(stems)
|
||||
|
||||
|
||||
def hierarchical_candidates(corpus: EnrichedNcmCorpus, description, k: int = 20, max_positions: int = 15):
|
||||
"""Two-stage: rank 4-digit positions first, then their 8-digit descendants (section 5)."""
|
||||
stems = commercial_tokens(description)
|
||||
if not stems:
|
||||
return []
|
||||
position_scores = sorted(((_score(stems, doc), position) for position, doc in corpus.positions.items()), reverse=True)
|
||||
ranked = []
|
||||
for position_score, position in position_scores[:max_positions]:
|
||||
if position_score <= 0:
|
||||
continue
|
||||
for code in corpus.children.get(position, []):
|
||||
code_score = _score(stems, corpus.documents[code])
|
||||
if code_score > 0:
|
||||
ranked.append((code, corpus.descriptions.get(code, ""), round(code_score, 4)))
|
||||
ranked.sort(key=lambda row: (-row[2], row[0]))
|
||||
return ranked[:k]
|
||||
|
||||
|
||||
# ---------------- KB neighbor bridge (sections 2-3) ----------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class KbNeighborIndex:
|
||||
entries: list # (product_id, stems, ncm, tenant_ids:set, observation_count, has_conflict)
|
||||
|
||||
@staticmethod
|
||||
def build(session: Session) -> "KbNeighborIndex":
|
||||
validated = {}
|
||||
for knowledge in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "validated_ncm")).all():
|
||||
validated.setdefault(knowledge.kb_product_id, set()).add(knowledge.value)
|
||||
conflicts = {product_id: len(values) > 1 for product_id, values in validated.items()}
|
||||
products = {product.id: product for product in session.scalars(select(KbProduct)).all()}
|
||||
observations = session.scalars(select(KbObservation)).all()
|
||||
per_product_tenants: dict = {}
|
||||
for observation in observations:
|
||||
per_product_tenants.setdefault(observation.kb_product_id, set()).add(observation.tenant_id)
|
||||
entries = []
|
||||
for product_id, values in validated.items():
|
||||
if conflicts.get(product_id):
|
||||
continue # conflicting knowledge never feeds neighbors
|
||||
for value in values:
|
||||
product = products.get(product_id)
|
||||
if product is None or not product.normalized_description:
|
||||
continue
|
||||
tenants = per_product_tenants.get(product_id, set())
|
||||
entries.append((product_id, commercial_tokens(product.normalized_description), value, tenants, len(tenants), False))
|
||||
return KbNeighborIndex(entries)
|
||||
|
||||
def corpus(self) -> dict:
|
||||
return {" ".join(sorted(stems)): ncm for _, stems, ncm, _, _, _ in self.entries}
|
||||
|
||||
|
||||
def kb_neighbor_candidates(index: KbNeighborIndex, description, k: int = 10, exclude_product_id: int | None = None):
|
||||
"""Aggregate NCMs of lexically similar known products. `exclude_product_id` implements
|
||||
leave-one-out evaluation: the queried product never retrieves itself. Returns
|
||||
(ncm, neighbor_score, observation_count, tenant_count) — agreement is the union of
|
||||
distinct tenants behind the contributing neighbors, reported as evidence, never used
|
||||
as an automatic majority decision."""
|
||||
stems = commercial_tokens(description)
|
||||
if not stems:
|
||||
return []
|
||||
best: dict = {}
|
||||
for product_id, neighbor_stems, ncm, tenant_ids, observation_count, _ in index.entries:
|
||||
if exclude_product_id is not None and product_id == exclude_product_id:
|
||||
continue
|
||||
similarity = _score(stems, neighbor_stems)
|
||||
if similarity <= 0:
|
||||
continue
|
||||
current = best.get(ncm)
|
||||
if current is None:
|
||||
best[ncm] = [ncm, round(similarity, 4), observation_count, set(tenant_ids)]
|
||||
else:
|
||||
current[1] = max(current[1], round(similarity, 4))
|
||||
current[3] |= set(tenant_ids)
|
||||
ranked = sorted(((row[0], row[1], row[2], len(row[3])) for row in best.values()), key=lambda row: (-row[1], -row[3], row[0]))
|
||||
return ranked[:k]
|
||||
|
||||
|
||||
# ---------------- hybrid ranking (section 4) ----------------
|
||||
|
||||
OFFICIAL_WEIGHT, KB_WEIGHT = 0.3, 0.7 # tuned on the V2 benchmark (KB bridge carries the signal;
|
||||
# the official component breaks ties and covers products absent from the KB)
|
||||
|
||||
|
||||
def hybrid_candidates(corpus: EnrichedNcmCorpus, index: KbNeighborIndex, description, k: int = 20, lexicon: dict | None = None, exclude_product_id: int | None = None):
|
||||
stems = expand_stems(commercial_tokens(description), lexicon or {})
|
||||
official = hierarchical_candidates(corpus, " ".join(sorted(stems)) or description, k=k)
|
||||
neighbors = kb_neighbor_candidates(index, description, k=k, exclude_product_id=exclude_product_id)
|
||||
combined: dict = {}
|
||||
for code, text, score in official:
|
||||
combined[code] = [code, text, OFFICIAL_WEIGHT * score, score, 0.0, 0]
|
||||
for code, similarity, observation_count, tenant_count in neighbors:
|
||||
row = combined.setdefault(code, [code, "", 0.0, 0.0, 0.0, 0])
|
||||
row[2] += KB_WEIGHT * similarity
|
||||
row[4] = similarity
|
||||
row[5] = tenant_count
|
||||
ranked = sorted(combined.values(), key=lambda row: (-round(row[2], 4), row[0]))
|
||||
return [(row[0], row[1], round(row[2], 4), row[3], row[4], row[5]) for row in ranked[:k]]
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Residual-question dedup: group pending cells by the KNOWLEDGE needed, not per cell.
|
||||
|
||||
- INVALID_NCM -> one work item per product fingerprint (same product in N databases collapses)
|
||||
- CEST/EX_TIPI -> one work item per (doubt, NCM): all databases sharing that NCM collapse
|
||||
- PRODUCT_ROLE -> tenant-specific by contract: one work item per (company, group)
|
||||
- Everything else (origin, ST retention, company profile, duplicate key) never becomes
|
||||
an AI work item.
|
||||
"""
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
_INVALID_NCM, _ROLE = "ncm", "product_role"
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkItem:
|
||||
task_type: str
|
||||
dedup_key: str
|
||||
question_refs: list = field(default_factory=list)
|
||||
cells_covered: int = 0
|
||||
payload: dict = field(default_factory=dict)
|
||||
kb_product_id: int | None = None
|
||||
|
||||
|
||||
def build_work_items(questions, product_fingerprints=None) -> list[WorkItem]:
|
||||
"""`product_fingerprints` maps "{DATABASE}:{erp_code}" -> KB fingerprint (from the seed pass)."""
|
||||
product_fingerprints = product_fingerprints or {}
|
||||
items: dict = {}
|
||||
|
||||
def _get(task_type, dedup_key, ref, cells):
|
||||
item = items.get(dedup_key)
|
||||
if item is None:
|
||||
item = WorkItem(task_type=task_type, dedup_key=dedup_key)
|
||||
items[dedup_key] = item
|
||||
item.question_refs.append(ref)
|
||||
item.cells_covered += cells
|
||||
return item
|
||||
|
||||
for question in questions:
|
||||
key = question.get("KEY")
|
||||
scope = question.get("SCOPE")
|
||||
cells = int(question.get("CELLS") or question.get("PRODUCTS_AFFECTED") or 0)
|
||||
ref = question.get("QUESTION_ID") or f"{question.get('DATABASE')}:{scope}:{question.get('NCM') or question.get('GROUP')}:{key}"
|
||||
if key in ("cest", "ex_tipi") and scope == "NCM":
|
||||
ncm = str(question.get("NCM") or "").strip()
|
||||
if ncm:
|
||||
item = _get(key.upper(), f"{key}:{ncm}", ref, cells)
|
||||
item.payload.setdefault("ncm", ncm)
|
||||
item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
|
||||
elif key == _INVALID_NCM and scope == "PRODUCT":
|
||||
database = question.get("DATABASE", "")
|
||||
erp_code = str(question.get("DATABASE_CODE") or "").strip()
|
||||
fingerprint = product_fingerprints.get(f"{database}:{erp_code}")
|
||||
dedup = f"invalid_ncm:{fingerprint}" if fingerprint else f"invalid_ncm:{database}:{erp_code}"
|
||||
item = _get("INVALID_NCM", dedup, ref, cells)
|
||||
item.payload.setdefault("erp_code", erp_code)
|
||||
item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
|
||||
elif key == _ROLE and scope == "GROUP":
|
||||
company = question.get("COMPANY", "")
|
||||
group = str(question.get("GROUP") or "").strip()
|
||||
if group:
|
||||
item = _get("PRODUCT_ROLE", f"product_role:{company}:{group}", ref, cells)
|
||||
item.payload.setdefault("company_key", company)
|
||||
item.payload["group"] = group
|
||||
return list(items.values())
|
||||
@@ -31,6 +31,20 @@ class Settings:
|
||||
cest_dataset_url: str = os.getenv("CEST_DATASET_URL", "https://www.confaz.fazenda.gov.br/legislacao/convenios/2018/CV142_18")
|
||||
dataset_storage_path: str = os.getenv("DATASET_STORAGE_PATH", "./data/datasets")
|
||||
http_timeout_seconds: float = float(os.getenv("HTTP_TIMEOUT_SECONDS", "30"))
|
||||
# ---- AI semantic layer V1 ----
|
||||
# Credentials live ONLY in the Hub environment; the Windows Agent never sees them.
|
||||
ai_enabled: bool = os.getenv("AI_ENABLED", "0") == "1"
|
||||
ai_provider: str = os.getenv("AI_PROVIDER", "baseline") # baseline | openai | ... (AISemanticProvider)
|
||||
ai_model: str = os.getenv("AI_MODEL", "baseline-deterministic-v1")
|
||||
ai_max_batch_size: int = int(os.getenv("AI_MAX_BATCH_SIZE", "20"))
|
||||
ai_max_run_cost_usd: float = float(os.getenv("AI_MAX_RUN_COST_USD", "5.0"))
|
||||
ai_min_confidence: float = float(os.getenv("AI_MIN_CONFIDENCE", "0.6"))
|
||||
ai_price_in_per_m: float | None = float(os.getenv("AI_PRICE_IN_PER_M")) if os.getenv("AI_PRICE_IN_PER_M") else None
|
||||
ai_price_out_per_m: float | None = float(os.getenv("AI_PRICE_OUT_PER_M")) if os.getenv("AI_PRICE_OUT_PER_M") else None
|
||||
# Real-LLM evaluation (OpenAI-compatible). The key lives ONLY in this environment.
|
||||
# Z.ai general endpoint (NOT the /api/coding/paas/v4 coding-plan endpoint).
|
||||
ai_api_key: str = os.getenv("AI_API_KEY", "")
|
||||
ai_base_url: str = os.getenv("AI_BASE_URL", "https://api.z.ai/api/paas/v4")
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
@@ -7,7 +7,9 @@ ENUMS = {
|
||||
"st_status": {"SUBJECT", "NOT_SUBJECT"},
|
||||
"st_withheld": {"Y", "N"},
|
||||
"icms_treatment": {"TAXED", "EXEMPT", "NOT_TAXED", "ST_WITHHELD", "SUSPENDED", "DEFERRED"},
|
||||
"entry_scope": {"INTRA", "INTER"},
|
||||
# MIXED_OR_DYNAMIC: purchases are not predominantly intra- or interstate, so no single
|
||||
# static entry CFOP is fiscally correct for all operations (section 0, AI semantic layer V1).
|
||||
"entry_scope": {"INTRA", "INTER", "MIXED_OR_DYNAMIC"},
|
||||
"rounding": {"A", "T"},
|
||||
"piscofins": {"N", "S"},
|
||||
"incidence": {"NENHUMA"},
|
||||
|
||||
@@ -362,20 +362,29 @@ def resolve_product(p, d, company, refs, ev, ctx, manual_ncm=None):
|
||||
out["CFOP_ENTRADA"] = blocked("CFOP_ENTRADA")
|
||||
else:
|
||||
exp_e = None
|
||||
if scope_ctx in ("INTRA", "INTER") and role in ("RESALE", "OWN_PRODUCTION", "INPUT") and st != "UNKNOWN" and not (st == "SUBJECT" and withheld is None):
|
||||
first = "1" if scope_ctx == "INTRA" else "2"
|
||||
stw = st == "SUBJECT" and withheld == "Y"
|
||||
exp_e = first + ("403" if stw else "102") if role == "RESALE" else first + ("401" if stw else "101")
|
||||
valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
|
||||
if valid_e and exp_e and cfe != exp_e and cfe in ENTRY_CFOP:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "PROVEN_ROLE_ST_AND_SCOPE_CONTRADICT_DECLARED_CFOP_ENTRADA")
|
||||
elif valid_e:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif exp_e:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "DERIVED_FROM_ROLE_ST_AND_ENTRY_SCOPE")
|
||||
if scope_ctx == "MIXED_OR_DYNAMIC":
|
||||
# Scope answered, but no single static entry CFOP is fiscally correct for every
|
||||
# operation: the ERP default / each operation decides 1xxx vs 2xxx. Never invent one.
|
||||
valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
|
||||
if valid_e:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_OPERATION_DEPENDENT_MIXED_SCOPE")
|
||||
else:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", MANUAL, cfe, reason="CFOP_ENTRADA_OPERATION_DEPENDENT_MIXED_SCOPE")
|
||||
else:
|
||||
key = "product_role" if role == "UNKNOWN" else ("entry_scope" if scope_ctx not in ("INTRA", "INTER") else ("st_status" if st == "UNKNOWN" else "st_withheld"))
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", MANUAL, cfe, reason="CFOP_ENTRADA_NEEDS_" + key.upper(), key=key, scope="COMPANY" if key == "entry_scope" else "GROUP")
|
||||
if scope_ctx in ("INTRA", "INTER") and role in ("RESALE", "OWN_PRODUCTION", "INPUT") and st != "UNKNOWN" and not (st == "SUBJECT" and withheld is None):
|
||||
first = "1" if scope_ctx == "INTRA" else "2"
|
||||
stw = st == "SUBJECT" and withheld == "Y"
|
||||
exp_e = first + ("403" if stw else "102") if role == "RESALE" else first + ("401" if stw else "101")
|
||||
valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
|
||||
if valid_e and exp_e and cfe != exp_e and cfe in ENTRY_CFOP:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "PROVEN_ROLE_ST_AND_SCOPE_CONTRADICT_DECLARED_CFOP_ENTRADA")
|
||||
elif valid_e:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif exp_e:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "DERIVED_FROM_ROLE_ST_AND_ENTRY_SCOPE")
|
||||
else:
|
||||
key = "product_role" if role == "UNKNOWN" else ("entry_scope" if scope_ctx not in ("INTRA", "INTER") else ("st_status" if st == "UNKNOWN" else "st_withheld"))
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", MANUAL, cfe, reason="CFOP_ENTRADA_NEEDS_" + key.upper(), key=key, scope="COMPANY" if key == "entry_scope" else "GROUP")
|
||||
|
||||
# ---------------- origin (needed by CST) ----------------
|
||||
cod_o = cur["COD_ORIGEM_MERCADORIA"]
|
||||
|
||||
@@ -339,3 +339,128 @@ class FiscalContextEntry(Base):
|
||||
reason: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
# ================= AI semantic layer V1 =================
|
||||
# The AI layer is NOT a fiscal authority: it only selects among official candidates and
|
||||
# never writes to Firebird. Knowledge is versioned, provenance-tracked and never deleted.
|
||||
|
||||
|
||||
class KbProduct(Base):
|
||||
"""GLOBAL product knowledge: only facts inherent to the product (never tenant context)."""
|
||||
__tablename__ = "kb_products"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
||||
gtin: Mapped[str | None] = mapped_column(String(16), unique=True, nullable=True, index=True)
|
||||
identity_kind: Mapped[str] = mapped_column(String(16), default="FINGERPRINT") # GTIN | FINGERPRINT | SEMANTIC | UNKNOWN
|
||||
normalized_description: Mapped[str] = mapped_column(Text, default="")
|
||||
brand: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
unit: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
package_quantity: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
category: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
semantic_attributes: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class KbKnowledge(Base):
|
||||
"""Versioned knowledge rows. Unique per (product, key, value): conflicting values for the
|
||||
same key are kept side by side and surfaced as KNOWLEDGE_CONFLICT, never majority-picked."""
|
||||
__tablename__ = "kb_knowledge"
|
||||
__table_args__ = (UniqueConstraint("kb_product_id", "knowledge_key", "value", name="uq_kb_knowledge_value"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
|
||||
knowledge_key: Mapped[str] = mapped_column(String(48), index=True) # candidate_ncm | validated_ncm | candidate_cest | candidate_ex_tipi | ...
|
||||
value: Mapped[str] = mapped_column(Text)
|
||||
source: Mapped[str] = mapped_column(String(32)) # OFFICIAL_DATASET | ERP_OBSERVATION | OPERATOR | AI | AI_PLUS_OFFICIAL_VALIDATION
|
||||
confidence: Mapped[float] = mapped_column(default=0.0)
|
||||
evidence: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
model: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
prompt_version: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
dataset_versions: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
validated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class KbObservation(Base):
|
||||
"""Raw per-tenant ERP observation of a KB product (evidence aggregation only)."""
|
||||
__tablename__ = "kb_observations"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
company_key: Mapped[str] = mapped_column(String(64), index=True)
|
||||
erp_code: Mapped[str] = mapped_column(String(32))
|
||||
database_label: Mapped[str] = mapped_column(String(64))
|
||||
observed_fields: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
source: Mapped[str] = mapped_column(String(32), default="ERP_OBSERVATION")
|
||||
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class TenantProductContext(Base):
|
||||
"""TENANT-specific product context: role, ST-withheld, operation profile, operator decisions.
|
||||
Never promoted to global knowledge."""
|
||||
__tablename__ = "tenant_product_context"
|
||||
__table_args__ = (UniqueConstraint("tenant_id", "company_key", "erp_code", name="uq_tenant_product_context"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
company_key: Mapped[str] = mapped_column(String(64), index=True)
|
||||
erp_code: Mapped[str] = mapped_column(String(32))
|
||||
kb_product_id: Mapped[int | None] = mapped_column(ForeignKey("kb_products.id"), nullable=True)
|
||||
product_role: Mapped[str | None] = mapped_column(String(16), nullable=True) # RESALE | OWN_PRODUCTION | INPUT | SERVICE | UNKNOWN
|
||||
st_withheld: Mapped[str | None] = mapped_column(String(4), nullable=True) # Y | N — never AI-decided
|
||||
operation_profile: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
source: Mapped[str] = mapped_column(String(32), default="ERP_OBSERVATION")
|
||||
confidence: Mapped[float] = mapped_column(default=0.0)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class AiWorkItem(Base):
|
||||
"""Deduplicated semantic work item (one knowledge need, many pending cells)."""
|
||||
__tablename__ = "ai_work_items"
|
||||
__table_args__ = (UniqueConstraint("dedup_key", name="uq_ai_work_item_dedup"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
task_type: Mapped[str] = mapped_column(String(24), index=True) # INVALID_NCM | CEST | EX_TIPI | PRODUCT_ROLE
|
||||
dedup_key: Mapped[str] = mapped_column(String(160))
|
||||
question_refs: Mapped[list] = mapped_column(JSON, default=list)
|
||||
payload: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
status: Mapped[str] = mapped_column(String(32), default="PENDING", index=True) # PENDING | KB_EXACT_HIT | KB_FINGERPRINT_HIT | HIGH_CONFIDENCE_RETRIEVAL | SINGLE_OFFICIAL_CANDIDATE | LLM_SELECTION_REQUIRED | UNRESOLVED | UNRESOLVED_NO_OFFICIAL_MAPPING | CONFLICT | DONE | SKIPPED
|
||||
kb_product_id: Mapped[int | None] = mapped_column(ForeignKey("kb_products.id"), nullable=True)
|
||||
cells_covered: Mapped[int] = mapped_column(Integer, default=0)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class AiRun(Base):
|
||||
"""One provider execution over a batch of work items, with budget-guard evidence."""
|
||||
__tablename__ = "ai_runs"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
provider: Mapped[str] = mapped_column(String(64))
|
||||
model: Mapped[str] = mapped_column(String(64))
|
||||
prompt_version: Mapped[str] = mapped_column(String(32))
|
||||
work_item_ids: Mapped[list] = mapped_column(JSON, default=list)
|
||||
input_tokens: Mapped[int] = mapped_column(Integer, default=0)
|
||||
output_tokens: Mapped[int] = mapped_column(Integer, default=0)
|
||||
cost_usd: Mapped[float] = mapped_column(default=0.0)
|
||||
status: Mapped[str] = mapped_column(String(16), default="RUN")
|
||||
results: Mapped[list] = mapped_column(JSON, default=list)
|
||||
budget_guard: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class AiEvalCache(Base):
|
||||
"""Persistent LLM result cache: identical (input_fingerprint, candidate_set_hash,
|
||||
prompt_version, model, dataset_versions) re-execution makes ZERO new calls."""
|
||||
__tablename__ = "ai_eval_cache"
|
||||
__table_args__ = (UniqueConstraint("input_fingerprint", "candidate_set_hash", "prompt_version", "model", "dataset_version_ncm", name="uq_ai_eval_cache"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
input_fingerprint: Mapped[str] = mapped_column(String(64), index=True)
|
||||
candidate_set_hash: Mapped[str] = mapped_column(String(64))
|
||||
prompt_version: Mapped[str] = mapped_column(String(32))
|
||||
model: Mapped[str] = mapped_column(String(64))
|
||||
dataset_version_ncm: Mapped[str] = mapped_column(String(64))
|
||||
phase: Mapped[str] = mapped_column(String(16))
|
||||
answer: Mapped[dict] = mapped_column(JSON)
|
||||
gold_ncm: Mapped[str | None] = mapped_column(String(8), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Cost plan for the residual AI work items, with budget guard (section 18).
|
||||
|
||||
Runs BEFORE any real execution: UNIQUE_AI_WORK_ITEMS, token estimates per task type,
|
||||
estimated cost, and the authorize/refuse verdict against AI_MAX_RUN_COST_USD.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from sqlalchemy import select
|
||||
from central.app.ai.provider import BudgetGuard
|
||||
from central.app.config import settings
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.models import AiWorkItem
|
||||
|
||||
TOKENS_PER_ITEM = { # conservative prompt+schema overhead per single work item
|
||||
"INVALID_NCM": (700, 120),
|
||||
"CEST": (500, 80),
|
||||
"EX_TIPI": (500, 80),
|
||||
"PRODUCT_ROLE": (400, 60),
|
||||
}
|
||||
|
||||
|
||||
def main(output="docs/AI_COST_PLAN_V1.json") -> dict:
|
||||
session = SessionLocal()
|
||||
items = session.scalars(select(AiWorkItem)).all()
|
||||
guard = BudgetGuard(settings.ai_max_run_cost_usd, settings.ai_price_in_per_m, settings.ai_price_out_per_m)
|
||||
per_task = {}
|
||||
in_total = out_total = 0
|
||||
resolved = {"KB_HIT", "KB_EXACT_HIT", "KB_FINGERPRINT_HIT", "HIGH_CONFIDENCE_RETRIEVAL", "SINGLE_OFFICIAL_CANDIDATE", "DONE", "SKIPPED", "CONFLICT"}
|
||||
for item in items:
|
||||
if item.status in resolved:
|
||||
continue # cache/official candidate resolved — no AI call needed
|
||||
tokens_in, tokens_out = TOKENS_PER_ITEM.get(item.task_type, (500, 80))
|
||||
bucket = per_task.setdefault(item.task_type, {"items": 0, "input_tokens": 0, "output_tokens": 0, "cells_covered": 0})
|
||||
bucket["items"] += 1; bucket["input_tokens"] += tokens_in; bucket["output_tokens"] += tokens_out
|
||||
bucket["cells_covered"] += item.cells_covered
|
||||
in_total += tokens_in; out_total += tokens_out
|
||||
plan = guard.estimate(in_total, out_total)
|
||||
report = {
|
||||
"artifact": "AI_COST_PLAN_V1",
|
||||
"unique_ai_work_items": sum(bucket["items"] for bucket in per_task.values()),
|
||||
"per_task_type": per_task,
|
||||
"estimated_input_tokens": plan["input_tokens"],
|
||||
"estimated_output_tokens": plan["output_tokens"],
|
||||
"estimated_cost_usd": plan["estimated_cost"],
|
||||
"ai_max_run_cost_usd": guard.max_run_cost_usd,
|
||||
"budget_authorized": guard.authorize(plan),
|
||||
"provider_configured": settings.ai_provider,
|
||||
"model_configured": settings.ai_model,
|
||||
"ai_enabled": settings.ai_enabled,
|
||||
"max_batch_size": settings.ai_max_batch_size,
|
||||
"min_confidence": settings.ai_min_confidence,
|
||||
}
|
||||
Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
session.close()
|
||||
return report
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="/tmp/ai_cost_plan.json")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(main(args.output), indent=2))
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
"""GTIN KB audit (section 10) — validity, cross-tenant matches and conflicts.
|
||||
|
||||
Reads the seed payload (ground truth of what was observed per tenant) plus the KB tables.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from central.app.ai.fingerprint import validate_gtin
|
||||
|
||||
|
||||
def main(payload_path="docs/runs/ai_kb_seed_payload.json", output="docs/GTIN_KB_AUDIT_V1.json") -> dict:
|
||||
payload = json.loads(Path(payload_path).read_text(encoding="utf8"))
|
||||
by_gtin = defaultdict(list)
|
||||
invalid_gtin_raw = 0
|
||||
for product in payload["products"]:
|
||||
raw = product.get("gtin")
|
||||
if raw is None:
|
||||
continue
|
||||
if validate_gtin(raw) is None:
|
||||
invalid_gtin_raw += 1 # defensive: export already validates
|
||||
continue
|
||||
by_gtin[raw].append(product)
|
||||
duplicated_same_product = sum(1 for entries in by_gtin.values() if len(entries) > 1)
|
||||
conflicting = 0
|
||||
cross_tenant = 0
|
||||
conflict_examples = []
|
||||
for gtin, entries in by_gtin.items():
|
||||
ncms = {entry.get("ncm") for entry in entries if entry.get("ncm")}
|
||||
databases = {obs["database"] for entry in entries for obs in entry["observations"]}
|
||||
if len(ncms) > 1:
|
||||
conflicting += 1
|
||||
if len(conflict_examples) < 10:
|
||||
conflict_examples.append({"gtin": gtin, "ncms": sorted(ncms)})
|
||||
if len(databases) > 1:
|
||||
cross_tenant += 1
|
||||
report = {
|
||||
"artifact": "GTIN_KB_AUDIT_V1",
|
||||
"gtin_valid": len(by_gtin),
|
||||
"gtin_invalid_raw": invalid_gtin_raw,
|
||||
"gtin_duplicated_same_product": duplicated_same_product,
|
||||
"gtin_conflicting_products": conflicting,
|
||||
"gtin_cross_tenant_matches": cross_tenant,
|
||||
"conflict_examples": conflict_examples,
|
||||
"policy": "same GTIN across tenants is STRONG evidence for sharing global product knowledge; tenant fiscal context is never shared",
|
||||
}
|
||||
Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
return report
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--payload-path", default="/tmp/ai_kb_seed_payload.json")
|
||||
parser.add_argument("--output", default="/tmp/gtin_audit.json")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(main(args.payload_path, args.output), indent=2))
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Build deduplicated AI work items from the residual questions and the seeded KB.
|
||||
|
||||
Reads docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json, resolves each invalid-NCM product
|
||||
question to its KB fingerprint (same product in several databases collapses into ONE
|
||||
work item), persists AiWorkItem rows and writes docs/AI_RESIDUAL_PLAN_V1.json.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from central.app.ai.kb import KnowledgeBase
|
||||
from central.app.ai.workitems import build_work_items
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.models import AiWorkItem, KbObservation
|
||||
|
||||
|
||||
def _erp_code_from_question(question) -> str:
|
||||
import re
|
||||
match = re.search(r"PRODUCT\s+(\d+)", question.get("QUESTION", ""))
|
||||
return match.group(1) if match else ""
|
||||
|
||||
|
||||
def main(questions_path="docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json", output="docs/AI_RESIDUAL_PLAN_V1.json") -> dict:
|
||||
session = SessionLocal()
|
||||
questions = json.loads(Path(questions_path).read_text(encoding="utf8"))["questions"]
|
||||
# map "{DATABASE}:{erp_code}" -> fingerprint using the seeded observations
|
||||
rows = session.execute(select(KbObservation.company_key, KbObservation.erp_code, KbObservation.kb_product_id, KbObservation.database_label)).all()
|
||||
product_ids = {f"{row.database_label}:{row.erp_code}": row.kb_product_id for row in rows}
|
||||
questions = [dict(question, DATABASE_CODE=_erp_code_from_question(question)) for question in questions]
|
||||
product_fingerprints = {key: f"kb:{value}" for key, value in product_ids.items()}
|
||||
items = build_work_items(questions, product_fingerprints)
|
||||
raw = sum(len(item.question_refs) for item in items)
|
||||
persisted = {"created": 0, "updated": 0, "kb_hits": 0, "conflicts": 0}
|
||||
kb = KnowledgeBase(session)
|
||||
for item in items:
|
||||
row = session.scalar(select(AiWorkItem).where(AiWorkItem.dedup_key == item.dedup_key))
|
||||
if row is None:
|
||||
row = AiWorkItem(task_type=item.task_type, dedup_key=item.dedup_key, question_refs=item.question_refs, payload=item.payload, cells_covered=item.cells_covered, status="PENDING")
|
||||
session.add(row); persisted["created"] += 1
|
||||
else:
|
||||
row.question_refs = item.question_refs; row.cells_covered = item.cells_covered; row.payload = item.payload
|
||||
persisted["updated"] += 1
|
||||
# cache-first: an existing non-conflicting validated answer makes this a KB hit, no AI call
|
||||
if item.dedup_key.startswith("invalid_ncm:kb:") and item.dedup_key != "invalid_ncm:kb:None":
|
||||
kb_product_id = int(item.dedup_key.rsplit(":", 1)[1])
|
||||
knowledge = kb.current_knowledge(kb_product_id)
|
||||
validated = knowledge.get("validated_ncm", {})
|
||||
if validated and not validated.get("conflict"):
|
||||
row.status = "KB_HIT"; persisted["kb_hits"] += 1
|
||||
elif validated.get("conflict"):
|
||||
row.status = "CONFLICT"; persisted["conflicts"] += 1
|
||||
session.commit()
|
||||
by_task = {}
|
||||
for item in items:
|
||||
bucket = by_task.setdefault(item.task_type, {"items": 0, "cells": 0})
|
||||
bucket["items"] += 1; bucket["cells"] += item.cells_covered
|
||||
report = {
|
||||
"artifact": "AI_RESIDUAL_PLAN_V1",
|
||||
"raw_residual_questions": len(questions),
|
||||
"ai_work_items_raw": raw,
|
||||
"ai_work_items_deduped": len(items),
|
||||
"by_task_type": by_task,
|
||||
"persisted": persisted,
|
||||
"ai_write_allowed": False,
|
||||
"company_decisions_remaining": 14,
|
||||
}
|
||||
Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
session.close()
|
||||
return report
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--questions", default="docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json")
|
||||
parser.add_argument("--output", default="docs/AI_RESIDUAL_PLAN_V1.json")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(main(args.questions, args.output), indent=2))
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Reprocess the 750 AI work items with V2 retrieval (section 16).
|
||||
|
||||
Classification per item (INVALID_NCM): identity resolve (GTIN -> fingerprint) against the
|
||||
KB; validated non-conflicting knowledge resolves without AI. Otherwise hybrid retrieval:
|
||||
a HIGH_CONFIDENCE_RETRIEVAL requires hybrid score >= 0.75, an officially valid NCM and
|
||||
cross-tenant agreement >= 2 — it is NOT an automatic fiscal resolution (validator still
|
||||
required). Everything else with candidates -> LLM_SELECTION_REQUIRED; without candidates
|
||||
-> UNRESOLVED.
|
||||
|
||||
CEST/EX_TIPI items never call retrieval semantics beyond the official candidate lists:
|
||||
exactly one official candidate resolves deterministically; none is UNRESOLVED
|
||||
(NO_OFFICIAL_MAPPING); more than one requires (future) LLM selection.
|
||||
PRODUCT_ROLE stays LLM/manual by contract.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from central.app.ai.candidates import cest_candidates_for_ncm, ex_tipi_candidates_for_ncm
|
||||
from central.app.ai.kb import EXACT, FINGERPRINT, KnowledgeBase
|
||||
from central.app.ai.retrieval import (EnrichedNcmCorpus, KbNeighborIndex, derive_abbreviation_lexicon, hybrid_candidates)
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.models import AiWorkItem, Dataset, KbObservation, KbProduct, NcmEntry
|
||||
|
||||
HIGH_CONFIDENCE_THRESHOLD = 0.75
|
||||
|
||||
|
||||
def _resolve_product(session, dedup_key: str):
|
||||
"""invalid_ncm:kb:{id} -> KB product by id; invalid_ncm:{db}:{code} -> via observation."""
|
||||
if dedup_key.startswith("invalid_ncm:kb:"):
|
||||
return session.get(KbProduct, int(dedup_key.rsplit(":", 1)[1]))
|
||||
parts = dedup_key.split(":")
|
||||
if len(parts) == 3:
|
||||
observation = session.scalar(select(KbObservation).where(KbObservation.database_label == parts[1], KbObservation.erp_code == parts[2]))
|
||||
return session.get(KbProduct, observation.kb_product_id) if observation else None
|
||||
return None
|
||||
|
||||
|
||||
def main(output="docs/AI_RESIDUAL_PLAN_V2.json") -> dict:
|
||||
session = SessionLocal()
|
||||
kb = KnowledgeBase(session)
|
||||
corpus = EnrichedNcmCorpus.build(session)
|
||||
index = KbNeighborIndex.build(session)
|
||||
lexicon = derive_abbreviation_lexicon(index.corpus(), min_support=3)
|
||||
dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all()) if dataset else set()
|
||||
items = session.scalars(select(AiWorkItem)).all()
|
||||
counts = {key: 0 for key in ("KB_EXACT_HIT", "KB_FINGERPRINT_HIT", "HIGH_CONFIDENCE_RETRIEVAL", "SINGLE_OFFICIAL_CANDIDATE", "LLM_SELECTION_REQUIRED", "UNRESOLVED")}
|
||||
llm_cells = 0
|
||||
for item in items:
|
||||
if item.task_type == "INVALID_NCM":
|
||||
product = None
|
||||
product = _resolve_product(session, item.dedup_key)
|
||||
if product is not None and product.gtin:
|
||||
resolved, hit = kb.resolve(gtin=product.gtin)
|
||||
if resolved and hit == EXACT:
|
||||
knowledge = kb.current_knowledge(resolved.id).get("validated_ncm", {})
|
||||
if knowledge and not knowledge.get("conflict"):
|
||||
item.status = "KB_EXACT_HIT"; counts["KB_EXACT_HIT"] += 1; continue
|
||||
if product is not None:
|
||||
resolved, hit = kb.resolve(fingerprint=product.fingerprint)
|
||||
if resolved and hit == FINGERPRINT:
|
||||
knowledge = kb.current_knowledge(resolved.id).get("validated_ncm", {})
|
||||
if knowledge and not knowledge.get("conflict"):
|
||||
item.status = "KB_FINGERPRINT_HIT"; counts["KB_FINGERPRINT_HIT"] += 1; continue
|
||||
description = product.normalized_description if product is not None else (item.payload or {}).get("description", "")
|
||||
ranked = hybrid_candidates(corpus, index, description, k=10, lexicon=lexicon, exclude_product_id=(product.id if product else None))
|
||||
if ranked and ranked[0][2] >= HIGH_CONFIDENCE_THRESHOLD and ranked[0][0] in valid_ncm and ranked[0][5] >= 2:
|
||||
item.status = "HIGH_CONFIDENCE_RETRIEVAL"; counts["HIGH_CONFIDENCE_RETRIEVAL"] += 1
|
||||
item.payload = {**item.payload, "retrieval_top1": ranked[0][0], "retrieval_score": ranked[0][2], "tenant_agreement": ranked[0][5]}
|
||||
elif ranked:
|
||||
item.status = "LLM_SELECTION_REQUIRED"; counts["LLM_SELECTION_REQUIRED"] += 1; llm_cells += item.cells_covered
|
||||
else:
|
||||
item.status = "UNRESOLVED"; counts["UNRESOLVED"] += 1
|
||||
elif item.task_type in ("CEST", "EX_TIPI"):
|
||||
ncm = (item.payload or {}).get("ncm", "")
|
||||
candidates = cest_candidates_for_ncm(session, ncm) if item.task_type == "CEST" else ex_tipi_candidates_for_ncm(session, ncm)
|
||||
if len(candidates) == 1:
|
||||
item.status = "SINGLE_OFFICIAL_CANDIDATE"; counts["SINGLE_OFFICIAL_CANDIDATE"] += 1
|
||||
elif not candidates:
|
||||
item.status = "UNRESOLVED_NO_OFFICIAL_MAPPING"; counts["UNRESOLVED"] += 1
|
||||
else:
|
||||
item.status = "LLM_SELECTION_REQUIRED"; counts["LLM_SELECTION_REQUIRED"] += 1; llm_cells += item.cells_covered
|
||||
else: # PRODUCT_ROLE
|
||||
item.status = "LLM_SELECTION_REQUIRED"; counts["LLM_SELECTION_REQUIRED"] += 1; llm_cells += item.cells_covered
|
||||
session.commit()
|
||||
llm_items = counts["LLM_SELECTION_REQUIRED"]
|
||||
report = {
|
||||
"artifact": "AI_RESIDUAL_PLAN_V2",
|
||||
"work_items": len(items),
|
||||
"classification": counts,
|
||||
"llm_selection_required_items": llm_items,
|
||||
"llm_selection_required_cells": llm_cells,
|
||||
"high_confidence_threshold": HIGH_CONFIDENCE_THRESHOLD,
|
||||
"note": "HIGH_CONFIDENCE_RETRIEVAL is a candidate, not a fiscal resolution: the official validator + operator gate still apply. AI_WRITE_ALLOWED=false.",
|
||||
"dataset_versions": {"ncm": dataset.sha256} if dataset else {},
|
||||
}
|
||||
Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
session.close()
|
||||
return report
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="/tmp/plan_v2.json")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(main(args.output), indent=2))
|
||||
@@ -0,0 +1,100 @@
|
||||
"""NCM evaluation benchmark (section 19) — same pipeline as production, known answer hidden.
|
||||
|
||||
For a stratified sample of products whose NCM is valid in the ACTIVE dataset:
|
||||
candidate generation (ncm_top_k) -> provider selection -> validator.
|
||||
Metrics: EXACT_MATCH / WRONG_SELECTION / NONE_WHEN_KNOWN / INVALID_OUTPUT (+ recall@K),
|
||||
per segment (database). With no AI credential the BaselineProvider runs: it measures the
|
||||
deterministic floor of the same pipeline and costs nothing.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from central.app.ai.candidates import ncm_top_k
|
||||
from central.app.ai.provider import BaselineProvider, PROMPT_VERSION, SemanticOutputError, result_from_dict
|
||||
from central.app.ai.fingerprint import normalize_description
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.models import Dataset, KbObservation, KbProduct, NcmEntry
|
||||
|
||||
SEGMENTS = {"CONSTRUCAO": "construcao", "MMMODAS": "moda", "RINCAO": "restaurante", "UNIAO": "mercado", "TESTESOFICIAIS": "base_testes"}
|
||||
|
||||
|
||||
def _samples(session, per_segment=40, seed=42):
|
||||
dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all())
|
||||
rows = session.execute(select(KbProduct.normalized_description, KbObservation.database_label, KbObservation.observed_fields).join(KbObservation, KbObservation.kb_product_id == KbProduct.id)).all()
|
||||
by_segment = {}
|
||||
for description, database, fields in rows:
|
||||
ncm = (fields or {}).get("ncm")
|
||||
if ncm and ncm in valid_ncm and description:
|
||||
by_segment.setdefault(SEGMENTS.get(database, database), set()).add((description, ncm))
|
||||
rng = random.Random(seed)
|
||||
samples = []
|
||||
for segment, values in sorted(by_segment.items()):
|
||||
picked = sorted(values)[: per_segment] if len(values) <= per_segment else rng.sample(sorted(values), per_segment)
|
||||
samples.extend({"segment": segment, "description": description, "true_ncm": ncm} for description, ncm in picked)
|
||||
return samples, dataset.sha256
|
||||
|
||||
|
||||
def main(per_segment=40, k=10, output="docs/AI_EVALUATION_V1.json") -> dict:
|
||||
session = SessionLocal()
|
||||
samples, ncm_sha = _samples(session, per_segment)
|
||||
provider = BaselineProvider()
|
||||
metrics = {"EXACT_MATCH": 0, "WRONG_SELECTION": 0, "NONE_WHEN_KNOWN": 0, "INVALID_OUTPUT": 0, "RECALL_AT_K": 0}
|
||||
per_segment = {}
|
||||
for sample in samples:
|
||||
candidates = ncm_top_k(session, sample["description"], k=k)
|
||||
codes = [code for code, _, _ in candidates]
|
||||
bucket = per_segment.setdefault(sample["segment"], {key: 0 for key in metrics} | {"TOTAL": 0})
|
||||
bucket["TOTAL"] += 1
|
||||
if sample["true_ncm"] in codes:
|
||||
metrics["RECALL_AT_K"] += 1; bucket["RECALL_AT_K"] += 1
|
||||
item = type("W", (), {"task_type": "INVALID_NCM", "dedup_key": f"eval:{sample['segment']}:{normalize_description(sample['description'])[:60]}", "payload": {"description": sample["description"]}})()
|
||||
try:
|
||||
result = result_from_dict(provider.select(item, candidates, dataset_versions={"ncm": ncm_sha}).to_dict())
|
||||
except SemanticOutputError:
|
||||
metrics["INVALID_OUTPUT"] += 1; bucket["INVALID_OUTPUT"] += 1
|
||||
continue
|
||||
if result.selected_candidate == sample["true_ncm"]:
|
||||
metrics["EXACT_MATCH"] += 1; bucket["EXACT_MATCH"] += 1
|
||||
elif result.selected_candidate == "NONE":
|
||||
metrics["NONE_WHEN_KNOWN"] += 1; bucket["NONE_WHEN_KNOWN"] += 1
|
||||
else:
|
||||
metrics["WRONG_SELECTION"] += 1; bucket["WRONG_SELECTION"] += 1
|
||||
total = len(samples)
|
||||
report = {
|
||||
"artifact": "AI_EVALUATION_V1",
|
||||
"task": "INVALID_NCM candidate selection",
|
||||
"pipeline": "candidate_generation -> provider_selection -> validator",
|
||||
"provider": provider.name,
|
||||
"model": provider.model,
|
||||
"prompt_version": PROMPT_VERSION,
|
||||
"dataset_versions": {"ncm": ncm_sha},
|
||||
"k": k,
|
||||
"eval_total": total,
|
||||
"metrics": metrics,
|
||||
"accuracy_exact_match": round(metrics["EXACT_MATCH"] / total, 4) if total else None,
|
||||
"recall_at_k": round(metrics["RECALL_AT_K"] / total, 4) if total else None,
|
||||
"by_segment": per_segment,
|
||||
"cost_usd": 0.0,
|
||||
"note": "baseline provider (deterministic, zero-cost): measures the pipeline floor; rerun with a real provider once credentials exist",
|
||||
}
|
||||
Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
session.close()
|
||||
return report
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--per-segment", type=int, default=40)
|
||||
parser.add_argument("--k", type=int, default=10)
|
||||
parser.add_argument("--output", default="docs/AI_EVALUATION_V1.json")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(main(args.per_segment, args.k, args.output), indent=2))
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
"""NCM retrieval evaluation V2 (sections 1, 13).
|
||||
|
||||
Same sampling as V1 (seed 42, per-segment) with each sample now quality-classified:
|
||||
GOLD - GTIN product or cross-tenant agreement on the same fingerprint/NCM
|
||||
SILVER - single-tenant valid NCM, no independent confirmation
|
||||
NOISY - commercial description with < 2 informative tokens (not pursued)
|
||||
Retrieval measured separately: OFFICIAL_ONLY (hierarchical enriched corpus),
|
||||
KB_ONLY (neighbor bridge), HYBRID; recall@1/5/10/20/50 + MRR, per class and segment.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from central.app.ai.retrieval import (EnrichedNcmCorpus, KbNeighborIndex, commercial_tokens, derive_abbreviation_lexicon,
|
||||
hierarchical_candidates, hybrid_candidates, kb_neighbor_candidates)
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.models import Dataset, KbObservation, KbProduct, NcmEntry
|
||||
|
||||
SEGMENTS = {"CONSTRUCAO": "construcao", "MMMODAS": "moda", "RINCAO": "restaurante", "UNIAO": "mercado", "TESTESOFICIAIS": "base_testes"}
|
||||
KS = (1, 5, 10, 20, 50)
|
||||
|
||||
|
||||
def _samples(session, per_segment=40, seed=42):
|
||||
import random
|
||||
dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all())
|
||||
rows = session.execute(
|
||||
select(KbProduct, KbObservation).join(KbObservation, KbObservation.kb_product_id == KbProduct.id)
|
||||
).all()
|
||||
per_product_tenants = {}
|
||||
for product, observation in rows:
|
||||
per_product_tenants.setdefault(product.id, set()).add(observation.tenant_id)
|
||||
by_segment = {}
|
||||
for product, observation in rows:
|
||||
ncm = (observation.observed_fields or {}).get("ncm")
|
||||
if ncm and ncm in valid_ncm and product.normalized_description:
|
||||
by_segment.setdefault(SEGMENTS.get(observation.database_label, observation.database_label), set()).add(
|
||||
(product.id, product.normalized_description, ncm, bool(product.gtin), len(per_product_tenants[product.id]) >= 2)
|
||||
)
|
||||
rng = random.Random(seed)
|
||||
samples = []
|
||||
for segment, values in sorted(by_segment.items()):
|
||||
picked = sorted(values) if len(values) <= per_segment else rng.sample(sorted(values), per_segment)
|
||||
for product_id, description, ncm, has_gtin, cross_tenant in picked:
|
||||
quality = "NOISY" if len(commercial_tokens(description)) < 2 else ("GOLD" if (has_gtin or cross_tenant) else "SILVER")
|
||||
samples.append({"segment": segment, "product_id": product_id, "description": description, "true_ncm": ncm, "quality": quality})
|
||||
return samples
|
||||
|
||||
|
||||
def _recall_mrr(runs, samples):
|
||||
"""runs: {mode: [ranked_codes_per_sample]}"""
|
||||
out = {}
|
||||
for mode, ranked_lists in runs.items():
|
||||
metrics = {f"recall_at_{k}": 0 for k in KS} | {"mrr": 0.0}
|
||||
total = 0
|
||||
for sample, ranked in zip(samples, ranked_lists):
|
||||
codes = [row[0] for row in ranked]
|
||||
total += 1
|
||||
for k in KS:
|
||||
if sample["true_ncm"] in codes[:k]:
|
||||
metrics[f"recall_at_{k}"] += 1
|
||||
if sample["true_ncm"] in codes:
|
||||
metrics["mrr"] += 1.0 / (codes.index(sample["true_ncm"]) + 1)
|
||||
if total:
|
||||
for key in metrics:
|
||||
metrics[key] = round(metrics[key] / total, 4)
|
||||
out[mode] = {"total": total, **metrics}
|
||||
return out
|
||||
|
||||
|
||||
def main(per_segment=40, output="docs/NCM_RETRIEVAL_EVALUATION_V2.json") -> dict:
|
||||
session = SessionLocal()
|
||||
samples = _samples(session, per_segment)
|
||||
corpus = EnrichedNcmCorpus.build(session)
|
||||
index = KbNeighborIndex.build(session)
|
||||
lexicon = derive_abbreviation_lexicon(index.corpus(), min_support=3)
|
||||
runs = {"OFFICIAL_ONLY": [], "KB_ONLY": [], "HYBRID": []}
|
||||
for sample in samples:
|
||||
# leave-one-out: the queried product never retrieves itself from the KB
|
||||
runs["OFFICIAL_ONLY"].append(hierarchical_candidates(corpus, sample["description"], k=50))
|
||||
runs["KB_ONLY"].append(kb_neighbor_candidates(index, sample["description"], k=50, exclude_product_id=sample["product_id"]))
|
||||
runs["HYBRID"].append(hybrid_candidates(corpus, index, sample["description"], k=50, lexicon=lexicon, exclude_product_id=sample["product_id"]))
|
||||
overall = _recall_mrr(runs, samples)
|
||||
by_quality = {quality: _recall_mrr({mode: [row for row, sample in zip(list_rows, samples) if sample["quality"] == quality] for mode, list_rows in runs.items()},
|
||||
[sample for sample in samples if sample["quality"] == quality]) for quality in ("GOLD", "SILVER", "NOISY")}
|
||||
by_segment = {segment: _recall_mrr({mode: [row for row, sample in zip(list_rows, samples) if sample["segment"] == segment] for mode, list_rows in runs.items()},
|
||||
[sample for sample in samples if sample["segment"] == segment]) for segment in sorted({sample["segment"] for sample in samples})}
|
||||
report = {
|
||||
"artifact": "NCM_RETRIEVAL_EVALUATION_V2",
|
||||
"eval_total": len(samples),
|
||||
"quality_counts": {quality: sum(1 for sample in samples if sample["quality"] == quality) for quality in ("GOLD", "SILVER", "NOISY")},
|
||||
"lexicon_size": len(lexicon),
|
||||
"lexicon_sample": dict(sorted(lexicon.items())[:40]),
|
||||
"overall": overall,
|
||||
"by_quality": by_quality,
|
||||
"by_segment": by_segment,
|
||||
"llm_liberation_gate": {"gold_recall_at_20_min": 0.80, "gold_recall_at_50_preferred": 0.90},
|
||||
"cost_usd": 0.0,
|
||||
}
|
||||
Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
session.close()
|
||||
return report
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--per-segment", type=int, default=40)
|
||||
parser.add_argument("--output", default="docs/NCM_RETRIEVAL_EVALUATION_V2.json")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(main(args.per_segment, args.output), indent=2))
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Real LLM NCM evaluation over known-answer cases (mission REAL LLM EVALUATION V1).
|
||||
|
||||
Phases: gold first (65 cases, gold completely hidden from the prompt); silver and noisy
|
||||
run ONLY after the gold quality gate passes (selection accuracy >= 0.95 among valid
|
||||
answers AND high-confidence wrong rate <= 0.02). Batching verified progressively
|
||||
(5/10/20) with 1:1 id matching; results cached permanently per
|
||||
(input_fingerprint, candidate_set_hash, prompt_version, model, dataset_versions) so an
|
||||
identical re-execution makes ZERO new calls. AI_WRITE_ALLOWED=false; nothing here can
|
||||
produce a Firebird write.
|
||||
|
||||
If no AI credential exists the run refuses to call anything and exits with
|
||||
BLOCKED_EXTERNAL_AI_CREDENTIAL (no key is ever invented).
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from central.app.ai.eval_metrics import attribution, confidence_buckets, metrics
|
||||
from central.app.ai.llm_provider import (PROMPT_VERSION_LLM, LlmAnswerError, LlmEvalItem, OpenAICompatibleProvider,
|
||||
candidate_set_hash, input_fingerprint, validate_answer)
|
||||
from central.app.ai.retrieval import EnrichedNcmCorpus, KbNeighborIndex, derive_abbreviation_lexicon, hybrid_candidates
|
||||
from central.app.config import settings
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.models import AiEvalCache, Dataset, NcmEntry
|
||||
|
||||
GATE_SELECTION_ACCURACY = 0.95
|
||||
GATE_HIGH_CONF_WRONG = 0.02
|
||||
BATCH_PROBE_SIZES = (20, 10, 5)
|
||||
|
||||
|
||||
def _load_samples(session, per_segment=40):
|
||||
from central.scripts.run_ncm_retrieval_eval_v2 import _samples
|
||||
return _samples(session, per_segment)
|
||||
|
||||
|
||||
def _build_items(samples, corpus, index, lexicon, top_k=20):
|
||||
items = []
|
||||
meta = []
|
||||
for sample in samples:
|
||||
ranked = hybrid_candidates(corpus, index, sample["description"], k=top_k, lexicon=lexicon, exclude_product_id=sample["product_id"])
|
||||
item = LlmEvalItem(
|
||||
work_item_id=f"{sample['segment']}:{sample['product_id']}",
|
||||
description=sample["description"],
|
||||
normalized_description=sample["description"],
|
||||
brand=None, unit=None, segment=sample["segment"],
|
||||
candidates=ranked,
|
||||
)
|
||||
items.append(item)
|
||||
meta.append({**sample, "gold_in_candidates": sample["true_ncm"] in {code for code, *_ in ranked}})
|
||||
return items, meta
|
||||
|
||||
|
||||
def _cached_answers(session, items, model, ncm_sha, phase):
|
||||
hits = {}
|
||||
for item in items:
|
||||
row = session.scalar(select(AiEvalCache).where(
|
||||
AiEvalCache.input_fingerprint == input_fingerprint(item),
|
||||
AiEvalCache.candidate_set_hash == candidate_set_hash(item),
|
||||
AiEvalCache.prompt_version == PROMPT_VERSION_LLM,
|
||||
AiEvalCache.model == model,
|
||||
AiEvalCache.dataset_version_ncm == ncm_sha,
|
||||
AiEvalCache.phase == phase,
|
||||
))
|
||||
if row:
|
||||
hits[item.work_item_id] = row.answer
|
||||
return hits
|
||||
|
||||
|
||||
def _diagnose_http(status_code: int) -> str:
|
||||
return {401: "credential invalid (AI_API_KEY)", 403: "permission/plan does not allow this endpoint or model", 404: "endpoint or model not found (AI_BASE_URL/AI_MODEL)", 429: "quota or rate limit"}.get(status_code, f"http {status_code}")
|
||||
|
||||
|
||||
def _select_model(available: list, requested: str) -> tuple:
|
||||
"""Prefer the operator-requested model; else a current GLM suitable for classification.
|
||||
Never invents a name and never switches vendor — returns (model, source)."""
|
||||
if requested in available:
|
||||
return requested, "requested"
|
||||
glm = [model for model in available if model.lower().startswith("glm")]
|
||||
for candidate in glm:
|
||||
if "flash" in candidate.lower():
|
||||
return candidate, "fallback_glm_flash"
|
||||
if glm:
|
||||
return sorted(glm)[-1], "fallback_glm_latest"
|
||||
return None, "no_glm_available"
|
||||
|
||||
|
||||
def _preflight(provider: OpenAICompatibleProvider, requested_model: str) -> dict:
|
||||
"""Cheap connectivity check (GET /models — no completion tokens). Validates auth,
|
||||
base URL and model availability; diagnoses 401/403/404/429 autonomously."""
|
||||
import httpx
|
||||
try:
|
||||
available = provider.list_models()
|
||||
except httpx.HTTPStatusError as error:
|
||||
return {"zai_connection": "FAIL", "diagnosis": _diagnose_http(error.response.status_code), "model_available": False}
|
||||
except httpx.HTTPError as error:
|
||||
return {"zai_connection": "FAIL", "diagnosis": f"transport: {type(error).__name__}", "model_available": False}
|
||||
model, source = _select_model(available, requested_model)
|
||||
return {"zai_connection": "PASS", "model_available": model is not None, "model": model, "model_source": source,
|
||||
"available_models": available if model is None else None, "base_url": provider.base_url}
|
||||
|
||||
|
||||
def _probe_batch_size(provider, items):
|
||||
"""Try the largest batch that keeps valid JSON with 1:1 ids on a small probe slice."""
|
||||
for size in BATCH_PROBE_SIZES:
|
||||
probe = items[:size]
|
||||
try:
|
||||
provider.select_batch(probe)
|
||||
return size
|
||||
except (LlmAnswerError, json.JSONDecodeError):
|
||||
continue
|
||||
return 1
|
||||
|
||||
|
||||
def run_phase(provider_factory, phase, samples, corpus, index, lexicon, session, ncm_sha, valid_ncm):
|
||||
items, meta = _build_items(samples, corpus, index, lexicon)
|
||||
cached = _cached_answers(session, items, provider_factory.model if hasattr(provider_factory, "model") else settings.ai_model, ncm_sha, phase)
|
||||
todo = [item for item in items if item.work_item_id not in cached]
|
||||
answers = dict(cached)
|
||||
provider = None
|
||||
batch_size = 0
|
||||
if todo:
|
||||
provider = provider_factory()
|
||||
batch_size = _probe_batch_size(provider, todo)
|
||||
for start in range(0, len(todo), batch_size):
|
||||
for answer in provider.select_batch(todo[start:start + batch_size]):
|
||||
answers[answer["work_item_id"]] = answer
|
||||
rows, errors = [], []
|
||||
for item, sample in zip(items, meta):
|
||||
answer = answers.get(item.work_item_id)
|
||||
invalid = False
|
||||
if answer is None:
|
||||
invalid = True
|
||||
answer = {"selected_ncm": None, "confidence": 0.0, "abstain": True, "reason_short": "missing"}
|
||||
else:
|
||||
try:
|
||||
validate_answer(answer, item)
|
||||
if answer["selected_ncm"] is not None and answer["selected_ncm"] not in valid_ncm:
|
||||
invalid = True
|
||||
except LlmAnswerError:
|
||||
invalid = True
|
||||
selected = None if (invalid or answer["abstain"] or answer["selected_ncm"] is None) else answer["selected_ncm"]
|
||||
rows.append({"gold_in_candidates": sample["gold_in_candidates"], "selected": selected, "gold": sample["true_ncm"], "confidence": answer["confidence"], "invalid": invalid})
|
||||
if selected is not None and selected != sample["true_ncm"]:
|
||||
errors.append({"work_item_id": item.work_item_id, "gold": sample["true_ncm"], "selected": selected, "confidence": answer["confidence"], "gold_in_candidates": sample["gold_in_candidates"], "reason_short": answer.get("reason_short", "")})
|
||||
if invalid or answer.get("abstain") or selected is not None:
|
||||
if item.work_item_id not in cached:
|
||||
session.add(AiEvalCache(input_fingerprint=input_fingerprint(item), candidate_set_hash=candidate_set_hash(item), prompt_version=PROMPT_VERSION_LLM,
|
||||
model=provider.model if provider else settings.ai_model, dataset_version_ncm=ncm_sha, phase=phase,
|
||||
answer=answer, gold_ncm=sample["true_ncm"]))
|
||||
session.commit()
|
||||
return rows, errors, provider, batch_size, len(cached)
|
||||
|
||||
|
||||
def main(output_dir="/tmp", per_segment=40) -> dict:
|
||||
if not settings.ai_api_key:
|
||||
print(json.dumps({"status": "BLOCKED_EXTERNAL_AI_CREDENTIAL",
|
||||
"operator_action": "set AI_API_KEY (and optionally AI_BASE_URL/AI_MODEL) in the Hub environment (.env); never commit or log it"}, indent=2))
|
||||
sys.exit(2)
|
||||
session = SessionLocal()
|
||||
dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all())
|
||||
corpus = EnrichedNcmCorpus.build(session)
|
||||
index = KbNeighborIndex.build(session)
|
||||
lexicon = derive_abbreviation_lexicon(index.corpus(), min_support=3)
|
||||
samples = _load_samples(session, per_segment)
|
||||
# ---- preflight: cheap /models call, never spends completion tokens ----
|
||||
pre_provider = OpenAICompatibleProvider(api_key=settings.ai_api_key, model=settings.ai_model, base_url=settings.ai_base_url)
|
||||
preflight = _preflight(pre_provider, settings.ai_model)
|
||||
if preflight["zai_connection"] != "PASS" or not preflight["model_available"]:
|
||||
print(json.dumps({"status": "ZAI_PREFLIGHT_FAIL", "preflight": preflight}, indent=2))
|
||||
sys.exit(3)
|
||||
model_name = preflight["model"]
|
||||
factory = lambda: OpenAICompatibleProvider(api_key=settings.ai_api_key, model=model_name, base_url=settings.ai_base_url)
|
||||
gold = [sample for sample in samples if sample["quality"] == "GOLD"]
|
||||
provider = factory()
|
||||
rows, errors, provider_used, batch_size, cache_hits = run_phase(lambda: provider, "gold", gold, corpus, index, lexicon, session, dataset.sha256, valid_ncm)
|
||||
m = metrics(rows)
|
||||
prices_configured = settings.ai_price_in_per_m is not None and settings.ai_price_out_per_m is not None
|
||||
usage = provider.usage if provider is not None else {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
|
||||
cost_total = round(usage["input_tokens"] / 1e6 * settings.ai_price_in_per_m + usage["output_tokens"] / 1e6 * settings.ai_price_out_per_m, 6) if prices_configured else "UNKNOWN"
|
||||
report = {
|
||||
"artifact": "REAL_LLM_NCM_EVALUATION_V1", "phase": "gold",
|
||||
"provider": "z.ai", "base_url": settings.ai_base_url, "model": model_name, "prompt_version": PROMPT_VERSION_LLM,
|
||||
"preflight": preflight,
|
||||
"metrics": m, "attribution": attribution(rows), "confidence_buckets": confidence_buckets(rows),
|
||||
"batch_size_selected": batch_size,
|
||||
"gate": {"selection_accuracy_min": GATE_SELECTION_ACCURACY, "high_conf_wrong_max": GATE_HIGH_CONF_WRONG,
|
||||
"passed": (m["SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE"] or 0) >= GATE_SELECTION_ACCURACY and (m["HIGH_CONFIDENCE_WRONG_RATE"] or 0) <= GATE_HIGH_CONF_WRONG},
|
||||
"cache_hits_first_run": cache_hits, "ai_write_allowed": False,
|
||||
"usage": {**usage, "http_calls": provider.http_calls, "retries": provider.retries,
|
||||
"average_items_per_call": round(len(rows) / max(provider.http_calls, 1), 2),
|
||||
"tokens_per_item": round((usage["input_tokens"] + usage["output_tokens"]) / max(len(rows), 1), 1),
|
||||
"cost_total": cost_total, "cost_per_item": (round(cost_total / max(len(rows), 1), 6) if prices_configured else "UNKNOWN"),
|
||||
"prices_configured": prices_configured},
|
||||
}
|
||||
Path(f"{output_dir}/REAL_LLM_NCM_EVALUATION_V1.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
Path(f"{output_dir}/REAL_LLM_NCM_ERRORS_V1.json").write_text(json.dumps(errors, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
# cache proof: identical re-execution must make zero new calls
|
||||
provider2 = factory()
|
||||
rows2, _, _, _, cache_hits2 = run_phase(lambda: provider2, "gold", gold, corpus, index, lexicon, session, dataset.sha256, valid_ncm)
|
||||
second_calls = provider2.http_calls
|
||||
report["cache_proof"] = {"second_run_calls": second_calls, "second_run_cache_hits": cache_hits2,
|
||||
"cache_hit_reexecution": second_calls == 0, "identical_metrics": metrics(rows2)["CORRECT"] == m["CORRECT"]}
|
||||
Path(f"{output_dir}/REAL_LLM_NCM_EVALUATION_V1.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
Path(f"{output_dir}/REAL_LLM_COST_V1.json").write_text(json.dumps({"artifact": "REAL_LLM_COST_V1", "usage": report["usage"], "previous_estimate_usd": 0.44, "cache_proof": report["cache_proof"]}, indent=2), encoding="utf-8")
|
||||
session.close()
|
||||
return report
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output-dir", default="/tmp")
|
||||
parser.add_argument("--per-segment", type=int, default=40)
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(main(args.output_dir, args.per_segment), indent=2))
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Seed the Product Knowledge Base deterministically from the 5 Firebird databases.
|
||||
|
||||
Two phases (Firebird is only reachable from Windows; Postgres only inside the compose network):
|
||||
|
||||
export (Windows, READ-ONLY SELECT against Firebird):
|
||||
python scripts/seed_product_kb.py export --output docs/runs/ai_kb_seed_payload.json
|
||||
load (inside the fiscal-hub container, against Postgres):
|
||||
python -m central.scripts.seed_product_kb load --input docs/runs/ai_kb_seed_payload.json
|
||||
|
||||
Export computes, per physical product row: the stable global fingerprint
|
||||
(description|unit; GTIN validated by checksum), and emits observations. Load upserts the
|
||||
GLOBAL product, records per-tenant observations, seeds knowledge already proven by the
|
||||
gate (validated_ncm when the declared NCM is active-official; ERP-declared values stay
|
||||
ERP_OBSERVATION evidence), and links TENANT_PRODUCT_CONTEXT rows. Tenant context is
|
||||
never promoted to global knowledge.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
DATABASES = [
|
||||
("TESTESOFICIAIS", None), # path from FIREBIRD_DATABASE_PATH
|
||||
("CONSTRUCAO", "Banco/CONSTRUCAO.FDB"),
|
||||
("MMMODAS", "Banco/MMMODAS.FDB"),
|
||||
("RINCAO", "Banco/RINCAO.FDB"),
|
||||
("UNIAO", "Banco/UNIAO.FDB"),
|
||||
]
|
||||
|
||||
QUERY = "SELECT p.CODIGO, p.PRODUTO, p.UNIDADE, p.CODBARRA, p.GTIN, p.CLASSIFICACAO_FISCAL, p.CEST FROM C000025 p"
|
||||
|
||||
|
||||
def _canonical_ncm(value) -> str:
|
||||
digits = "".join(char for char in str(value or "") if char.isdigit())
|
||||
return digits if len(digits) == 8 else ""
|
||||
|
||||
|
||||
def export_payload(limit_per_db: int = 0, output: str = "docs/runs/ai_kb_seed_payload.json") -> dict:
|
||||
import fdb
|
||||
from dotenv import dotenv_values
|
||||
from central.app.ai.fingerprint import normalize_description, product_fingerprint, validate_gtin
|
||||
env = dotenv_values(".env")
|
||||
stats = {"databases": [], "rows_read": 0, "firebird_writes": 0}
|
||||
products = {}
|
||||
for label, relative in DATABASES:
|
||||
path = env["FIREBIRD_DATABASE_PATH"] if relative is None else str(Path(relative).resolve())
|
||||
connection = fdb.connect(database=path, user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"))
|
||||
try:
|
||||
cursor = connection.cursor(); cursor.execute(QUERY)
|
||||
columns = [column[0].strip().upper() for column in cursor.description]
|
||||
rows = cursor.fetchmany(limit_per_db) if limit_per_db else cursor.fetchall()
|
||||
finally:
|
||||
connection.close()
|
||||
# collapse the physical duplicate CODIGO rows into observations of the same product
|
||||
seen_codes = set()
|
||||
for row in rows:
|
||||
record = dict(zip(columns, row))
|
||||
code = str(record.get("CODIGO") or "").strip()
|
||||
dedupe = f"{label}:{code}"
|
||||
if dedupe in seen_codes: # duplicate physical key (e.g. MMMODAS 017421) -> one observation
|
||||
continue
|
||||
seen_codes.add(dedupe)
|
||||
unit = normalize_description(record.get("UNIDADE")) or None
|
||||
fingerprint = product_fingerprint(record.get("PRODUTO"), None, unit)
|
||||
gtin = validate_gtin(record.get("GTIN")) or validate_gtin(record.get("CODBARRA"))
|
||||
entry = products.setdefault(fingerprint, {
|
||||
"fingerprint": fingerprint,
|
||||
"gtin": gtin,
|
||||
"normalized_description": normalize_description(record.get("PRODUTO")),
|
||||
"unit": unit,
|
||||
"ncm": _canonical_ncm(record.get("CLASSIFICACAO_FISCAL")),
|
||||
"cest": str(record.get("CEST") or "").strip() or None,
|
||||
"observations": [],
|
||||
})
|
||||
if gtin and not entry["gtin"]:
|
||||
entry["gtin"] = gtin
|
||||
entry["observations"].append({"database": label, "company_key": label, "erp_code": code})
|
||||
stats["databases"].append({"database": label, "rows": len(rows)})
|
||||
stats["rows_read"] += len(rows)
|
||||
payload = {"artifact": "AI_KB_SEED_PAYLOAD", "exported_at": datetime.now(timezone.utc).isoformat(), "stats": stats, "products": list(products.values())}
|
||||
Path(output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(output).write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
stats["unique_products"] = len(products)
|
||||
stats["payload_path"] = output
|
||||
return stats
|
||||
|
||||
|
||||
def load_payload(input_path: str) -> dict:
|
||||
from sqlalchemy import select
|
||||
from central.app.ai.kb import KnowledgeBase
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.models import Dataset, NcmEntry, Tenant, TenantProductContext
|
||||
payload = json.loads(Path(input_path).read_text(encoding="utf8"))
|
||||
session = SessionLocal()
|
||||
kb = KnowledgeBase(session)
|
||||
dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
active_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all()) if dataset else set()
|
||||
tenant_ids = {}
|
||||
stats = {"products": 0, "observations": 0, "validated_ncm": 0, "erp_ncm_observations": 0, "erp_cest_observations": 0, "tenant_contexts": 0, "firebird_writes": 0}
|
||||
for product in payload["products"]:
|
||||
row = kb.upsert_product(fingerprint=product["fingerprint"], gtin=product.get("gtin"), normalized_description=product.get("normalized_description", ""), unit=product.get("unit"))
|
||||
stats["products"] += 1
|
||||
for observation in product["observations"]:
|
||||
label = observation["database"]
|
||||
if label not in tenant_ids:
|
||||
tenant = session.scalar(select(Tenant).where(Tenant.name == label))
|
||||
if tenant is None:
|
||||
tenant = Tenant(name=label); session.add(tenant); session.flush()
|
||||
tenant_ids[label] = tenant.id
|
||||
kb.record_observation(row.id, tenant_id=tenant_ids[label], company_key=label, erp_code=observation["erp_code"], database_label=label, observed_fields={"ncm": product.get("ncm") or None, "cest": product.get("cest")})
|
||||
stats["observations"] += 1
|
||||
existing = session.scalar(select(TenantProductContext).where(TenantProductContext.tenant_id == tenant_ids[label], TenantProductContext.company_key == label, TenantProductContext.erp_code == observation["erp_code"]))
|
||||
if existing is None:
|
||||
session.add(TenantProductContext(tenant_id=tenant_ids[label], company_key=label, erp_code=observation["erp_code"], kb_product_id=row.id))
|
||||
stats["tenant_contexts"] += 1
|
||||
ncm = product.get("ncm") or ""
|
||||
if ncm and ncm in active_ncm:
|
||||
kb.add_knowledge(row.id, "validated_ncm", ncm, source="OFFICIAL_DATASET", confidence=1.0, dataset_versions={"ncm": dataset.sha256})
|
||||
stats["validated_ncm"] += 1
|
||||
elif ncm:
|
||||
kb.add_knowledge(row.id, "candidate_ncm", ncm, source="ERP_OBSERVATION", confidence=0.3, evidence={"declared_in": [observation["database"] for observation in product["observations"]]})
|
||||
stats["erp_ncm_observations"] += 1
|
||||
if product.get("cest"):
|
||||
kb.add_knowledge(row.id, "candidate_cest", product["cest"], source="ERP_OBSERVATION", confidence=0.3, evidence={"declared_in": [observation["database"] for observation in product["observations"]]})
|
||||
stats["erp_cest_observations"] += 1
|
||||
if stats["products"] % 2000 == 0:
|
||||
session.commit()
|
||||
session.commit()
|
||||
stats.update(kb.stats())
|
||||
session.close()
|
||||
return stats
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers(dest="phase", required=True)
|
||||
export_parser = sub.add_parser("export")
|
||||
export_parser.add_argument("--limit", type=int, default=0)
|
||||
export_parser.add_argument("--output", default="docs/runs/ai_kb_seed_payload.json")
|
||||
load_parser = sub.add_parser("load")
|
||||
load_parser.add_argument("--input", default="docs/runs/ai_kb_seed_payload.json")
|
||||
load_parser.add_argument("--output", default="docs/runs/ai_kb_seed.json")
|
||||
args = parser.parse_args(argv)
|
||||
if args.phase == "export":
|
||||
result = export_payload(args.limit, args.output)
|
||||
else:
|
||||
result = load_payload(args.input)
|
||||
result["loaded_at"] = datetime.now(timezone.utc).isoformat()
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,171 @@
|
||||
"""AI semantic layer V1: fingerprint, KB, dedup, provider contract, budget guard, candidates."""
|
||||
import json
|
||||
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from central.app.models import Base, Dataset, NcmEntry, CestEntry, CestNcmEntry, TipiEntry
|
||||
|
||||
|
||||
def _engine():
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
return engine
|
||||
|
||||
|
||||
def _ncm_dataset(session, entries):
|
||||
dataset = Dataset(kind="NCM", provider="Siscomex", sha256="a" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=len(entries))
|
||||
session.add(dataset); session.flush()
|
||||
for code, description in entries:
|
||||
session.add(NcmEntry(dataset_id=dataset.id, code=code, description=description, valid_from="", valid_to=""))
|
||||
session.flush()
|
||||
return dataset
|
||||
|
||||
|
||||
# ---------------- fingerprint / identity ----------------
|
||||
|
||||
def test_fingerprint_is_stable_and_ignores_unstable_context():
|
||||
from central.app.ai.fingerprint import normalize_description, product_fingerprint, validate_gtin
|
||||
a = product_fingerprint("Coca-Cola 2LT", "COCA-COLA", "UN", "2")
|
||||
b = product_fingerprint("coca cola 2lt ", "coca-cola", "un", "2.0")
|
||||
assert a == b
|
||||
# fiscal/company context never participates in global identity
|
||||
c = product_fingerprint("Coca-Cola 2LT", "COCA-COLA", "UN", "2")
|
||||
assert a == c
|
||||
assert normalize_description("Água Mineral 500ml") == "AGUA MINERAL 500ML"
|
||||
assert validate_gtin("7891234567895") == "7891234567895" # valid GTIN-13
|
||||
assert validate_gtin("7891234567890") is None # bad checksum
|
||||
assert validate_gtin("123") is None
|
||||
|
||||
|
||||
# ---------------- knowledge base ----------------
|
||||
|
||||
def test_kb_exact_fingerprint_hit_and_conflict_without_majority():
|
||||
from central.app.ai.kb import KnowledgeBase, EXACT, FINGERPRINT
|
||||
engine = _engine()
|
||||
with Session(engine) as session:
|
||||
kb = KnowledgeBase(session)
|
||||
p1 = kb.upsert_product(fingerprint="f1", gtin="7891234567895", normalized_description="AGUA MINERAL 500ML")
|
||||
# same GTIN arrives with a different fingerprint -> same product, exact hit
|
||||
p2, hit = kb.resolve(gtin="7891234567895", fingerprint="other")
|
||||
assert p2.id == p1.id and hit == EXACT
|
||||
# same fingerprint, no GTIN -> fingerprint hit
|
||||
p3, hit = kb.resolve(gtin=None, fingerprint="f2")
|
||||
assert p3 is None and hit is None
|
||||
p4 = kb.upsert_product(fingerprint="f2", normalized_description="OUTRO")
|
||||
_, hit = kb.resolve(gtin=None, fingerprint="f2")
|
||||
assert hit == FINGERPRINT
|
||||
kb.add_knowledge(p1.id, "candidate_ncm", "22011000", source="OFFICIAL_DATASET", confidence=1.0)
|
||||
kb.add_knowledge(p1.id, "candidate_ncm", "22019000", source="OFFICIAL_DATASET", confidence=1.0)
|
||||
current = kb.current_knowledge(p1.id)
|
||||
assert current["candidate_ncm"]["conflict"] is True # no majority auto-pick
|
||||
assert current["candidate_ncm"]["values"] == {"22011000", "22019000"}
|
||||
stats = kb.stats()
|
||||
assert stats["products"] == 2 and stats["gtin_products"] == 1 and stats["conflicts"] == 1
|
||||
|
||||
|
||||
def test_kb_aggregates_observations_across_tenants():
|
||||
from central.app.ai.kb import KnowledgeBase
|
||||
engine = _engine()
|
||||
with Session(engine) as session:
|
||||
kb = KnowledgeBase(session)
|
||||
p = kb.upsert_product(fingerprint="f1", normalized_description="REFRIGERANTE 2L")
|
||||
kb.record_observation(p.id, tenant_id=1, company_key="MMMODAS", erp_code="000001", database_label="MMMODAS", observed_fields={"ncm": "22021000"})
|
||||
kb.record_observation(p.id, tenant_id=2, company_key="UNIAO", erp_code="000009", database_label="UNIAO", observed_fields={"ncm": "22021000"})
|
||||
agg = kb.observation_summary(p.id)
|
||||
assert agg["observation_count"] == 2 and agg["tenant_count"] == 2
|
||||
assert agg["agreement"]["ncm"] == {"22021000": 2} and agg["conflicts"] == {}
|
||||
|
||||
|
||||
# ---------------- work item dedup ----------------
|
||||
|
||||
def test_work_items_dedup_by_knowledge_needed():
|
||||
from central.app.ai.workitems import build_work_items
|
||||
questions = [
|
||||
{"QUESTION_ID": "Q1", "DATABASE": "MMMODAS", "COMPANY": "MMMODAS", "SCOPE": "NCM", "GROUP": "", "NCM": "33059000", "KEY": "ex_tipi", "CELLS": 10},
|
||||
{"QUESTION_ID": "Q2", "DATABASE": "UNIAO", "COMPANY": "UNIAO", "SCOPE": "NCM", "GROUP": "", "NCM": "33059000", "KEY": "ex_tipi", "CELLS": 5},
|
||||
{"QUESTION_ID": "Q3", "DATABASE": "RINCAO", "COMPANY": "RINCAO", "SCOPE": "GROUP", "GROUP": "000015", "KEY": "product_role", "CELLS": 40},
|
||||
{"QUESTION_ID": "Q4", "DATABASE": "CONSTRUCAO", "COMPANY": "CONSTRUCAO", "SCOPE": "PRODUCT", "GROUP": "", "NCM": "", "KEY": "ncm", "CELLS": 8, "QUESTION": "produto PRODUCT 000054", "DATABASE_CODE": "000054"},
|
||||
]
|
||||
items = build_work_items(questions, product_fingerprints={"CONSTRUCAO:000054": "fp-a"})
|
||||
# 4 raw questions -> 3 deduped work items (two ex_tipi questions for NCM 33059000 collapse)
|
||||
by_key = {i.dedup_key: i for i in items}
|
||||
assert len(items) == 3
|
||||
assert by_key["ex_tipi:33059000"].cells_covered == 15
|
||||
assert by_key["ex_tipi:33059000"].question_refs == ["Q1", "Q2"]
|
||||
assert by_key["product_role:RINCAO:000015"].cells_covered == 40
|
||||
assert by_key["invalid_ncm:fp-a"].cells_covered == 8
|
||||
# AI-ineligible keys never become work items
|
||||
assert all(i.task_type in ("INVALID_NCM", "CEST", "EX_TIPI", "PRODUCT_ROLE") for i in items)
|
||||
|
||||
|
||||
# ---------------- provider contract / structured output / budget ----------------
|
||||
|
||||
def test_structured_output_schema_and_baseline_provider():
|
||||
from central.app.ai.provider import BaselineProvider, result_from_dict, SemanticOutputError
|
||||
provider = BaselineProvider()
|
||||
item = type("W", (), {"task_type": "INVALID_NCM", "dedup_key": "invalid_ncm:fp-a", "payload": {"description": "AGUA MINERAL NATURAL 500ML"}})()
|
||||
candidates = [("22011000", "AGUAS MINERAIS NATURAIS", 0.75), ("22019000", "AGUAS ... OUTRAS", 0.25)]
|
||||
result = provider.select(item, candidates, dataset_versions={"ncm": "sha-ncm"})
|
||||
assert result.task_type == "INVALID_NCM" and result.selected_candidate == "22011000"
|
||||
assert result.candidate_ids_seen == ["22011000", "22019000"]
|
||||
assert result.model == "baseline-deterministic-v1" and result.prompt_version == "ai-semantic-v1"
|
||||
assert result.dataset_versions and result.engine_version
|
||||
as_dict = result.to_dict()
|
||||
assert result_from_dict(as_dict).selected_candidate == "22011000"
|
||||
# schema fail-closed: a provider answer outside the candidate list is invalid
|
||||
try:
|
||||
result_from_dict({**as_dict, "selected_candidate": "99999999"})
|
||||
raise AssertionError("must reject")
|
||||
except SemanticOutputError:
|
||||
pass
|
||||
# no candidates at all -> NONE, never invented
|
||||
none_result = provider.select(item, [])
|
||||
assert none_result.selected_candidate == "NONE"
|
||||
|
||||
|
||||
def test_budget_guard_blocks_run_over_limit():
|
||||
from central.app.ai.provider import BudgetGuard
|
||||
guard = BudgetGuard(max_run_cost_usd=1.0, price_in_per_m=1.0, price_out_per_m=2.0)
|
||||
plan = guard.estimate(items_in_tokens=600_000, items_out_tokens=300_000)
|
||||
assert plan["estimated_cost"] > 0
|
||||
assert guard.authorize(plan) is False # (600k*1 + 300k*2)/1M = 1.2 > 1.0
|
||||
assert guard.authorize(guard.estimate(100_000, 50_000)) is True
|
||||
|
||||
|
||||
# ---------------- deterministic candidates ----------------
|
||||
|
||||
def test_ncm_top_k_and_ex_tipi_candidates():
|
||||
from central.app.ai.candidates import ncm_top_k, ex_tipi_candidates_for_ncm, cest_candidates_for_ncm
|
||||
engine = _engine()
|
||||
with Session(engine) as session:
|
||||
_ncm_dataset(session, [("22011000", "AGUAS MINERAIS NATURAIS"), ("22019000", "AGUAS MINERAIS GASEIFICADAS OUTRAS"), ("08043000", "ABACAXIS FRESCOS")])
|
||||
top = ncm_top_k(session, "AGUA MINERAL NATURAL", k=2)
|
||||
assert [c[0] for c in top] == ["22011000", "22019000"]
|
||||
dataset = Dataset(kind="TIPI_IPI", provider="RFB", sha256="b" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=2)
|
||||
session.add(dataset); session.flush()
|
||||
session.add(TipiEntry(dataset_id=dataset.id, ncm="33059000", ex=None, description="outros", ipi_rate=None, ipi_rate_text="NT", source_row=1))
|
||||
session.add(TipiEntry(dataset_id=dataset.id, ncm="33059000", ex="01", description="preparacoes para alisar", ipi_rate=None, ipi_rate_text="NT", source_row=2))
|
||||
cest_dataset = Dataset(kind="CEST", provider="CONFAZ", sha256="c" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=1)
|
||||
session.add(cest_dataset); session.flush()
|
||||
cest = CestEntry(dataset_id=cest_dataset.id, code="03.999.99", description="x", annex="II")
|
||||
session.add(cest); session.flush()
|
||||
session.add(CestNcmEntry(cest_entry_id=cest.id, ncm_code="22011000"))
|
||||
session.commit()
|
||||
assert [e.ex for e in ex_tipi_candidates_for_ncm(session, "33059000")] == ["01"]
|
||||
assert [c.code for c in cest_candidates_for_ncm(session, "22011000")] == ["03.999.99"]
|
||||
|
||||
|
||||
def test_kb_persist_roundtrip_models():
|
||||
from central.app.models import KbProduct, KbKnowledge, KbObservation, TenantProductContext, AiWorkItem, AiRun
|
||||
engine = _engine()
|
||||
with Session(engine) as session:
|
||||
p = KbProduct(fingerprint="f", normalized_description="D")
|
||||
session.add(p); session.flush()
|
||||
session.add(KbKnowledge(kb_product_id=p.id, knowledge_key="candidate_ncm", value="22011000", source="OFFICIAL_DATASET", confidence=1.0))
|
||||
session.add(KbObservation(kb_product_id=p.id, tenant_id=1, company_key="C", erp_code="1", database_label="C", observed_fields={}))
|
||||
session.add(TenantProductContext(tenant_id=1, company_key="C", erp_code="1", kb_product_id=p.id))
|
||||
session.add(AiWorkItem(task_type="CEST", dedup_key="cest:1", payload={}, question_refs=[], cells_covered=1))
|
||||
session.add(AiRun(provider="baseline", model="m", prompt_version="v1", input_tokens=1, output_tokens=1, cost_usd=0.0, status="DONE", results=[]))
|
||||
session.commit()
|
||||
assert session.query(KbProduct).count() == 1 and session.query(AiWorkItem).count() == 1
|
||||
@@ -121,6 +121,18 @@ def test_cfop_entrada_needs_company_entry_scope_then_derives():
|
||||
assert run(p=product(CFOP_ENTRADA="1102"))["CFOP_ENTRADA"]["state"] == ALREADY
|
||||
|
||||
|
||||
def test_cfop_entrada_mixed_scope_is_operation_dependent_and_never_invents_a_cfop():
|
||||
mixed = [{"scope": "COMPANY", "scope_key": "", "key": "entry_scope", "value": "MIXED_OR_DYNAMIC"}]
|
||||
r = run(p=product(CFOP_ENTRADA=None), ctx=mixed)
|
||||
e = r["CFOP_ENTRADA"]
|
||||
assert e["state"] == MANUAL and e["proposed"] is None and e.get("input_key") is None
|
||||
assert e["reason"] == "CFOP_ENTRADA_OPERATION_DEPENDENT_MIXED_SCOPE"
|
||||
r2 = run(p=product(CFOP_ENTRADA="2102"), ctx=mixed)
|
||||
assert r2["CFOP_ENTRADA"]["state"] == ALREADY and r2["CFOP_ENTRADA"]["reason"] == "DECLARED_VALID_OPERATION_DEPENDENT_MIXED_SCOPE"
|
||||
from central.app.context_keys import validate_entry
|
||||
assert validate_entry({"scope": "COMPANY", "key": "entry_scope", "value": "MIXED_OR_DYNAMIC"})[0]["value"] == "MIXED_OR_DYNAMIC"
|
||||
|
||||
|
||||
def test_icms_code_fields_follow_the_regime():
|
||||
r1 = run()
|
||||
assert r1["CST"]["state"] == NA and r1["CST_ENTRADA"]["state"] == NA and r1["CSOSN"]["state"] == ALREADY
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Real LLM provider contract: strict answers, batching safety, cache, metrics attribution."""
|
||||
import json
|
||||
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from central.app.models import Base, AiEvalCache
|
||||
|
||||
|
||||
def _item(work_item_id="w1", candidates=None):
|
||||
from central.app.ai.llm_provider import LlmEvalItem
|
||||
return LlmEvalItem(work_item_id=work_item_id, description="AGUA MINERAL 500ML", normalized_description="AGUA MINERAL 500ML",
|
||||
brand=None, unit="ML", segment="mercado", candidates=candidates or [("22011000", "AGUAS MINERAIS", 0.9), ("22019000", "AGUAS OUTRAS", 0.4)])
|
||||
|
||||
|
||||
def _answer(work_item_id="w1", ncm="22011000", confidence=0.9, abstain=False):
|
||||
return {"work_item_id": work_item_id, "selected_ncm": ncm, "confidence": confidence, "abstain": abstain, "reason_short": "desc match"}
|
||||
|
||||
|
||||
def test_validate_answer_accepts_only_candidate_or_abstain():
|
||||
from central.app.ai.llm_provider import LlmAnswerError, validate_answer
|
||||
item = _item()
|
||||
assert validate_answer(_answer(), item)["selected_ncm"] == "22011000"
|
||||
abstain = validate_answer(_answer(ncm=None, abstain=True), item)
|
||||
assert abstain["abstain"] is True and abstain["selected_ncm"] is None
|
||||
for bad in (_answer(ncm="99999999"), _answer(work_item_id="other"), _answer(ncm=None), _answer(confidence=1.5), _answer(ncm="22011000", abstain=True)):
|
||||
try:
|
||||
validate_answer(bad, item)
|
||||
raise AssertionError("must reject")
|
||||
except LlmAnswerError:
|
||||
pass
|
||||
|
||||
|
||||
def test_provider_batch_requires_1_to_1_answers():
|
||||
from central.app.ai.llm_provider import LlmAnswerError, OpenAICompatibleProvider
|
||||
|
||||
def transport(payload):
|
||||
content = json.dumps([_answer(), _answer()]) # two answers for one item
|
||||
return {"choices": [{"message": {"content": content}}], "usage": {"prompt_tokens": 10, "completion_tokens": 5}}
|
||||
|
||||
provider = OpenAICompatibleProvider(api_key="test-key", model="m", transport=transport)
|
||||
try:
|
||||
provider.select_batch([_item()])
|
||||
raise AssertionError("must reject")
|
||||
except LlmAnswerError:
|
||||
pass
|
||||
assert provider.http_calls == 1 # contract violations are NOT retried
|
||||
|
||||
|
||||
def test_provider_retries_transport_failures_and_counts_usage():
|
||||
from central.app.ai.llm_provider import OpenAICompatibleProvider
|
||||
calls = {"n": 0}
|
||||
|
||||
def transport(payload):
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
raise ConnectionError("boom")
|
||||
content = json.dumps([_answer()])
|
||||
return {"choices": [{"message": {"content": content}}], "usage": {"prompt_tokens": 100, "completion_tokens": 20, "prompt_tokens_details": {"cached_tokens": 50}}}
|
||||
|
||||
provider = OpenAICompatibleProvider(api_key="k", model="m", transport=transport)
|
||||
answers = provider.select_batch([_item()])
|
||||
assert answers[0]["selected_ncm"] == "22011000"
|
||||
assert provider.http_calls == 2 and provider.retries == 1
|
||||
assert provider.usage == {"input_tokens": 100, "output_tokens": 20, "cached_tokens": 50}
|
||||
|
||||
|
||||
def test_prompt_hides_gold_and_carries_candidates():
|
||||
from central.app.ai.llm_provider import candidate_set_hash, input_fingerprint, item_prompt_payload
|
||||
item = _item()
|
||||
payload = item_prompt_payload(item)
|
||||
assert "22011000" in json.dumps(payload) # candidates present
|
||||
assert item.normalized_description in json.dumps(payload)
|
||||
assert len(candidate_set_hash(item)) == 64 and len(input_fingerprint(item)) == 64
|
||||
changed = _item(candidates=[("11111111", "X", 0.1)])
|
||||
assert candidate_set_hash(changed) != candidate_set_hash(item)
|
||||
|
||||
|
||||
def test_eval_cache_roundtrip_deduplicates():
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
row = AiEvalCache(input_fingerprint="f", candidate_set_hash="c", prompt_version="p", model="m", dataset_version_ncm="d", phase="gold", answer={"selected_ncm": "22011000"}, gold_ncm="22011000")
|
||||
session.add(row); session.commit()
|
||||
existing = session.scalar(select(AiEvalCache).where(AiEvalCache.input_fingerprint == "f", AiEvalCache.candidate_set_hash == "c", AiEvalCache.prompt_version == "p", AiEvalCache.model == "m", AiEvalCache.dataset_version_ncm == "d"))
|
||||
assert existing.answer["selected_ncm"] == "22011000"
|
||||
|
||||
|
||||
def test_metrics_and_error_attribution():
|
||||
from central.app.ai.eval_metrics import attribution, confidence_buckets, metrics
|
||||
rows = [
|
||||
{"gold_in_candidates": True, "selected": "A", "gold": "A", "confidence": 0.99, "invalid": False},
|
||||
{"gold_in_candidates": True, "selected": "B", "gold": "A", "confidence": 0.97, "invalid": False},
|
||||
{"gold_in_candidates": True, "selected": None, "gold": "A", "confidence": 0.5, "invalid": False},
|
||||
{"gold_in_candidates": False, "selected": "B", "gold": "A", "confidence": 0.9, "invalid": False},
|
||||
{"gold_in_candidates": True, "selected": "zzz", "gold": "A", "confidence": 0.9, "invalid": True},
|
||||
]
|
||||
m = metrics(rows)
|
||||
assert m["TOTAL"] == 5 and m["CORRECT"] == 1 and m["WRONG"] == 2 and m["ABSTAIN"] == 1 and m["INVALID"] == 1
|
||||
assert m["GOLD_IN_CANDIDATES"] == 4
|
||||
assert abs(m["SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE"] - 1 / 3) < 1e-4 # correct / (correct+wrong) among valid answers
|
||||
assert abs(m["ABSTENTION_RATE"] - 0.2) < 1e-9
|
||||
a = attribution(rows)
|
||||
assert a["GOLD_NOT_IN_CANDIDATES"] == 1 and a["GOLD_IN_CANDIDATES_BUT_WRONG"] == 1 and a["GOLD_IN_CANDIDATES_BUT_ABSTAIN"] == 1
|
||||
buckets = confidence_buckets(rows)
|
||||
assert buckets[">=0.99"] == {"total": 1, "correct": 1} and buckets["0.95-0.9899"] == {"total": 1, "correct": 0}
|
||||
|
||||
|
||||
def test_zai_model_selection_rules():
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location("eval_runner", "central/scripts/run_real_llm_ncm_eval.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
available = ["glm-5.3", "glm-5.3-flash", "glm-4.7", "other-vendor-x"]
|
||||
assert module._select_model(available, "glm-5.3") == ("glm-5.3", "requested")
|
||||
# requested missing -> current GLM flash fallback, never another vendor, never invented
|
||||
assert module._select_model(available, "glm-9.9") == ("glm-5.3-flash", "fallback_glm_flash")
|
||||
only_generic = ["glm-4", "glm-4.7"]
|
||||
assert module._select_model(only_generic, "glm-9.9") == ("glm-4.7", "fallback_glm_latest")
|
||||
assert module._select_model(["gpt-x", "claude-y"], "glm-5.3") == (None, "no_glm_available")
|
||||
assert module._diagnose_http(401) == "credential invalid (AI_API_KEY)"
|
||||
|
||||
|
||||
def test_budget_guard_unknown_cost_limited_run():
|
||||
from central.app.ai.provider import BudgetGuard
|
||||
guard = BudgetGuard(max_run_cost_usd=5.0, price_in_per_m=None, price_out_per_m=None)
|
||||
plan = guard.estimate(1000, 100)
|
||||
assert plan["estimated_cost"] is None and plan["cost_unknown"] is True
|
||||
assert guard.authorize(plan) is False # residual runs refused without prices
|
||||
assert guard.authorize(plan, allow_unknown_cost=True) is True # explicitly limited GOLD benchmark may opt in
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Semantic candidate retrieval V2: hierarchical official corpus + KB-neighbor bridge + hybrid ranking."""
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from central.app.models import Base, Dataset, KbKnowledge, KbObservation, KbProduct, NcmEntry, Tenant
|
||||
|
||||
|
||||
def _engine():
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
return engine
|
||||
|
||||
|
||||
def _seed(session):
|
||||
ncm = Dataset(kind="NCM", provider="Siscomex", sha256="n" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=7)
|
||||
session.add(ncm); session.flush()
|
||||
rows = [
|
||||
("22", "BEBIDAS, LIQUIDOS ALCOOLICOS E VINAGRES"),
|
||||
("2202", "AGUAS, INCLUIDAS AS AGUAS GASEIFICADAS E MINERAIS"),
|
||||
("2202.10", "AGUAS MINERAIS E GASEIFICADAS"),
|
||||
("22021000", "AGUAS MINERAIS NAO GASEIFICADAS"),
|
||||
("61", "VESTUARIO E SEUS ACESSORIOS, DE MALHA"),
|
||||
("6105", "CAMISAS DE MALHA, DE USO MASCULINO"),
|
||||
("61051000", "DE ALGODAO"),
|
||||
]
|
||||
for code, description in rows:
|
||||
session.add(NcmEntry(dataset_id=ncm.id, code=code.replace(".", ""), description=description, valid_from="", valid_to=""))
|
||||
session.flush()
|
||||
# two tenants already selling mineral water with a validated NCM
|
||||
tenant_a, tenant_b = Tenant(name="A"), Tenant(name="B")
|
||||
session.add_all([tenant_a, tenant_b]); session.flush()
|
||||
p1 = KbProduct(fingerprint="f1", normalized_description="AGUA MINERAL SEM GAS 500ML")
|
||||
p2 = KbProduct(fingerprint="f2", normalized_description="AGUA MINERAL GARRAFA 1L")
|
||||
p3 = KbProduct(fingerprint="f3", normalized_description="CAMISA MASCULINA MALHA ALGODAO")
|
||||
session.add_all([p1, p2, p3]); session.flush()
|
||||
for product, ncm_value, tenant in ((p1, "22021000", tenant_a), (p2, "22021000", tenant_b), (p3, "61051000", tenant_a)):
|
||||
session.add(KbKnowledge(kb_product_id=product.id, knowledge_key="validated_ncm", value=ncm_value, source="OFFICIAL_DATASET", confidence=1.0))
|
||||
session.add(KbObservation(kb_product_id=p1.id, tenant_id=tenant_a.id, company_key="A", erp_code="1", database_label="A", observed_fields={}))
|
||||
session.add(KbObservation(kb_product_id=p2.id, tenant_id=tenant_b.id, company_key="B", erp_code="2", database_label="B", observed_fields={}))
|
||||
session.add(KbObservation(kb_product_id=p3.id, tenant_id=tenant_a.id, company_key="A", erp_code="3", database_label="A", observed_fields={}))
|
||||
session.commit()
|
||||
return ncm.sha256
|
||||
|
||||
|
||||
def test_enriched_corpus_carries_ancestor_context():
|
||||
from central.app.ai.retrieval import EnrichedNcmCorpus
|
||||
engine = _engine()
|
||||
with Session(engine) as session:
|
||||
_seed(session)
|
||||
corpus = EnrichedNcmCorpus.build(session)
|
||||
doc = corpus.document("22021000")
|
||||
# chapter + position + subposition context enriches the very short legal text (stemmed)
|
||||
assert "BEBIDA" in doc and "AGUA" in doc
|
||||
assert "VESTUARIO" not in doc
|
||||
|
||||
|
||||
def test_hierarchical_retrieval_prefers_right_positions():
|
||||
from central.app.ai.retrieval import EnrichedNcmCorpus, hierarchical_candidates
|
||||
engine = _engine()
|
||||
with Session(engine) as session:
|
||||
_seed(session)
|
||||
corpus = EnrichedNcmCorpus.build(session)
|
||||
codes = [code for code, _, _ in hierarchical_candidates(corpus, "AGUA MINERAL", k=2)]
|
||||
assert codes[0] == "22021000"
|
||||
|
||||
|
||||
def test_kb_neighbor_bridge_maps_commercial_to_ncm():
|
||||
from central.app.ai.retrieval import KbNeighborIndex, kb_neighbor_candidates
|
||||
engine = _engine()
|
||||
with Session(engine) as session:
|
||||
_seed(session)
|
||||
index = KbNeighborIndex.build(session)
|
||||
neighbors = kb_neighbor_candidates(index, "AGUA MINERAL PET 500ML", k=5)
|
||||
assert neighbors and neighbors[0][0] == "22021000"
|
||||
# cross-tenant agreement is reported, never auto-majority
|
||||
assert neighbors[0][3] >= 2
|
||||
|
||||
|
||||
def test_hybrid_merges_official_and_kb_rankings():
|
||||
from central.app.ai.retrieval import EnrichedNcmCorpus, KbNeighborIndex, hybrid_candidates
|
||||
engine = _engine()
|
||||
with Session(engine) as session:
|
||||
_seed(session)
|
||||
corpus = EnrichedNcmCorpus.build(session)
|
||||
index = KbNeighborIndex.build(session)
|
||||
hybrid = hybrid_candidates(corpus, index, "CAMISA MASCULINA ALGODAO", k=2)
|
||||
assert hybrid[0][0] == "61051000"
|
||||
scores = hybrid_candidates(corpus, index, "AGUA MINERAL SEM GAS", k=2)
|
||||
assert scores[0][0] == "22021000"
|
||||
|
||||
|
||||
def test_lexicon_expands_abbreviations_supported_by_kb():
|
||||
from central.app.ai.retrieval import derive_abbreviation_lexicon, expand_stems
|
||||
corpus = {
|
||||
"REFRIGERANTE COCA 2L": "22021000",
|
||||
"REFRI GUARANA 1L": "22021000",
|
||||
"REFRI LARANJA 350ML": "22021000",
|
||||
"CAMISA POLO ALGODAO": "61051000",
|
||||
"CAM POLO MASC": "61051000",
|
||||
"CAM POLO AZUL": "61051000",
|
||||
"BOTA COURO MASC": "64039900",
|
||||
}
|
||||
lexicon = derive_abbreviation_lexicon(corpus, min_support=2)
|
||||
assert lexicon.get("REFRI") == "REFRIGERANTE"
|
||||
assert lexicon.get("CAM") == "CAMISA"
|
||||
expanded = expand_stems({"REFRI", "COLA"}, lexicon)
|
||||
assert "REFRIGERANTE" in expanded
|
||||
|
||||
|
||||
def test_commercial_tokens_strip_units_and_sizes():
|
||||
from central.app.ai.retrieval import commercial_tokens
|
||||
tokens = commercial_tokens("CREME TRATAMENTO NOVEX 1KG CX 12 COR AZUL TAM M REF 453")
|
||||
assert "NOVEX" in tokens and "CREME" in tokens and "TRATAMENTO" in tokens
|
||||
assert "1KG" not in tokens and "CX" not in tokens and "COR" not in tokens and "TAM" not in tokens
|
||||
assert "AZUL" not in tokens and "REF" not in tokens and "453" not in tokens
|
||||
assert "12" not in tokens
|
||||
Reference in New Issue
Block a user