101 lines
5.0 KiB
Python
101 lines
5.0 KiB
Python
"""NCM evaluation benchmark (section 19) — same pipeline as production, known answer hidden.
|
|
|
|
For a stratified sample of products whose NCM is valid in the ACTIVE dataset:
|
|
candidate generation (ncm_top_k) -> provider selection -> validator.
|
|
Metrics: EXACT_MATCH / WRONG_SELECTION / NONE_WHEN_KNOWN / INVALID_OUTPUT (+ recall@K),
|
|
per segment (database). With no AI credential the BaselineProvider runs: it measures the
|
|
deterministic floor of the same pipeline and costs nothing.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import random
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
|
|
from sqlalchemy import select
|
|
|
|
from central.app.ai.candidates import ncm_top_k
|
|
from central.app.ai.provider import BaselineProvider, PROMPT_VERSION, SemanticOutputError, result_from_dict
|
|
from central.app.ai.fingerprint import normalize_description
|
|
from central.app.db import SessionLocal
|
|
from central.app.models import Dataset, KbObservation, KbProduct, NcmEntry
|
|
|
|
SEGMENTS = {"CONSTRUCAO": "construcao", "MMMODAS": "moda", "RINCAO": "restaurante", "UNIAO": "mercado", "TESTESOFICIAIS": "base_testes"}
|
|
|
|
|
|
def _samples(session, per_segment=40, seed=42):
|
|
dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
|
valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all())
|
|
rows = session.execute(select(KbProduct.normalized_description, KbObservation.database_label, KbObservation.observed_fields).join(KbObservation, KbObservation.kb_product_id == KbProduct.id)).all()
|
|
by_segment = {}
|
|
for description, database, fields in rows:
|
|
ncm = (fields or {}).get("ncm")
|
|
if ncm and ncm in valid_ncm and description:
|
|
by_segment.setdefault(SEGMENTS.get(database, database), set()).add((description, ncm))
|
|
rng = random.Random(seed)
|
|
samples = []
|
|
for segment, values in sorted(by_segment.items()):
|
|
picked = sorted(values)[: per_segment] if len(values) <= per_segment else rng.sample(sorted(values), per_segment)
|
|
samples.extend({"segment": segment, "description": description, "true_ncm": ncm} for description, ncm in picked)
|
|
return samples, dataset.sha256
|
|
|
|
|
|
def main(per_segment=40, k=10, output="docs/AI_EVALUATION_V1.json") -> dict:
|
|
session = SessionLocal()
|
|
samples, ncm_sha = _samples(session, per_segment)
|
|
provider = BaselineProvider()
|
|
metrics = {"EXACT_MATCH": 0, "WRONG_SELECTION": 0, "NONE_WHEN_KNOWN": 0, "INVALID_OUTPUT": 0, "RECALL_AT_K": 0}
|
|
per_segment = {}
|
|
for sample in samples:
|
|
candidates = ncm_top_k(session, sample["description"], k=k)
|
|
codes = [code for code, _, _ in candidates]
|
|
bucket = per_segment.setdefault(sample["segment"], {key: 0 for key in metrics} | {"TOTAL": 0})
|
|
bucket["TOTAL"] += 1
|
|
if sample["true_ncm"] in codes:
|
|
metrics["RECALL_AT_K"] += 1; bucket["RECALL_AT_K"] += 1
|
|
item = type("W", (), {"task_type": "INVALID_NCM", "dedup_key": f"eval:{sample['segment']}:{normalize_description(sample['description'])[:60]}", "payload": {"description": sample["description"]}})()
|
|
try:
|
|
result = result_from_dict(provider.select(item, candidates, dataset_versions={"ncm": ncm_sha}).to_dict())
|
|
except SemanticOutputError:
|
|
metrics["INVALID_OUTPUT"] += 1; bucket["INVALID_OUTPUT"] += 1
|
|
continue
|
|
if result.selected_candidate == sample["true_ncm"]:
|
|
metrics["EXACT_MATCH"] += 1; bucket["EXACT_MATCH"] += 1
|
|
elif result.selected_candidate == "NONE":
|
|
metrics["NONE_WHEN_KNOWN"] += 1; bucket["NONE_WHEN_KNOWN"] += 1
|
|
else:
|
|
metrics["WRONG_SELECTION"] += 1; bucket["WRONG_SELECTION"] += 1
|
|
total = len(samples)
|
|
report = {
|
|
"artifact": "AI_EVALUATION_V1",
|
|
"task": "INVALID_NCM candidate selection",
|
|
"pipeline": "candidate_generation -> provider_selection -> validator",
|
|
"provider": provider.name,
|
|
"model": provider.model,
|
|
"prompt_version": PROMPT_VERSION,
|
|
"dataset_versions": {"ncm": ncm_sha},
|
|
"k": k,
|
|
"eval_total": total,
|
|
"metrics": metrics,
|
|
"accuracy_exact_match": round(metrics["EXACT_MATCH"] / total, 4) if total else None,
|
|
"recall_at_k": round(metrics["RECALL_AT_K"] / total, 4) if total else None,
|
|
"by_segment": per_segment,
|
|
"cost_usd": 0.0,
|
|
"note": "baseline provider (deterministic, zero-cost): measures the pipeline floor; rerun with a real provider once credentials exist",
|
|
}
|
|
Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
session.close()
|
|
return report
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--per-segment", type=int, default=40)
|
|
parser.add_argument("--k", type=int, default=10)
|
|
parser.add_argument("--output", default="docs/AI_EVALUATION_V1.json")
|
|
args = parser.parse_args()
|
|
print(json.dumps(main(args.per_segment, args.k, args.output), indent=2))
|
|
|