Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'

git-subtree-dir: fiscal git-subtree-mainline: d95165fc8f git-subtree-split: 102ba84500
2026-09-29 14:43:23 -03:00
parent d95165fc8f 102ba84500
commit d21a9d1140
868 changed files with 566394 additions and 0 deletions
@@ -0,0 +1,100 @@
+"""NCM evaluation benchmark (section 19) — same pipeline as production, known answer hidden.
+
+For a stratified sample of products whose NCM is valid in the ACTIVE dataset:
+  candidate generation (ncm_top_k)  ->  provider selection  ->  validator.
+Metrics: EXACT_MATCH / WRONG_SELECTION / NONE_WHEN_KNOWN / INVALID_OUTPUT (+ recall@K),
+per segment (database). With no AI credential the BaselineProvider runs: it measures the
+deterministic floor of the same pipeline and costs nothing.
+"""
+import argparse
+import json
+import random
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select
+
+from central.app.ai.candidates import ncm_top_k
+from central.app.ai.provider import BaselineProvider, PROMPT_VERSION, SemanticOutputError, result_from_dict
+from central.app.ai.fingerprint import normalize_description
+from central.app.db import SessionLocal
+from central.app.models import Dataset, KbObservation, KbProduct, NcmEntry
+
+SEGMENTS = {"CONSTRUCAO": "construcao", "MMMODAS": "moda", "RINCAO": "restaurante", "UNIAO": "mercado", "TESTESOFICIAIS": "base_testes"}
+
+
+def _samples(session, per_segment=40, seed=42):
+    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+    valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all())
+    rows = session.execute(select(KbProduct.normalized_description, KbObservation.database_label, KbObservation.observed_fields).join(KbObservation, KbObservation.kb_product_id == KbProduct.id)).all()
+    by_segment = {}
+    for description, database, fields in rows:
+        ncm = (fields or {}).get("ncm")
+        if ncm and ncm in valid_ncm and description:
+            by_segment.setdefault(SEGMENTS.get(database, database), set()).add((description, ncm))
+    rng = random.Random(seed)
+    samples = []
+    for segment, values in sorted(by_segment.items()):
+        picked = sorted(values)[: per_segment] if len(values) <= per_segment else rng.sample(sorted(values), per_segment)
+        samples.extend({"segment": segment, "description": description, "true_ncm": ncm} for description, ncm in picked)
+    return samples, dataset.sha256
+
+
+def main(per_segment=40, k=10, output="docs/AI_EVALUATION_V1.json") -> dict:
+    session = SessionLocal()
+    samples, ncm_sha = _samples(session, per_segment)
+    provider = BaselineProvider()
+    metrics = {"EXACT_MATCH": 0, "WRONG_SELECTION": 0, "NONE_WHEN_KNOWN": 0, "INVALID_OUTPUT": 0, "RECALL_AT_K": 0}
+    per_segment = {}
+    for sample in samples:
+        candidates = ncm_top_k(session, sample["description"], k=k)
+        codes = [code for code, _, _ in candidates]
+        bucket = per_segment.setdefault(sample["segment"], {key: 0 for key in metrics} | {"TOTAL": 0})
+        bucket["TOTAL"] += 1
+        if sample["true_ncm"] in codes:
+            metrics["RECALL_AT_K"] += 1; bucket["RECALL_AT_K"] += 1
+        item = type("W", (), {"task_type": "INVALID_NCM", "dedup_key": f"eval:{sample['segment']}:{normalize_description(sample['description'])[:60]}", "payload": {"description": sample["description"]}})()
+        try:
+            result = result_from_dict(provider.select(item, candidates, dataset_versions={"ncm": ncm_sha}).to_dict())
+        except SemanticOutputError:
+            metrics["INVALID_OUTPUT"] += 1; bucket["INVALID_OUTPUT"] += 1
+            continue
+        if result.selected_candidate == sample["true_ncm"]:
+            metrics["EXACT_MATCH"] += 1; bucket["EXACT_MATCH"] += 1
+        elif result.selected_candidate == "NONE":
+            metrics["NONE_WHEN_KNOWN"] += 1; bucket["NONE_WHEN_KNOWN"] += 1
+        else:
+            metrics["WRONG_SELECTION"] += 1; bucket["WRONG_SELECTION"] += 1
+    total = len(samples)
+    report = {
+        "artifact": "AI_EVALUATION_V1",
+        "task": "INVALID_NCM candidate selection",
+        "pipeline": "candidate_generation -> provider_selection -> validator",
+        "provider": provider.name,
+        "model": provider.model,
+        "prompt_version": PROMPT_VERSION,
+        "dataset_versions": {"ncm": ncm_sha},
+        "k": k,
+        "eval_total": total,
+        "metrics": metrics,
+        "accuracy_exact_match": round(metrics["EXACT_MATCH"] / total, 4) if total else None,
+        "recall_at_k": round(metrics["RECALL_AT_K"] / total, 4) if total else None,
+        "by_segment": per_segment,
+        "cost_usd": 0.0,
+        "note": "baseline provider (deterministic, zero-cost): measures the pipeline floor; rerun with a real provider once credentials exist",
+    }
+    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    session.close()
+    return report
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--per-segment", type=int, default=40)
+    parser.add_argument("--k", type=int, default=10)
+    parser.add_argument("--output", default="docs/AI_EVALUATION_V1.json")
+    args = parser.parse_args()
+    print(json.dumps(main(args.per_segment, args.k, args.output), indent=2))
+