"""NCM evaluation benchmark (section 19) — same pipeline as production, known answer hidden.

For a stratified sample of products whose NCM is valid in the ACTIVE dataset:
  candidate generation (ncm_top_k)  ->  provider selection  ->  validator.
Metrics: EXACT_MATCH / WRONG_SELECTION / NONE_WHEN_KNOWN / INVALID_OUTPUT (+ recall@K),
per segment (database). With no AI credential the BaselineProvider runs: it measures the
deterministic floor of the same pipeline and costs nothing.
"""
import argparse
import json
import random
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))

from sqlalchemy import select

from central.app.ai.candidates import ncm_top_k
from central.app.ai.provider import BaselineProvider, PROMPT_VERSION, SemanticOutputError, result_from_dict
from central.app.ai.fingerprint import normalize_description
from central.app.db import SessionLocal
from central.app.models import Dataset, KbObservation, KbProduct, NcmEntry

SEGMENTS = {"CONSTRUCAO": "construcao", "MMMODAS": "moda", "RINCAO": "restaurante", "UNIAO": "mercado", "TESTESOFICIAIS": "base_testes"}


def _samples(session, per_segment=40, seed=42):
    dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
    valid_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all())
    rows = session.execute(select(KbProduct.normalized_description, KbObservation.database_label, KbObservation.observed_fields).join(KbObservation, KbObservation.kb_product_id == KbProduct.id)).all()
    by_segment = {}
    for description, database, fields in rows:
        ncm = (fields or {}).get("ncm")
        if ncm and ncm in valid_ncm and description:
            by_segment.setdefault(SEGMENTS.get(database, database), set()).add((description, ncm))
    rng = random.Random(seed)
    samples = []
    for segment, values in sorted(by_segment.items()):
        picked = sorted(values)[: per_segment] if len(values) <= per_segment else rng.sample(sorted(values), per_segment)
        samples.extend({"segment": segment, "description": description, "true_ncm": ncm} for description, ncm in picked)
    return samples, dataset.sha256


def main(per_segment=40, k=10, output="docs/AI_EVALUATION_V1.json") -> dict:
    session = SessionLocal()
    samples, ncm_sha = _samples(session, per_segment)
    provider = BaselineProvider()
    metrics = {"EXACT_MATCH": 0, "WRONG_SELECTION": 0, "NONE_WHEN_KNOWN": 0, "INVALID_OUTPUT": 0, "RECALL_AT_K": 0}
    per_segment = {}
    for sample in samples:
        candidates = ncm_top_k(session, sample["description"], k=k)
        codes = [code for code, _, _ in candidates]
        bucket = per_segment.setdefault(sample["segment"], {key: 0 for key in metrics} | {"TOTAL": 0})
        bucket["TOTAL"] += 1
        if sample["true_ncm"] in codes:
            metrics["RECALL_AT_K"] += 1; bucket["RECALL_AT_K"] += 1
        item = type("W", (), {"task_type": "INVALID_NCM", "dedup_key": f"eval:{sample['segment']}:{normalize_description(sample['description'])[:60]}", "payload": {"description": sample["description"]}})()
        try:
            result = result_from_dict(provider.select(item, candidates, dataset_versions={"ncm": ncm_sha}).to_dict())
        except SemanticOutputError:
            metrics["INVALID_OUTPUT"] += 1; bucket["INVALID_OUTPUT"] += 1
            continue
        if result.selected_candidate == sample["true_ncm"]:
            metrics["EXACT_MATCH"] += 1; bucket["EXACT_MATCH"] += 1
        elif result.selected_candidate == "NONE":
            metrics["NONE_WHEN_KNOWN"] += 1; bucket["NONE_WHEN_KNOWN"] += 1
        else:
            metrics["WRONG_SELECTION"] += 1; bucket["WRONG_SELECTION"] += 1
    total = len(samples)
    report = {
        "artifact": "AI_EVALUATION_V1",
        "task": "INVALID_NCM candidate selection",
        "pipeline": "candidate_generation -> provider_selection -> validator",
        "provider": provider.name,
        "model": provider.model,
        "prompt_version": PROMPT_VERSION,
        "dataset_versions": {"ncm": ncm_sha},
        "k": k,
        "eval_total": total,
        "metrics": metrics,
        "accuracy_exact_match": round(metrics["EXACT_MATCH"] / total, 4) if total else None,
        "recall_at_k": round(metrics["RECALL_AT_K"] / total, 4) if total else None,
        "by_segment": per_segment,
        "cost_usd": 0.0,
        "note": "baseline provider (deterministic, zero-cost): measures the pipeline floor; rerun with a real provider once credentials exist",
    }
    Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
    session.close()
    return report


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--per-segment", type=int, default=40)
    parser.add_argument("--k", type=int, default=10)
    parser.add_argument("--output", default="docs/AI_EVALUATION_V1.json")
    args = parser.parse_args()
    print(json.dumps(main(args.per_segment, args.k, args.output), indent=2))