GTI_GERENCIADOR_FISCAL/fiscal/scripts/enrichment_history_analysis.py

"""Local (provider-free) analysis of the historical evidence + leave-one-client-out coverage benchmark.

    python scripts/enrichment_history_analysis.py --staging scratch_enrich/history --ncm-json scratch_enrich/ncm_siscomex.json --out docs/runs/enrichment_history_analysis.json

Everything runs on an in-memory SQLite copy of the KB model; no provider is contacted and no client database is touched.
"""
import argparse
import json
import sys
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from sqlalchemy import create_engine, select  # noqa: E402
from sqlalchemy.orm import Session  # noqa: E402
from sqlalchemy.pool import StaticPool  # noqa: E402

from central.app.datasets.ncm import parse_ncm_payload  # noqa: E402
from central.app.datasets.service import DatasetService  # noqa: E402
from central.app.enrichment import history, kb  # noqa: E402
from central.app.models import Base, KbKnowledge, KbProduct  # noqa: E402


def fresh_session(ncm_json):
    engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
    Base.metadata.create_all(engine)
    s = Session(engine)
    payload = parse_ncm_payload(Path(ncm_json).read_bytes())
    DatasetService(s).import_payload(payload, "official-json", datetime.now(timezone.utc))
    return s


def rate(n, d):
    return round(100 * n / d, 1) if d else None


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--staging", required=True)
    ap.add_argument("--ncm-json", required=True)
    ap.add_argument("--out", required=True)
    args = ap.parse_args()
    files = sorted(f for f in Path(args.staging).glob("*.json") if not f.name.startswith("_"))
    stagings = [json.loads(f.read_text(encoding="utf-8")) for f in files]
    miner = json.loads((Path(args.staging) / "_miner_report.json").read_text(encoding="utf-8"))
    obs = history.build_observations(stagings)
    session = fresh_session(args.ncm_json)
    current = history.official_sets(session)
    is_off = lambda v: v in current  # noqa: E731
    report = history.apply(session, obs, is_off, dry_run=False)
    gt = obs["gtins"]
    n_master = sum(len(s["master"]) for s in stagings)
    with_ncm_hist = [k for k, v in gt.items() if v["ncm"]]
    clients_per = {k: len(v["clients"]) for k, v in gt.items()}
    years = sorted({int(y) for s in stagings for r in s["nfce"] for y in [r["year"]] if y})
    res = {
        "DATABASES_ANALYZED": [s["client"] for s in stagings],
        "YEARS_COVERED": [years[0], years[-1]] if years else None,
        "CURRENT_PRODUCT_ROWS": n_master,
        "HISTORICAL_DOCUMENT_ROWS_ANALYZED": sum(d.get("nfce_rows_scanned", 0) + d.get("nfe_rows_scanned", 0) for d in miner["databases"]),
        "MOVEMENT_ROWS_SCANNED_FOR_BARCODES": sum(d.get("c000032_rows_scanned", 0) + d.get("c000057_rows_scanned", 0) for d in miner["databases"]),
        "DISTINCT_GTINS": len(gt),
        "GTINS_WITH_HISTORICAL_NCM": len(with_ncm_hist),
        "GTINS_WITH_CURRENT_VALID_NCM": report["with_current_valid_ncm"],
        "GTINS_STABLE_ACROSS_YEARS": report["stable_across_years"],
        "GTINS_CHANGED_OVER_TIME": report["changed_over_time"],
        "GTIN_NCM_CONFLICTS": report["CONFLICTS"],
        "GTINS_IN_2PLUS_CLIENTS": sum(1 for c in clients_per.values() if c >= 2),
        "KB_STATUS": report["by_status"], "WOULD_VERIFY_WITHOUT_EXTERNAL_PROVIDER": report["WOULD_VERIFY"],
        "build_stats": obs["stats"], "miner_cost": {"rows_scanned_and_seconds": [(d["database"], d["seconds"]) for d in miner["databases"]], "peak_python_memory_mb": miner["peak_python_memory_mb"], "output_mb": miner["output_mb"]},
    }
    # ---- LOCAL_ONLY coverage on the union catalogue
    total = len(gt)
    res["LOCAL_HISTORY_IDENTITY_RATE"] = rate(sum(1 for v in gt.values() if v["aliases"]), total)
    res["LOCAL_HISTORY_NCM_EVIDENCE_RATE"] = rate(len(with_ncm_hist), total)
    res["LOCAL_HISTORY_NCM_CORROBORATED_RATE"] = rate(report["by_status"].get("CORROBORATED", 0) + report["by_status"].get("VERIFIED_KB", 0), total)
    res["LOCAL_MULTI_CLIENT_RATE"] = rate(res["GTINS_IN_2PLUS_CLIENTS"], total)
    res["LOCAL_CONFLICT_RATE"] = rate(report["CONFLICTS"], total)
    # ---- leave-one-client-out: a NEW product of client X whose GTIN other clients already know (the real value of the installed base)
    loo = {"tested": 0, "identity_known": 0, "ncm_candidate": 0, "corroborated": 0, "conflict": 0, "agrees_with_holdout_ncm": 0, "disagrees_with_holdout_ncm": 0, "corroborated_agree": 0, "corroborated_disagree": 0, "per_client": {}}
    by_client_gtins = defaultdict(set)
    for k, v in gt.items():
        for c in v["clients"]:
            by_client_gtins[c].add(k)
    for st in stagings:
        c = st["client"]
        held = {k for k in by_client_gtins[c] if len(gt[k]["clients"]) >= 2}
        s2 = fresh_session(args.ncm_json)
        cur2 = history.official_sets(s2)
        others = history.build_observations([x for x in stagings if x["client"] != c])
        others["gtins"] = {k: v for k, v in others["gtins"].items() if k in held}
        rep2 = history.apply(s2, others, lambda v: v in cur2, dry_run=False)
        mine = history.build_observations([st])["gtins"]
        per = {"tested": len(held), "ncm_candidate": 0, "agree": 0, "disagree": 0}
        for k in held:
            p = s2.scalar(select(KbProduct).where(KbProduct.gtin == k))
            ev = kb.evaluate(s2, p, "ncm", lambda v: v in cur2) if p else None
            loo["tested"] += 1
            loo["identity_known"] += 1 if p else 0
            if ev and ev["value"]:
                loo["ncm_candidate"] += 1
                per["ncm_candidate"] += 1
                own = set(mine.get(k, {}).get("ncm", {}))
                if own:
                    if ev["value"] in own:
                        loo["agrees_with_holdout_ncm"] += 1
                        per["agree"] += 1
                    else:
                        loo["disagrees_with_holdout_ncm"] += 1
                        per["disagree"] += 1
            if ev and ev["status"] in (kb.CORROBORATED, kb.VERIFIED_KB):
                loo["corroborated"] += 1
                own2 = set(mine.get(k, {}).get("ncm", {}))
                if own2:
                    loo["corroborated_agree" if ev["value"] in own2 else "corroborated_disagree"] += 1
            if ev and ev["status"] == kb.CONFLICT:
                loo["conflict"] += 1
        loo["per_client"][c] = per
        s2.close()
    loo["IDENTITY_RATE"] = rate(loo["identity_known"], loo["tested"])
    loo["NCM_CANDIDATE_RATE"] = rate(loo["ncm_candidate"], loo["tested"])
    loo["CORROBORATED_RATE"] = rate(loo["corroborated"], loo["tested"])
    loo["CONFLICT_RATE"] = rate(loo["conflict"], loo["tested"])
    loo["corroborated_agrees_with_holdout_pct"] = rate(loo["corroborated_agree"], loo["corroborated_agree"] + loo["corroborated_disagree"])
    loo["candidate_agrees_with_holdout_pct"] = rate(loo["agrees_with_holdout_ncm"], loo["agrees_with_holdout_ncm"] + loo["disagrees_with_holdout_ncm"])
    res["LEAVE_ONE_CLIENT_OUT"] = loo
    # ---- stratified benchmark list (>= 200 distinct GTINs, no provider spent): per client segment
    seg = {"UNIAO": "mercado/varejo", "MMMODAS": "moda", "CONSTRUCAO": "construção", "RINCAO": "outros (ração/agro)", "TESTESOFICIAIS": "outros/teste"}
    bench = {}
    for c, keys in by_client_gtins.items():
        with_ncm = [k for k in sorted(keys) if gt[k]["ncm"]]
        bench[c] = {"segment": seg.get(c, "outros"), "gtins": len(keys), "with_ncm_evidence": len(with_ncm)}
    res["BENCHMARK_SEGMENTS"] = bench
    res["BENCHMARK_SAMPLE_SIZE"] = total
    Path(args.out).write_text(json.dumps(res, indent=1, default=str), encoding="utf-8")
    print(json.dumps(res, indent=1, default=str))


if __name__ == "__main__":
    main()