143 lines
7.8 KiB
Python
143 lines
7.8 KiB
Python
"""Local (provider-free) analysis of the historical evidence + leave-one-client-out coverage benchmark.
|
|
|
|
python scripts/enrichment_history_analysis.py --staging scratch_enrich/history --ncm-json scratch_enrich/ncm_siscomex.json --out docs/runs/enrichment_history_analysis.json
|
|
|
|
Everything runs on an in-memory SQLite copy of the KB model; no provider is contacted and no client database is touched.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
from sqlalchemy import create_engine, select # noqa: E402
|
|
from sqlalchemy.orm import Session # noqa: E402
|
|
from sqlalchemy.pool import StaticPool # noqa: E402
|
|
|
|
from central.app.datasets.ncm import parse_ncm_payload # noqa: E402
|
|
from central.app.datasets.service import DatasetService # noqa: E402
|
|
from central.app.enrichment import history, kb # noqa: E402
|
|
from central.app.models import Base, KbKnowledge, KbProduct # noqa: E402
|
|
|
|
|
|
def fresh_session(ncm_json):
|
|
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
|
Base.metadata.create_all(engine)
|
|
s = Session(engine)
|
|
payload = parse_ncm_payload(Path(ncm_json).read_bytes())
|
|
DatasetService(s).import_payload(payload, "official-json", datetime.now(timezone.utc))
|
|
return s
|
|
|
|
|
|
def rate(n, d):
|
|
return round(100 * n / d, 1) if d else None
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--staging", required=True)
|
|
ap.add_argument("--ncm-json", required=True)
|
|
ap.add_argument("--out", required=True)
|
|
args = ap.parse_args()
|
|
files = sorted(f for f in Path(args.staging).glob("*.json") if not f.name.startswith("_"))
|
|
stagings = [json.loads(f.read_text(encoding="utf-8")) for f in files]
|
|
miner = json.loads((Path(args.staging) / "_miner_report.json").read_text(encoding="utf-8"))
|
|
obs = history.build_observations(stagings)
|
|
session = fresh_session(args.ncm_json)
|
|
current = history.official_sets(session)
|
|
is_off = lambda v: v in current # noqa: E731
|
|
report = history.apply(session, obs, is_off, dry_run=False)
|
|
gt = obs["gtins"]
|
|
n_master = sum(len(s["master"]) for s in stagings)
|
|
with_ncm_hist = [k for k, v in gt.items() if v["ncm"]]
|
|
clients_per = {k: len(v["clients"]) for k, v in gt.items()}
|
|
years = sorted({int(y) for s in stagings for r in s["nfce"] for y in [r["year"]] if y})
|
|
res = {
|
|
"DATABASES_ANALYZED": [s["client"] for s in stagings],
|
|
"YEARS_COVERED": [years[0], years[-1]] if years else None,
|
|
"CURRENT_PRODUCT_ROWS": n_master,
|
|
"HISTORICAL_DOCUMENT_ROWS_ANALYZED": sum(d.get("nfce_rows_scanned", 0) + d.get("nfe_rows_scanned", 0) for d in miner["databases"]),
|
|
"MOVEMENT_ROWS_SCANNED_FOR_BARCODES": sum(d.get("c000032_rows_scanned", 0) + d.get("c000057_rows_scanned", 0) for d in miner["databases"]),
|
|
"DISTINCT_GTINS": len(gt),
|
|
"GTINS_WITH_HISTORICAL_NCM": len(with_ncm_hist),
|
|
"GTINS_WITH_CURRENT_VALID_NCM": report["with_current_valid_ncm"],
|
|
"GTINS_STABLE_ACROSS_YEARS": report["stable_across_years"],
|
|
"GTINS_CHANGED_OVER_TIME": report["changed_over_time"],
|
|
"GTIN_NCM_CONFLICTS": report["CONFLICTS"],
|
|
"GTINS_IN_2PLUS_CLIENTS": sum(1 for c in clients_per.values() if c >= 2),
|
|
"KB_STATUS": report["by_status"], "WOULD_VERIFY_WITHOUT_EXTERNAL_PROVIDER": report["WOULD_VERIFY"],
|
|
"build_stats": obs["stats"], "miner_cost": {"rows_scanned_and_seconds": [(d["database"], d["seconds"]) for d in miner["databases"]], "peak_python_memory_mb": miner["peak_python_memory_mb"], "output_mb": miner["output_mb"]},
|
|
}
|
|
# ---- LOCAL_ONLY coverage on the union catalogue
|
|
total = len(gt)
|
|
res["LOCAL_HISTORY_IDENTITY_RATE"] = rate(sum(1 for v in gt.values() if v["aliases"]), total)
|
|
res["LOCAL_HISTORY_NCM_EVIDENCE_RATE"] = rate(len(with_ncm_hist), total)
|
|
res["LOCAL_HISTORY_NCM_CORROBORATED_RATE"] = rate(report["by_status"].get("CORROBORATED", 0) + report["by_status"].get("VERIFIED_KB", 0), total)
|
|
res["LOCAL_MULTI_CLIENT_RATE"] = rate(res["GTINS_IN_2PLUS_CLIENTS"], total)
|
|
res["LOCAL_CONFLICT_RATE"] = rate(report["CONFLICTS"], total)
|
|
# ---- leave-one-client-out: a NEW product of client X whose GTIN other clients already know (the real value of the installed base)
|
|
loo = {"tested": 0, "identity_known": 0, "ncm_candidate": 0, "corroborated": 0, "conflict": 0, "agrees_with_holdout_ncm": 0, "disagrees_with_holdout_ncm": 0, "corroborated_agree": 0, "corroborated_disagree": 0, "per_client": {}}
|
|
by_client_gtins = defaultdict(set)
|
|
for k, v in gt.items():
|
|
for c in v["clients"]:
|
|
by_client_gtins[c].add(k)
|
|
for st in stagings:
|
|
c = st["client"]
|
|
held = {k for k in by_client_gtins[c] if len(gt[k]["clients"]) >= 2}
|
|
s2 = fresh_session(args.ncm_json)
|
|
cur2 = history.official_sets(s2)
|
|
others = history.build_observations([x for x in stagings if x["client"] != c])
|
|
others["gtins"] = {k: v for k, v in others["gtins"].items() if k in held}
|
|
rep2 = history.apply(s2, others, lambda v: v in cur2, dry_run=False)
|
|
mine = history.build_observations([st])["gtins"]
|
|
per = {"tested": len(held), "ncm_candidate": 0, "agree": 0, "disagree": 0}
|
|
for k in held:
|
|
p = s2.scalar(select(KbProduct).where(KbProduct.gtin == k))
|
|
ev = kb.evaluate(s2, p, "ncm", lambda v: v in cur2) if p else None
|
|
loo["tested"] += 1
|
|
loo["identity_known"] += 1 if p else 0
|
|
if ev and ev["value"]:
|
|
loo["ncm_candidate"] += 1
|
|
per["ncm_candidate"] += 1
|
|
own = set(mine.get(k, {}).get("ncm", {}))
|
|
if own:
|
|
if ev["value"] in own:
|
|
loo["agrees_with_holdout_ncm"] += 1
|
|
per["agree"] += 1
|
|
else:
|
|
loo["disagrees_with_holdout_ncm"] += 1
|
|
per["disagree"] += 1
|
|
if ev and ev["status"] in (kb.CORROBORATED, kb.VERIFIED_KB):
|
|
loo["corroborated"] += 1
|
|
own2 = set(mine.get(k, {}).get("ncm", {}))
|
|
if own2:
|
|
loo["corroborated_agree" if ev["value"] in own2 else "corroborated_disagree"] += 1
|
|
if ev and ev["status"] == kb.CONFLICT:
|
|
loo["conflict"] += 1
|
|
loo["per_client"][c] = per
|
|
s2.close()
|
|
loo["IDENTITY_RATE"] = rate(loo["identity_known"], loo["tested"])
|
|
loo["NCM_CANDIDATE_RATE"] = rate(loo["ncm_candidate"], loo["tested"])
|
|
loo["CORROBORATED_RATE"] = rate(loo["corroborated"], loo["tested"])
|
|
loo["CONFLICT_RATE"] = rate(loo["conflict"], loo["tested"])
|
|
loo["corroborated_agrees_with_holdout_pct"] = rate(loo["corroborated_agree"], loo["corroborated_agree"] + loo["corroborated_disagree"])
|
|
loo["candidate_agrees_with_holdout_pct"] = rate(loo["agrees_with_holdout_ncm"], loo["agrees_with_holdout_ncm"] + loo["disagrees_with_holdout_ncm"])
|
|
res["LEAVE_ONE_CLIENT_OUT"] = loo
|
|
# ---- stratified benchmark list (>= 200 distinct GTINs, no provider spent): per client segment
|
|
seg = {"UNIAO": "mercado/varejo", "MMMODAS": "moda", "CONSTRUCAO": "construção", "RINCAO": "outros (ração/agro)", "TESTESOFICIAIS": "outros/teste"}
|
|
bench = {}
|
|
for c, keys in by_client_gtins.items():
|
|
with_ncm = [k for k in sorted(keys) if gt[k]["ncm"]]
|
|
bench[c] = {"segment": seg.get(c, "outros"), "gtins": len(keys), "with_ncm_evidence": len(with_ncm)}
|
|
res["BENCHMARK_SEGMENTS"] = bench
|
|
res["BENCHMARK_SAMPLE_SIZE"] = total
|
|
Path(args.out).write_text(json.dumps(res, indent=1, default=str), encoding="utf-8")
|
|
print(json.dumps(res, indent=1, default=str))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|