GTI_GERENCIADOR_FISCAL/fiscal/scripts/enrichment_lab_kb.py

"""READ-ONLY study of the five lab ERP databases for the product enrichment: GTIN quality, NCM coverage, cross-database agreement, and KB seeding.

    python scripts/enrichment_lab_kb.py [--seed-url sqlite:///kb.db] [--sample 40 --sample-out scratch/benchmark_gtins.json]

Firebird is opened READ-ONLY (no write path exists here). The KB seed goes ONLY to the database given by --seed-url (default: an in-memory SQLite, i.e. statistics only) and stores
CLIENT_OBSERVATION evidence, never VERIFIED_KB by itself. Company-specific fiscal fields are not read at all.
"""
import argparse
import json
import os
import sys
from collections import Counter, defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "scripts"))
from dotenv import load_dotenv  # noqa: E402

load_dotenv(ROOT / ".env")
import local_completion as lc  # noqa: E402
from sqlalchemy import create_engine, select  # noqa: E402
from sqlalchemy.orm import Session  # noqa: E402

from central.app.enrichment import kb  # noqa: E402
from central.app.enrichment.gtin import normalize_gtin  # noqa: E402
from central.app.enrichment.observations import observe  # noqa: E402
from central.app.enrichment.resolver import ncm_is_official  # noqa: E402
from central.app.models import Base, Dataset, KbKnowledge, NcmEntry  # noqa: E402


def read_lab(name, path):
    conn = lc.connect(path)
    try:
        cur = conn.cursor()
        cur.execute("SELECT CODIGO, PRODUTO, GTIN, CLASSIFICACAO_FISCAL, CODBARRA FROM C000025")
        # the SGV operator types the EAN in CODBARRA; GTIN (fiscal) is mostly empty. The first VALID one wins (same rule as the Agent).
        return [{"CODIGO": r[0], "PRODUTO": (r[1] or "").strip(), "GTIN": next((x for x in (r[2], r[4]) if normalize_gtin(x)["status"] == "VALID_GTIN"), r[2] or r[4]), "NCM": r[3]} for r in cur.fetchall()]
    finally:
        conn.close()


def load_ncm_official(session, ncm_json: Path | None):
    """Official NCM into the throw-away/seed session when it has none (parsed from the saved official JSON)."""
    if session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")):
        return
    from central.app.datasets.ncm import parse_ncm_payload
    from central.app.datasets.service import DatasetService
    from datetime import datetime, timezone
    payload = parse_ncm_payload(ncm_json.read_bytes())
    DatasetService(session).import_payload(payload, "official-json", datetime.now(timezone.utc))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--seed-url", default="sqlite://")
    ap.add_argument("--ncm-json", default=None, help="official Siscomex JSON (used only if the seed DB has no NCM dataset)")
    ap.add_argument("--sample", type=int, default=0)
    ap.add_argument("--sample-out", default=None)
    args = ap.parse_args()
    engine = create_engine(args.seed_url)
    Base.metadata.create_all(engine)
    dbs = dict(lc.allowlist())
    dbs["TESTESOFICIAIS"] = os.environ.get("FIREBIRD_DATABASE_PATH", str(ROOT / "TESTESOFICIAIS.FDB"))
    report, all_rows = {}, {}
    with Session(engine) as session:
        load_ncm_official(session, Path(args.ncm_json) if args.ncm_json else None)
        official = lambda v: ncm_is_official(session, v)  # noqa: E731
        for name, path in dbs.items():
            if not Path(path).exists():
                report[name] = {"error": "missing"}
                continue
            rows = read_lab(name, path)
            all_rows[name] = rows
            status = Counter(normalize_gtin(r["GTIN"])["status"] for r in rows)
            stats = observe(session, rows, name, official)
            ncm_ok = sum(1 for r in rows if len("".join(c for c in str(r["NCM"] or "") if c.isdigit())) == 8 and official("".join(c for c in str(r["NCM"]) if c.isdigit())))
            report[name] = {"products": len(rows), "gtin_status": dict(status), "valid_gtin_rate": round(100 * status["VALID_GTIN"] / max(1, len(rows)), 1), "official_ncm_rate": round(100 * ncm_ok / max(1, len(rows)), 1), **stats}
        session.commit()
        by_gtin = defaultdict(set)
        for name, rows in all_rows.items():
            for r in rows:
                g = normalize_gtin(r["GTIN"])
                if g["status"] == "VALID_GTIN":
                    by_gtin[g["key"]].add(name)
        shared = {k: v for k, v in by_gtin.items() if len(v) >= 2}
        conflicts = 0
        for kp in session.scalars(select(kb.KbProduct)):
            ncms = {r.value for r in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == kp.id, KbKnowledge.knowledge_key == "ncm"))}
            conflicts += len(ncms) > 1
        summary = {"databases": report, "distinct_valid_gtin": len(by_gtin), "gtin_in_2plus_databases": len(shared), "kb_products": session.scalar(select(kb.KbProduct.id).limit(1)) is not None and session.query(kb.KbProduct).count(),
                   "kb_ncm_conflicts_between_clients": conflicts}
        if args.sample and args.sample_out:
            picks = sorted(by_gtin)[:: max(1, len(by_gtin) // args.sample)][: args.sample]
            names = {normalize_gtin(r["GTIN"])["key"]: r["PRODUTO"] for rows in all_rows.values() for r in rows if normalize_gtin(r["GTIN"])["status"] == "VALID_GTIN"}
            Path(args.sample_out).parent.mkdir(parents=True, exist_ok=True)
            Path(args.sample_out).write_text(json.dumps([{"gtin": k[1:] if k.startswith("0") else k, "erp_description": names[k]} for k in picks], ensure_ascii=False, indent=1), encoding="utf-8")
            summary["sample_written"] = len(picks)
    print(json.dumps(summary, ensure_ascii=False, indent=1))


if __name__ == "__main__":
    main()