feat: product enrichment observability, lab KB study, benchmark, copy + local-Hub E2E, docs; 1.3.0

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

scripts/enrichment_lab_kb.py

@@ -0,0 +1,103 @@
+"""READ-ONLY study of the five lab ERP databases for the product enrichment: GTIN quality, NCM coverage, cross-database agreement, and KB seeding.
+
+    python scripts/enrichment_lab_kb.py [--seed-url sqlite:///kb.db] [--sample 40 --sample-out scratch/benchmark_gtins.json]
+
+Firebird is opened READ-ONLY (no write path exists here). The KB seed goes ONLY to the database given by --seed-url (default: an in-memory SQLite, i.e. statistics only) and stores
+CLIENT_OBSERVATION evidence, never VERIFIED_KB by itself. Company-specific fiscal fields are not read at all.
+"""
+import argparse
+import json
+import os
+import sys
+from collections import Counter, defaultdict
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "scripts"))
+from dotenv import load_dotenv  # noqa: E402
+
+load_dotenv(ROOT / ".env")
+import local_completion as lc  # noqa: E402
+from sqlalchemy import create_engine, select  # noqa: E402
+from sqlalchemy.orm import Session  # noqa: E402
+
+from central.app.enrichment import kb  # noqa: E402
+from central.app.enrichment.gtin import normalize_gtin  # noqa: E402
+from central.app.enrichment.observations import observe  # noqa: E402
+from central.app.enrichment.resolver import ncm_is_official  # noqa: E402
+from central.app.models import Base, Dataset, KbKnowledge, NcmEntry  # noqa: E402
+
+
+def read_lab(name, path):
+    conn = lc.connect(path)
+    try:
+        cur = conn.cursor()
+        cur.execute("SELECT CODIGO, PRODUTO, GTIN, CLASSIFICACAO_FISCAL, CODBARRA FROM C000025")
+        # the SGV operator types the EAN in CODBARRA; GTIN (fiscal) is mostly empty. The first VALID one wins (same rule as the Agent).
+        return [{"CODIGO": r[0], "PRODUTO": (r[1] or "").strip(), "GTIN": next((x for x in (r[2], r[4]) if normalize_gtin(x)["status"] == "VALID_GTIN"), r[2] or r[4]), "NCM": r[3]} for r in cur.fetchall()]
+    finally:
+        conn.close()
+
+
+def load_ncm_official(session, ncm_json: Path | None):
+    """Official NCM into the throw-away/seed session when it has none (parsed from the saved official JSON)."""
+    if session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")):
+        return
+    from central.app.datasets.ncm import parse_ncm_payload
+    from central.app.datasets.service import DatasetService
+    from datetime import datetime, timezone
+    payload = parse_ncm_payload(ncm_json.read_bytes())
+    DatasetService(session).import_payload(payload, "official-json", datetime.now(timezone.utc))
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--seed-url", default="sqlite://")
+    ap.add_argument("--ncm-json", default=None, help="official Siscomex JSON (used only if the seed DB has no NCM dataset)")
+    ap.add_argument("--sample", type=int, default=0)
+    ap.add_argument("--sample-out", default=None)
+    args = ap.parse_args()
+    engine = create_engine(args.seed_url)
+    Base.metadata.create_all(engine)
+    dbs = dict(lc.allowlist())
+    dbs["TESTESOFICIAIS"] = os.environ.get("FIREBIRD_DATABASE_PATH", str(ROOT / "TESTESOFICIAIS.FDB"))
+    report, all_rows = {}, {}
+    with Session(engine) as session:
+        load_ncm_official(session, Path(args.ncm_json) if args.ncm_json else None)
+        official = lambda v: ncm_is_official(session, v)  # noqa: E731
+        for name, path in dbs.items():
+            if not Path(path).exists():
+                report[name] = {"error": "missing"}
+                continue
+            rows = read_lab(name, path)
+            all_rows[name] = rows
+            status = Counter(normalize_gtin(r["GTIN"])["status"] for r in rows)
+            stats = observe(session, rows, name, official)
+            ncm_ok = sum(1 for r in rows if len("".join(c for c in str(r["NCM"] or "") if c.isdigit())) == 8 and official("".join(c for c in str(r["NCM"]) if c.isdigit())))
+            report[name] = {"products": len(rows), "gtin_status": dict(status), "valid_gtin_rate": round(100 * status["VALID_GTIN"] / max(1, len(rows)), 1), "official_ncm_rate": round(100 * ncm_ok / max(1, len(rows)), 1), **stats}
+        session.commit()
+        by_gtin = defaultdict(set)
+        for name, rows in all_rows.items():
+            for r in rows:
+                g = normalize_gtin(r["GTIN"])
+                if g["status"] == "VALID_GTIN":
+                    by_gtin[g["key"]].add(name)
+        shared = {k: v for k, v in by_gtin.items() if len(v) >= 2}
+        conflicts = 0
+        for kp in session.scalars(select(kb.KbProduct)):
+            ncms = {r.value for r in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == kp.id, KbKnowledge.knowledge_key == "ncm"))}
+            conflicts += len(ncms) > 1
+        summary = {"databases": report, "distinct_valid_gtin": len(by_gtin), "gtin_in_2plus_databases": len(shared), "kb_products": session.scalar(select(kb.KbProduct.id).limit(1)) is not None and session.query(kb.KbProduct).count(),
+                   "kb_ncm_conflicts_between_clients": conflicts}
+        if args.sample and args.sample_out:
+            picks = sorted(by_gtin)[:: max(1, len(by_gtin) // args.sample)][: args.sample]
+            names = {normalize_gtin(r["GTIN"])["key"]: r["PRODUTO"] for rows in all_rows.values() for r in rows if normalize_gtin(r["GTIN"])["status"] == "VALID_GTIN"}
+            Path(args.sample_out).parent.mkdir(parents=True, exist_ok=True)
+            Path(args.sample_out).write_text(json.dumps([{"gtin": k[1:] if k.startswith("0") else k, "erp_description": names[k]} for k in picks], ensure_ascii=False, indent=1), encoding="utf-8")
+            summary["sample_written"] = len(picks)
+    print(json.dumps(summary, ensure_ascii=False, indent=1))
+
+
+if __name__ == "__main__":
+    main()