"""Seed the Product Knowledge Base deterministically from the 5 Firebird databases. Two phases (Firebird is only reachable from Windows; Postgres only inside the compose network): export (Windows, READ-ONLY SELECT against Firebird): python scripts/seed_product_kb.py export --output docs/runs/ai_kb_seed_payload.json load (inside the fiscal-hub container, against Postgres): python -m central.scripts.seed_product_kb load --input docs/runs/ai_kb_seed_payload.json Export computes, per physical product row: the stable global fingerprint (description|unit; GTIN validated by checksum), and emits observations. Load upserts the GLOBAL product, records per-tenant observations, seeds knowledge already proven by the gate (validated_ncm when the declared NCM is active-official; ERP-declared values stay ERP_OBSERVATION evidence), and links TENANT_PRODUCT_CONTEXT rows. Tenant context is never promoted to global knowledge. """ import argparse import json import sys from datetime import datetime, timezone from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[2])) DATABASES = [ ("TESTESOFICIAIS", None), # path from FIREBIRD_DATABASE_PATH ("CONSTRUCAO", "Banco/CONSTRUCAO.FDB"), ("MMMODAS", "Banco/MMMODAS.FDB"), ("RINCAO", "Banco/RINCAO.FDB"), ("UNIAO", "Banco/UNIAO.FDB"), ] QUERY = "SELECT p.CODIGO, p.PRODUTO, p.UNIDADE, p.CODBARRA, p.GTIN, p.CLASSIFICACAO_FISCAL, p.CEST FROM C000025 p" def _canonical_ncm(value) -> str: digits = "".join(char for char in str(value or "") if char.isdigit()) return digits if len(digits) == 8 else "" def export_payload(limit_per_db: int = 0, output: str = "docs/runs/ai_kb_seed_payload.json") -> dict: import fdb from dotenv import dotenv_values from central.app.ai.fingerprint import normalize_description, product_fingerprint, validate_gtin env = dotenv_values(".env") stats = {"databases": [], "rows_read": 0, "firebird_writes": 0} products = {} for label, relative in DATABASES: path = env["FIREBIRD_DATABASE_PATH"] if relative is None else str(Path(relative).resolve()) connection = fdb.connect(database=path, user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY")) try: cursor = connection.cursor(); cursor.execute(QUERY) columns = [column[0].strip().upper() for column in cursor.description] rows = cursor.fetchmany(limit_per_db) if limit_per_db else cursor.fetchall() finally: connection.close() # collapse the physical duplicate CODIGO rows into observations of the same product seen_codes = set() for row in rows: record = dict(zip(columns, row)) code = str(record.get("CODIGO") or "").strip() dedupe = f"{label}:{code}" if dedupe in seen_codes: # duplicate physical key (e.g. MMMODAS 017421) -> one observation continue seen_codes.add(dedupe) unit = normalize_description(record.get("UNIDADE")) or None fingerprint = product_fingerprint(record.get("PRODUTO"), None, unit) gtin = validate_gtin(record.get("GTIN")) or validate_gtin(record.get("CODBARRA")) entry = products.setdefault(fingerprint, { "fingerprint": fingerprint, "gtin": gtin, "normalized_description": normalize_description(record.get("PRODUTO")), "unit": unit, "ncm": _canonical_ncm(record.get("CLASSIFICACAO_FISCAL")), "cest": str(record.get("CEST") or "").strip() or None, "observations": [], }) if gtin and not entry["gtin"]: entry["gtin"] = gtin entry["observations"].append({"database": label, "company_key": label, "erp_code": code}) stats["databases"].append({"database": label, "rows": len(rows)}) stats["rows_read"] += len(rows) payload = {"artifact": "AI_KB_SEED_PAYLOAD", "exported_at": datetime.now(timezone.utc).isoformat(), "stats": stats, "products": list(products.values())} Path(output).parent.mkdir(parents=True, exist_ok=True) Path(output).write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") stats["unique_products"] = len(products) stats["payload_path"] = output return stats def load_payload(input_path: str) -> dict: from sqlalchemy import select from central.app.ai.kb import KnowledgeBase from central.app.db import SessionLocal from central.app.models import Dataset, NcmEntry, Tenant, TenantProductContext payload = json.loads(Path(input_path).read_text(encoding="utf8")) session = SessionLocal() kb = KnowledgeBase(session) dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")) active_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all()) if dataset else set() tenant_ids = {} stats = {"products": 0, "observations": 0, "validated_ncm": 0, "erp_ncm_observations": 0, "erp_cest_observations": 0, "tenant_contexts": 0, "firebird_writes": 0} for product in payload["products"]: row = kb.upsert_product(fingerprint=product["fingerprint"], gtin=product.get("gtin"), normalized_description=product.get("normalized_description", ""), unit=product.get("unit")) stats["products"] += 1 for observation in product["observations"]: label = observation["database"] if label not in tenant_ids: tenant = session.scalar(select(Tenant).where(Tenant.name == label)) if tenant is None: tenant = Tenant(name=label); session.add(tenant); session.flush() tenant_ids[label] = tenant.id kb.record_observation(row.id, tenant_id=tenant_ids[label], company_key=label, erp_code=observation["erp_code"], database_label=label, observed_fields={"ncm": product.get("ncm") or None, "cest": product.get("cest")}) stats["observations"] += 1 existing = session.scalar(select(TenantProductContext).where(TenantProductContext.tenant_id == tenant_ids[label], TenantProductContext.company_key == label, TenantProductContext.erp_code == observation["erp_code"])) if existing is None: session.add(TenantProductContext(tenant_id=tenant_ids[label], company_key=label, erp_code=observation["erp_code"], kb_product_id=row.id)) stats["tenant_contexts"] += 1 ncm = product.get("ncm") or "" if ncm and ncm in active_ncm: kb.add_knowledge(row.id, "validated_ncm", ncm, source="OFFICIAL_DATASET", confidence=1.0, dataset_versions={"ncm": dataset.sha256}) stats["validated_ncm"] += 1 elif ncm: kb.add_knowledge(row.id, "candidate_ncm", ncm, source="ERP_OBSERVATION", confidence=0.3, evidence={"declared_in": [observation["database"] for observation in product["observations"]]}) stats["erp_ncm_observations"] += 1 if product.get("cest"): kb.add_knowledge(row.id, "candidate_cest", product["cest"], source="ERP_OBSERVATION", confidence=0.3, evidence={"declared_in": [observation["database"] for observation in product["observations"]]}) stats["erp_cest_observations"] += 1 if stats["products"] % 2000 == 0: session.commit() session.commit() stats.update(kb.stats()) session.close() return stats def main(argv=None): parser = argparse.ArgumentParser() sub = parser.add_subparsers(dest="phase", required=True) export_parser = sub.add_parser("export") export_parser.add_argument("--limit", type=int, default=0) export_parser.add_argument("--output", default="docs/runs/ai_kb_seed_payload.json") load_parser = sub.add_parser("load") load_parser.add_argument("--input", default="docs/runs/ai_kb_seed_payload.json") load_parser.add_argument("--output", default="docs/runs/ai_kb_seed.json") args = parser.parse_args(argv) if args.phase == "export": result = export_payload(args.limit, args.output) else: result = load_payload(args.input) result["loaded_at"] = datetime.now(timezone.utc).isoformat() Path(args.output).parent.mkdir(parents=True, exist_ok=True) Path(args.output).write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") print(json.dumps(result, indent=2, ensure_ascii=False)) if __name__ == "__main__": main()