159 lines
8.4 KiB
Python
159 lines
8.4 KiB
Python
"""Seed the Product Knowledge Base deterministically from the 5 Firebird databases.
|
|
|
|
Two phases (Firebird is only reachable from Windows; Postgres only inside the compose network):
|
|
|
|
export (Windows, READ-ONLY SELECT against Firebird):
|
|
python scripts/seed_product_kb.py export --output docs/runs/ai_kb_seed_payload.json
|
|
load (inside the fiscal-hub container, against Postgres):
|
|
python -m central.scripts.seed_product_kb load --input docs/runs/ai_kb_seed_payload.json
|
|
|
|
Export computes, per physical product row: the stable global fingerprint
|
|
(description|unit; GTIN validated by checksum), and emits observations. Load upserts the
|
|
GLOBAL product, records per-tenant observations, seeds knowledge already proven by the
|
|
gate (validated_ncm when the declared NCM is active-official; ERP-declared values stay
|
|
ERP_OBSERVATION evidence), and links TENANT_PRODUCT_CONTEXT rows. Tenant context is
|
|
never promoted to global knowledge.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
|
|
DATABASES = [
|
|
("TESTESOFICIAIS", None), # path from FIREBIRD_DATABASE_PATH
|
|
("CONSTRUCAO", "Banco/CONSTRUCAO.FDB"),
|
|
("MMMODAS", "Banco/MMMODAS.FDB"),
|
|
("RINCAO", "Banco/RINCAO.FDB"),
|
|
("UNIAO", "Banco/UNIAO.FDB"),
|
|
]
|
|
|
|
QUERY = "SELECT p.CODIGO, p.PRODUTO, p.UNIDADE, p.CODBARRA, p.GTIN, p.CLASSIFICACAO_FISCAL, p.CEST FROM C000025 p"
|
|
|
|
|
|
def _canonical_ncm(value) -> str:
|
|
digits = "".join(char for char in str(value or "") if char.isdigit())
|
|
return digits if len(digits) == 8 else ""
|
|
|
|
|
|
def export_payload(limit_per_db: int = 0, output: str = "docs/runs/ai_kb_seed_payload.json") -> dict:
|
|
import fdb
|
|
from dotenv import dotenv_values
|
|
from central.app.ai.fingerprint import normalize_description, product_fingerprint, validate_gtin
|
|
env = dotenv_values(".env")
|
|
stats = {"databases": [], "rows_read": 0, "firebird_writes": 0}
|
|
products = {}
|
|
for label, relative in DATABASES:
|
|
path = env["FIREBIRD_DATABASE_PATH"] if relative is None else str(Path(relative).resolve())
|
|
connection = fdb.connect(database=path, user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"))
|
|
try:
|
|
cursor = connection.cursor(); cursor.execute(QUERY)
|
|
columns = [column[0].strip().upper() for column in cursor.description]
|
|
rows = cursor.fetchmany(limit_per_db) if limit_per_db else cursor.fetchall()
|
|
finally:
|
|
connection.close()
|
|
# collapse the physical duplicate CODIGO rows into observations of the same product
|
|
seen_codes = set()
|
|
for row in rows:
|
|
record = dict(zip(columns, row))
|
|
code = str(record.get("CODIGO") or "").strip()
|
|
dedupe = f"{label}:{code}"
|
|
if dedupe in seen_codes: # duplicate physical key (e.g. MMMODAS 017421) -> one observation
|
|
continue
|
|
seen_codes.add(dedupe)
|
|
unit = normalize_description(record.get("UNIDADE")) or None
|
|
fingerprint = product_fingerprint(record.get("PRODUTO"), None, unit)
|
|
gtin = validate_gtin(record.get("GTIN")) or validate_gtin(record.get("CODBARRA"))
|
|
entry = products.setdefault(fingerprint, {
|
|
"fingerprint": fingerprint,
|
|
"gtin": gtin,
|
|
"normalized_description": normalize_description(record.get("PRODUTO")),
|
|
"unit": unit,
|
|
"ncm": _canonical_ncm(record.get("CLASSIFICACAO_FISCAL")),
|
|
"cest": str(record.get("CEST") or "").strip() or None,
|
|
"observations": [],
|
|
})
|
|
if gtin and not entry["gtin"]:
|
|
entry["gtin"] = gtin
|
|
entry["observations"].append({"database": label, "company_key": label, "erp_code": code})
|
|
stats["databases"].append({"database": label, "rows": len(rows)})
|
|
stats["rows_read"] += len(rows)
|
|
payload = {"artifact": "AI_KB_SEED_PAYLOAD", "exported_at": datetime.now(timezone.utc).isoformat(), "stats": stats, "products": list(products.values())}
|
|
Path(output).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(output).write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
|
stats["unique_products"] = len(products)
|
|
stats["payload_path"] = output
|
|
return stats
|
|
|
|
|
|
def load_payload(input_path: str) -> dict:
|
|
from sqlalchemy import select
|
|
from central.app.ai.kb import KnowledgeBase
|
|
from central.app.db import SessionLocal
|
|
from central.app.models import Dataset, NcmEntry, Tenant, TenantProductContext
|
|
payload = json.loads(Path(input_path).read_text(encoding="utf8"))
|
|
session = SessionLocal()
|
|
kb = KnowledgeBase(session)
|
|
dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
|
active_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all()) if dataset else set()
|
|
tenant_ids = {}
|
|
stats = {"products": 0, "observations": 0, "validated_ncm": 0, "erp_ncm_observations": 0, "erp_cest_observations": 0, "tenant_contexts": 0, "firebird_writes": 0}
|
|
for product in payload["products"]:
|
|
row = kb.upsert_product(fingerprint=product["fingerprint"], gtin=product.get("gtin"), normalized_description=product.get("normalized_description", ""), unit=product.get("unit"))
|
|
stats["products"] += 1
|
|
for observation in product["observations"]:
|
|
label = observation["database"]
|
|
if label not in tenant_ids:
|
|
tenant = session.scalar(select(Tenant).where(Tenant.name == label))
|
|
if tenant is None:
|
|
tenant = Tenant(name=label); session.add(tenant); session.flush()
|
|
tenant_ids[label] = tenant.id
|
|
kb.record_observation(row.id, tenant_id=tenant_ids[label], company_key=label, erp_code=observation["erp_code"], database_label=label, observed_fields={"ncm": product.get("ncm") or None, "cest": product.get("cest")})
|
|
stats["observations"] += 1
|
|
existing = session.scalar(select(TenantProductContext).where(TenantProductContext.tenant_id == tenant_ids[label], TenantProductContext.company_key == label, TenantProductContext.erp_code == observation["erp_code"]))
|
|
if existing is None:
|
|
session.add(TenantProductContext(tenant_id=tenant_ids[label], company_key=label, erp_code=observation["erp_code"], kb_product_id=row.id))
|
|
stats["tenant_contexts"] += 1
|
|
ncm = product.get("ncm") or ""
|
|
if ncm and ncm in active_ncm:
|
|
kb.add_knowledge(row.id, "validated_ncm", ncm, source="OFFICIAL_DATASET", confidence=1.0, dataset_versions={"ncm": dataset.sha256})
|
|
stats["validated_ncm"] += 1
|
|
elif ncm:
|
|
kb.add_knowledge(row.id, "candidate_ncm", ncm, source="ERP_OBSERVATION", confidence=0.3, evidence={"declared_in": [observation["database"] for observation in product["observations"]]})
|
|
stats["erp_ncm_observations"] += 1
|
|
if product.get("cest"):
|
|
kb.add_knowledge(row.id, "candidate_cest", product["cest"], source="ERP_OBSERVATION", confidence=0.3, evidence={"declared_in": [observation["database"] for observation in product["observations"]]})
|
|
stats["erp_cest_observations"] += 1
|
|
if stats["products"] % 2000 == 0:
|
|
session.commit()
|
|
session.commit()
|
|
stats.update(kb.stats())
|
|
session.close()
|
|
return stats
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser()
|
|
sub = parser.add_subparsers(dest="phase", required=True)
|
|
export_parser = sub.add_parser("export")
|
|
export_parser.add_argument("--limit", type=int, default=0)
|
|
export_parser.add_argument("--output", default="docs/runs/ai_kb_seed_payload.json")
|
|
load_parser = sub.add_parser("load")
|
|
load_parser.add_argument("--input", default="docs/runs/ai_kb_seed_payload.json")
|
|
load_parser.add_argument("--output", default="docs/runs/ai_kb_seed.json")
|
|
args = parser.parse_args(argv)
|
|
if args.phase == "export":
|
|
result = export_payload(args.limit, args.output)
|
|
else:
|
|
result = load_payload(args.input)
|
|
result["loaded_at"] = datetime.now(timezone.utc).isoformat()
|
|
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(args.output).write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
print(json.dumps(result, indent=2, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|