chore: capture semantic pipeline baseline
This commit is contained in:
OpenCode
@@ -0,0 +1,158 @@
|
||||
"""Seed the Product Knowledge Base deterministically from the 5 Firebird databases.
|
||||
|
||||
Two phases (Firebird is only reachable from Windows; Postgres only inside the compose network):
|
||||
|
||||
export (Windows, READ-ONLY SELECT against Firebird):
|
||||
python scripts/seed_product_kb.py export --output docs/runs/ai_kb_seed_payload.json
|
||||
load (inside the fiscal-hub container, against Postgres):
|
||||
python -m central.scripts.seed_product_kb load --input docs/runs/ai_kb_seed_payload.json
|
||||
|
||||
Export computes, per physical product row: the stable global fingerprint
|
||||
(description|unit; GTIN validated by checksum), and emits observations. Load upserts the
|
||||
GLOBAL product, records per-tenant observations, seeds knowledge already proven by the
|
||||
gate (validated_ncm when the declared NCM is active-official; ERP-declared values stay
|
||||
ERP_OBSERVATION evidence), and links TENANT_PRODUCT_CONTEXT rows. Tenant context is
|
||||
never promoted to global knowledge.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
DATABASES = [
|
||||
("TESTESOFICIAIS", None), # path from FIREBIRD_DATABASE_PATH
|
||||
("CONSTRUCAO", "Banco/CONSTRUCAO.FDB"),
|
||||
("MMMODAS", "Banco/MMMODAS.FDB"),
|
||||
("RINCAO", "Banco/RINCAO.FDB"),
|
||||
("UNIAO", "Banco/UNIAO.FDB"),
|
||||
]
|
||||
|
||||
QUERY = "SELECT p.CODIGO, p.PRODUTO, p.UNIDADE, p.CODBARRA, p.GTIN, p.CLASSIFICACAO_FISCAL, p.CEST FROM C000025 p"
|
||||
|
||||
|
||||
def _canonical_ncm(value) -> str:
|
||||
digits = "".join(char for char in str(value or "") if char.isdigit())
|
||||
return digits if len(digits) == 8 else ""
|
||||
|
||||
|
||||
def export_payload(limit_per_db: int = 0, output: str = "docs/runs/ai_kb_seed_payload.json") -> dict:
|
||||
import fdb
|
||||
from dotenv import dotenv_values
|
||||
from central.app.ai.fingerprint import normalize_description, product_fingerprint, validate_gtin
|
||||
env = dotenv_values(".env")
|
||||
stats = {"databases": [], "rows_read": 0, "firebird_writes": 0}
|
||||
products = {}
|
||||
for label, relative in DATABASES:
|
||||
path = env["FIREBIRD_DATABASE_PATH"] if relative is None else str(Path(relative).resolve())
|
||||
connection = fdb.connect(database=path, user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"))
|
||||
try:
|
||||
cursor = connection.cursor(); cursor.execute(QUERY)
|
||||
columns = [column[0].strip().upper() for column in cursor.description]
|
||||
rows = cursor.fetchmany(limit_per_db) if limit_per_db else cursor.fetchall()
|
||||
finally:
|
||||
connection.close()
|
||||
# collapse the physical duplicate CODIGO rows into observations of the same product
|
||||
seen_codes = set()
|
||||
for row in rows:
|
||||
record = dict(zip(columns, row))
|
||||
code = str(record.get("CODIGO") or "").strip()
|
||||
dedupe = f"{label}:{code}"
|
||||
if dedupe in seen_codes: # duplicate physical key (e.g. MMMODAS 017421) -> one observation
|
||||
continue
|
||||
seen_codes.add(dedupe)
|
||||
unit = normalize_description(record.get("UNIDADE")) or None
|
||||
fingerprint = product_fingerprint(record.get("PRODUTO"), None, unit)
|
||||
gtin = validate_gtin(record.get("GTIN")) or validate_gtin(record.get("CODBARRA"))
|
||||
entry = products.setdefault(fingerprint, {
|
||||
"fingerprint": fingerprint,
|
||||
"gtin": gtin,
|
||||
"normalized_description": normalize_description(record.get("PRODUTO")),
|
||||
"unit": unit,
|
||||
"ncm": _canonical_ncm(record.get("CLASSIFICACAO_FISCAL")),
|
||||
"cest": str(record.get("CEST") or "").strip() or None,
|
||||
"observations": [],
|
||||
})
|
||||
if gtin and not entry["gtin"]:
|
||||
entry["gtin"] = gtin
|
||||
entry["observations"].append({"database": label, "company_key": label, "erp_code": code})
|
||||
stats["databases"].append({"database": label, "rows": len(rows)})
|
||||
stats["rows_read"] += len(rows)
|
||||
payload = {"artifact": "AI_KB_SEED_PAYLOAD", "exported_at": datetime.now(timezone.utc).isoformat(), "stats": stats, "products": list(products.values())}
|
||||
Path(output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(output).write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
stats["unique_products"] = len(products)
|
||||
stats["payload_path"] = output
|
||||
return stats
|
||||
|
||||
|
||||
def load_payload(input_path: str) -> dict:
|
||||
from sqlalchemy import select
|
||||
from central.app.ai.kb import KnowledgeBase
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.models import Dataset, NcmEntry, Tenant, TenantProductContext
|
||||
payload = json.loads(Path(input_path).read_text(encoding="utf8"))
|
||||
session = SessionLocal()
|
||||
kb = KnowledgeBase(session)
|
||||
dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
active_ncm = set(session.scalars(select(NcmEntry.code).where(NcmEntry.dataset_id == dataset.id)).all()) if dataset else set()
|
||||
tenant_ids = {}
|
||||
stats = {"products": 0, "observations": 0, "validated_ncm": 0, "erp_ncm_observations": 0, "erp_cest_observations": 0, "tenant_contexts": 0, "firebird_writes": 0}
|
||||
for product in payload["products"]:
|
||||
row = kb.upsert_product(fingerprint=product["fingerprint"], gtin=product.get("gtin"), normalized_description=product.get("normalized_description", ""), unit=product.get("unit"))
|
||||
stats["products"] += 1
|
||||
for observation in product["observations"]:
|
||||
label = observation["database"]
|
||||
if label not in tenant_ids:
|
||||
tenant = session.scalar(select(Tenant).where(Tenant.name == label))
|
||||
if tenant is None:
|
||||
tenant = Tenant(name=label); session.add(tenant); session.flush()
|
||||
tenant_ids[label] = tenant.id
|
||||
kb.record_observation(row.id, tenant_id=tenant_ids[label], company_key=label, erp_code=observation["erp_code"], database_label=label, observed_fields={"ncm": product.get("ncm") or None, "cest": product.get("cest")})
|
||||
stats["observations"] += 1
|
||||
existing = session.scalar(select(TenantProductContext).where(TenantProductContext.tenant_id == tenant_ids[label], TenantProductContext.company_key == label, TenantProductContext.erp_code == observation["erp_code"]))
|
||||
if existing is None:
|
||||
session.add(TenantProductContext(tenant_id=tenant_ids[label], company_key=label, erp_code=observation["erp_code"], kb_product_id=row.id))
|
||||
stats["tenant_contexts"] += 1
|
||||
ncm = product.get("ncm") or ""
|
||||
if ncm and ncm in active_ncm:
|
||||
kb.add_knowledge(row.id, "validated_ncm", ncm, source="OFFICIAL_DATASET", confidence=1.0, dataset_versions={"ncm": dataset.sha256})
|
||||
stats["validated_ncm"] += 1
|
||||
elif ncm:
|
||||
kb.add_knowledge(row.id, "candidate_ncm", ncm, source="ERP_OBSERVATION", confidence=0.3, evidence={"declared_in": [observation["database"] for observation in product["observations"]]})
|
||||
stats["erp_ncm_observations"] += 1
|
||||
if product.get("cest"):
|
||||
kb.add_knowledge(row.id, "candidate_cest", product["cest"], source="ERP_OBSERVATION", confidence=0.3, evidence={"declared_in": [observation["database"] for observation in product["observations"]]})
|
||||
stats["erp_cest_observations"] += 1
|
||||
if stats["products"] % 2000 == 0:
|
||||
session.commit()
|
||||
session.commit()
|
||||
stats.update(kb.stats())
|
||||
session.close()
|
||||
return stats
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers(dest="phase", required=True)
|
||||
export_parser = sub.add_parser("export")
|
||||
export_parser.add_argument("--limit", type=int, default=0)
|
||||
export_parser.add_argument("--output", default="docs/runs/ai_kb_seed_payload.json")
|
||||
load_parser = sub.add_parser("load")
|
||||
load_parser.add_argument("--input", default="docs/runs/ai_kb_seed_payload.json")
|
||||
load_parser.add_argument("--output", default="docs/runs/ai_kb_seed.json")
|
||||
args = parser.parse_args(argv)
|
||||
if args.phase == "export":
|
||||
result = export_payload(args.limit, args.output)
|
||||
else:
|
||||
result = load_payload(args.input)
|
||||
result["loaded_at"] = datetime.now(timezone.utc).isoformat()
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user