84 lines
4.0 KiB
Python
84 lines
4.0 KiB
Python
"""Build deduplicated AI work items from the residual questions and the seeded KB.
|
|
|
|
Reads docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json, resolves each invalid-NCM product
|
|
question to its KB fingerprint (same product in several databases collapses into ONE
|
|
work item), persists AiWorkItem rows and writes docs/AI_RESIDUAL_PLAN_V1.json.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
|
|
from sqlalchemy import select
|
|
|
|
from central.app.ai.kb import KnowledgeBase
|
|
from central.app.ai.workitems import build_work_items
|
|
from central.app.db import SessionLocal
|
|
from central.app.models import AiWorkItem, KbObservation
|
|
|
|
|
|
def _erp_code_from_question(question) -> str:
|
|
import re
|
|
match = re.search(r"PRODUCT\s+(\d+)", question.get("QUESTION", ""))
|
|
return match.group(1) if match else ""
|
|
|
|
|
|
def main(questions_path="docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json", output="docs/AI_RESIDUAL_PLAN_V1.json") -> dict:
|
|
session = SessionLocal()
|
|
questions = json.loads(Path(questions_path).read_text(encoding="utf8"))["questions"]
|
|
# map "{DATABASE}:{erp_code}" -> fingerprint using the seeded observations
|
|
rows = session.execute(select(KbObservation.company_key, KbObservation.erp_code, KbObservation.kb_product_id, KbObservation.database_label)).all()
|
|
product_ids = {f"{row.database_label}:{row.erp_code}": row.kb_product_id for row in rows}
|
|
questions = [dict(question, DATABASE_CODE=_erp_code_from_question(question)) for question in questions]
|
|
product_fingerprints = {key: f"kb:{value}" for key, value in product_ids.items()}
|
|
items = build_work_items(questions, product_fingerprints)
|
|
raw = sum(len(item.question_refs) for item in items)
|
|
persisted = {"created": 0, "updated": 0, "kb_hits": 0, "conflicts": 0}
|
|
kb = KnowledgeBase(session)
|
|
for item in items:
|
|
row = session.scalar(select(AiWorkItem).where(AiWorkItem.dedup_key == item.dedup_key))
|
|
if row is None:
|
|
row = AiWorkItem(task_type=item.task_type, dedup_key=item.dedup_key, question_refs=item.question_refs, payload=item.payload, cells_covered=item.cells_covered, status="PENDING")
|
|
session.add(row); persisted["created"] += 1
|
|
else:
|
|
row.question_refs = item.question_refs; row.cells_covered = item.cells_covered; row.payload = item.payload
|
|
persisted["updated"] += 1
|
|
# cache-first: an existing non-conflicting validated answer makes this a KB hit, no AI call
|
|
if item.dedup_key.startswith("invalid_ncm:kb:") and item.dedup_key != "invalid_ncm:kb:None":
|
|
kb_product_id = int(item.dedup_key.rsplit(":", 1)[1])
|
|
knowledge = kb.current_knowledge(kb_product_id)
|
|
validated = knowledge.get("validated_ncm", {})
|
|
if validated and not validated.get("conflict"):
|
|
row.status = "KB_HIT"; persisted["kb_hits"] += 1
|
|
elif validated.get("conflict"):
|
|
row.status = "CONFLICT"; persisted["conflicts"] += 1
|
|
session.commit()
|
|
by_task = {}
|
|
for item in items:
|
|
bucket = by_task.setdefault(item.task_type, {"items": 0, "cells": 0})
|
|
bucket["items"] += 1; bucket["cells"] += item.cells_covered
|
|
report = {
|
|
"artifact": "AI_RESIDUAL_PLAN_V1",
|
|
"raw_residual_questions": len(questions),
|
|
"ai_work_items_raw": raw,
|
|
"ai_work_items_deduped": len(items),
|
|
"by_task_type": by_task,
|
|
"persisted": persisted,
|
|
"ai_write_allowed": False,
|
|
"company_decisions_remaining": 14,
|
|
}
|
|
Path(output).write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
session.close()
|
|
return report
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--questions", default="docs/LOCAL_FISCAL_CONTEXT_QUESTIONS_V2.json")
|
|
parser.add_argument("--output", default="docs/AI_RESIDUAL_PLAN_V1.json")
|
|
args = parser.parse_args()
|
|
print(json.dumps(main(args.questions, args.output), indent=2))
|
|
|