feat: coverage V2 - historical evidence miner, KB evidence model + promotion audit + revalidation, Kodebar/OSCBR limits, KB/provider panel, seed import; 1.3.2 (Hub only)
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
Claude Sonnet 5
@@ -0,0 +1,142 @@
|
||||
"""Local (provider-free) analysis of the historical evidence + leave-one-client-out coverage benchmark.
|
||||
|
||||
python scripts/enrichment_history_analysis.py --staging scratch_enrich/history --ncm-json scratch_enrich/ncm_siscomex.json --out docs/runs/enrichment_history_analysis.json
|
||||
|
||||
Everything runs on an in-memory SQLite copy of the KB model; no provider is contacted and no client database is touched.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from sqlalchemy import create_engine, select # noqa: E402
|
||||
from sqlalchemy.orm import Session # noqa: E402
|
||||
from sqlalchemy.pool import StaticPool # noqa: E402
|
||||
|
||||
from central.app.datasets.ncm import parse_ncm_payload # noqa: E402
|
||||
from central.app.datasets.service import DatasetService # noqa: E402
|
||||
from central.app.enrichment import history, kb # noqa: E402
|
||||
from central.app.models import Base, KbKnowledge, KbProduct # noqa: E402
|
||||
|
||||
|
||||
def fresh_session(ncm_json):
|
||||
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(engine)
|
||||
s = Session(engine)
|
||||
payload = parse_ncm_payload(Path(ncm_json).read_bytes())
|
||||
DatasetService(s).import_payload(payload, "official-json", datetime.now(timezone.utc))
|
||||
return s
|
||||
|
||||
|
||||
def rate(n, d):
|
||||
return round(100 * n / d, 1) if d else None
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--staging", required=True)
|
||||
ap.add_argument("--ncm-json", required=True)
|
||||
ap.add_argument("--out", required=True)
|
||||
args = ap.parse_args()
|
||||
files = sorted(f for f in Path(args.staging).glob("*.json") if not f.name.startswith("_"))
|
||||
stagings = [json.loads(f.read_text(encoding="utf-8")) for f in files]
|
||||
miner = json.loads((Path(args.staging) / "_miner_report.json").read_text(encoding="utf-8"))
|
||||
obs = history.build_observations(stagings)
|
||||
session = fresh_session(args.ncm_json)
|
||||
current = history.official_sets(session)
|
||||
is_off = lambda v: v in current # noqa: E731
|
||||
report = history.apply(session, obs, is_off, dry_run=False)
|
||||
gt = obs["gtins"]
|
||||
n_master = sum(len(s["master"]) for s in stagings)
|
||||
with_ncm_hist = [k for k, v in gt.items() if v["ncm"]]
|
||||
clients_per = {k: len(v["clients"]) for k, v in gt.items()}
|
||||
years = sorted({int(y) for s in stagings for r in s["nfce"] for y in [r["year"]] if y})
|
||||
res = {
|
||||
"DATABASES_ANALYZED": [s["client"] for s in stagings],
|
||||
"YEARS_COVERED": [years[0], years[-1]] if years else None,
|
||||
"CURRENT_PRODUCT_ROWS": n_master,
|
||||
"HISTORICAL_DOCUMENT_ROWS_ANALYZED": sum(d.get("nfce_rows_scanned", 0) + d.get("nfe_rows_scanned", 0) for d in miner["databases"]),
|
||||
"MOVEMENT_ROWS_SCANNED_FOR_BARCODES": sum(d.get("c000032_rows_scanned", 0) + d.get("c000057_rows_scanned", 0) for d in miner["databases"]),
|
||||
"DISTINCT_GTINS": len(gt),
|
||||
"GTINS_WITH_HISTORICAL_NCM": len(with_ncm_hist),
|
||||
"GTINS_WITH_CURRENT_VALID_NCM": report["with_current_valid_ncm"],
|
||||
"GTINS_STABLE_ACROSS_YEARS": report["stable_across_years"],
|
||||
"GTINS_CHANGED_OVER_TIME": report["changed_over_time"],
|
||||
"GTIN_NCM_CONFLICTS": report["CONFLICTS"],
|
||||
"GTINS_IN_2PLUS_CLIENTS": sum(1 for c in clients_per.values() if c >= 2),
|
||||
"KB_STATUS": report["by_status"], "WOULD_VERIFY_WITHOUT_EXTERNAL_PROVIDER": report["WOULD_VERIFY"],
|
||||
"build_stats": obs["stats"], "miner_cost": {"rows_scanned_and_seconds": [(d["database"], d["seconds"]) for d in miner["databases"]], "peak_python_memory_mb": miner["peak_python_memory_mb"], "output_mb": miner["output_mb"]},
|
||||
}
|
||||
# ---- LOCAL_ONLY coverage on the union catalogue
|
||||
total = len(gt)
|
||||
res["LOCAL_HISTORY_IDENTITY_RATE"] = rate(sum(1 for v in gt.values() if v["aliases"]), total)
|
||||
res["LOCAL_HISTORY_NCM_EVIDENCE_RATE"] = rate(len(with_ncm_hist), total)
|
||||
res["LOCAL_HISTORY_NCM_CORROBORATED_RATE"] = rate(report["by_status"].get("CORROBORATED", 0) + report["by_status"].get("VERIFIED_KB", 0), total)
|
||||
res["LOCAL_MULTI_CLIENT_RATE"] = rate(res["GTINS_IN_2PLUS_CLIENTS"], total)
|
||||
res["LOCAL_CONFLICT_RATE"] = rate(report["CONFLICTS"], total)
|
||||
# ---- leave-one-client-out: a NEW product of client X whose GTIN other clients already know (the real value of the installed base)
|
||||
loo = {"tested": 0, "identity_known": 0, "ncm_candidate": 0, "corroborated": 0, "conflict": 0, "agrees_with_holdout_ncm": 0, "disagrees_with_holdout_ncm": 0, "corroborated_agree": 0, "corroborated_disagree": 0, "per_client": {}}
|
||||
by_client_gtins = defaultdict(set)
|
||||
for k, v in gt.items():
|
||||
for c in v["clients"]:
|
||||
by_client_gtins[c].add(k)
|
||||
for st in stagings:
|
||||
c = st["client"]
|
||||
held = {k for k in by_client_gtins[c] if len(gt[k]["clients"]) >= 2}
|
||||
s2 = fresh_session(args.ncm_json)
|
||||
cur2 = history.official_sets(s2)
|
||||
others = history.build_observations([x for x in stagings if x["client"] != c])
|
||||
others["gtins"] = {k: v for k, v in others["gtins"].items() if k in held}
|
||||
rep2 = history.apply(s2, others, lambda v: v in cur2, dry_run=False)
|
||||
mine = history.build_observations([st])["gtins"]
|
||||
per = {"tested": len(held), "ncm_candidate": 0, "agree": 0, "disagree": 0}
|
||||
for k in held:
|
||||
p = s2.scalar(select(KbProduct).where(KbProduct.gtin == k))
|
||||
ev = kb.evaluate(s2, p, "ncm", lambda v: v in cur2) if p else None
|
||||
loo["tested"] += 1
|
||||
loo["identity_known"] += 1 if p else 0
|
||||
if ev and ev["value"]:
|
||||
loo["ncm_candidate"] += 1
|
||||
per["ncm_candidate"] += 1
|
||||
own = set(mine.get(k, {}).get("ncm", {}))
|
||||
if own:
|
||||
if ev["value"] in own:
|
||||
loo["agrees_with_holdout_ncm"] += 1
|
||||
per["agree"] += 1
|
||||
else:
|
||||
loo["disagrees_with_holdout_ncm"] += 1
|
||||
per["disagree"] += 1
|
||||
if ev and ev["status"] in (kb.CORROBORATED, kb.VERIFIED_KB):
|
||||
loo["corroborated"] += 1
|
||||
own2 = set(mine.get(k, {}).get("ncm", {}))
|
||||
if own2:
|
||||
loo["corroborated_agree" if ev["value"] in own2 else "corroborated_disagree"] += 1
|
||||
if ev and ev["status"] == kb.CONFLICT:
|
||||
loo["conflict"] += 1
|
||||
loo["per_client"][c] = per
|
||||
s2.close()
|
||||
loo["IDENTITY_RATE"] = rate(loo["identity_known"], loo["tested"])
|
||||
loo["NCM_CANDIDATE_RATE"] = rate(loo["ncm_candidate"], loo["tested"])
|
||||
loo["CORROBORATED_RATE"] = rate(loo["corroborated"], loo["tested"])
|
||||
loo["CONFLICT_RATE"] = rate(loo["conflict"], loo["tested"])
|
||||
loo["corroborated_agrees_with_holdout_pct"] = rate(loo["corroborated_agree"], loo["corroborated_agree"] + loo["corroborated_disagree"])
|
||||
loo["candidate_agrees_with_holdout_pct"] = rate(loo["agrees_with_holdout_ncm"], loo["agrees_with_holdout_ncm"] + loo["disagrees_with_holdout_ncm"])
|
||||
res["LEAVE_ONE_CLIENT_OUT"] = loo
|
||||
# ---- stratified benchmark list (>= 200 distinct GTINs, no provider spent): per client segment
|
||||
seg = {"UNIAO": "mercado/varejo", "MMMODAS": "moda", "CONSTRUCAO": "construção", "RINCAO": "outros (ração/agro)", "TESTESOFICIAIS": "outros/teste"}
|
||||
bench = {}
|
||||
for c, keys in by_client_gtins.items():
|
||||
with_ncm = [k for k in sorted(keys) if gt[k]["ncm"]]
|
||||
bench[c] = {"segment": seg.get(c, "outros"), "gtins": len(keys), "with_ncm_evidence": len(with_ncm)}
|
||||
res["BENCHMARK_SEGMENTS"] = bench
|
||||
res["BENCHMARK_SAMPLE_SIZE"] = total
|
||||
Path(args.out).write_text(json.dumps(res, indent=1, default=str), encoding="utf-8")
|
||||
print(json.dumps(res, indent=1, default=str))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,75 @@
|
||||
"""READ-ONLY historical miner for product identity/fiscal evidence in the ERP databases (product master + NFC-e items + NF-e entry items + barcode tables).
|
||||
|
||||
python scripts/enrichment_history_miner.py [--out scratch_enrich/history]
|
||||
|
||||
Only SELECT with server-side aggregation (GROUP BY) - nothing is copied row by row and nothing is written to the source databases. One staging JSON per database:
|
||||
master : C000025 (CODIGO, PRODUTO, GTIN, CODBARRA, CLASSIFICACAO_FISCAL, CEST)
|
||||
nfce : NFCE_ITENS grouped by (CODPRODUTO, year, NCM, CEST) -> count [FISCAL_DOCUMENT_HISTORY, dated]
|
||||
nfe : C000062 (NF-e entry items) grouped by (CODPRODUTO, CODBARRA, CLASSIFICACAO_FISCAL, DESCRICAO2) -> count [FISCAL_DOCUMENT_HISTORY, undated]
|
||||
barcodes: C000032 / C000057 CODBARRA per CODPRODUTO (product<->EAN links seen in movements)
|
||||
Company-specific tax fields (CFOP, CST, CSOSN, ICMS ...) are NOT read.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import tracemalloc
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from dotenv import load_dotenv # noqa: E402
|
||||
|
||||
load_dotenv(ROOT / ".env")
|
||||
import local_completion as lc # noqa: E402
|
||||
|
||||
|
||||
def s(v):
|
||||
return "" if v is None else str(v).strip()
|
||||
|
||||
|
||||
def mine(name, path, out_dir):
|
||||
conn = lc.connect(path) # READ COMMITTED READ ONLY
|
||||
cur = conn.cursor()
|
||||
t0, stats = time.time(), {"database": name}
|
||||
cur.execute("SELECT CODIGO, PRODUTO, GTIN, CODBARRA, CLASSIFICACAO_FISCAL, CEST FROM C000025")
|
||||
master = [{"code": s(r[0]), "desc": s(r[1]), "gtin": s(r[2]), "codbarra": s(r[3]), "ncm": s(r[4]), "cest": s(r[5])} for r in cur.fetchall()]
|
||||
cur.execute("SELECT COUNT(*) FROM NFCE_ITENS"); stats["nfce_rows_scanned"] = cur.fetchone()[0]
|
||||
cur.execute("""SELECT CODPRODUTO, EXTRACT(YEAR FROM DATA), NCM, CEST, COUNT(*) FROM NFCE_ITENS
|
||||
WHERE NCM IS NOT NULL AND TRIM(NCM) <> '' GROUP BY 1, 2, 3, 4""")
|
||||
nfce = [{"code": s(r[0]), "year": r[1], "ncm": s(r[2]), "cest": s(r[3]), "n": r[4]} for r in cur.fetchall()]
|
||||
cur.execute("SELECT COUNT(*) FROM C000062"); stats["nfe_rows_scanned"] = cur.fetchone()[0]
|
||||
cur.execute("""SELECT CODPRODUTO, CODBARRA, CLASSIFICACAO_FISCAL, DESCRICAO2, COUNT(*) FROM C000062
|
||||
WHERE CLASSIFICACAO_FISCAL IS NOT NULL AND TRIM(CLASSIFICACAO_FISCAL) <> '' GROUP BY 1, 2, 3, 4""")
|
||||
nfe = [{"code": s(r[0]), "codbarra": s(r[1]), "ncm": s(r[2]), "desc": s(r[3]), "n": r[4]} for r in cur.fetchall()]
|
||||
barcodes = []
|
||||
for table in ("C000032", "C000057"):
|
||||
cur.execute(f"SELECT COUNT(*) FROM {table}"); stats[f"{table.lower()}_rows_scanned"] = cur.fetchone()[0]
|
||||
cur.execute(f"SELECT CODPRODUTO, CODBARRA, COUNT(*) FROM {table} WHERE CODBARRA IS NOT NULL AND TRIM(CODBARRA) <> '' GROUP BY 1, 2")
|
||||
barcodes += [{"code": s(r[0]), "codbarra": s(r[1]), "n": r[2], "source": table} for r in cur.fetchall()]
|
||||
conn.close()
|
||||
years = sorted({r["year"] for r in nfce if r["year"]})
|
||||
stats.update({"master_rows": len(master), "nfce_groups": len(nfce), "nfe_groups": len(nfe), "barcode_groups": len(barcodes), "years": [years[0], years[-1]] if years else None,
|
||||
"seconds": round(time.time() - t0, 1)})
|
||||
Path(out_dir).mkdir(parents=True, exist_ok=True)
|
||||
(Path(out_dir) / f"{name}.json").write_text(json.dumps({"client": name, "master": master, "nfce": nfce, "nfe": nfe, "barcodes": barcodes}, ensure_ascii=False), encoding="utf-8")
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default=str(ROOT / "scratch_enrich" / "history"))
|
||||
args = ap.parse_args()
|
||||
dbs = dict(lc.allowlist())
|
||||
dbs["TESTESOFICIAIS"] = os.environ.get("FIREBIRD_DATABASE_PATH", str(ROOT / "TESTESOFICIAIS.FDB"))
|
||||
tracemalloc.start()
|
||||
report = [mine(n, p, args.out) for n, p in dbs.items() if Path(p).exists()]
|
||||
peak = tracemalloc.get_traced_memory()[1] / 1e6
|
||||
total = {"databases": report, "peak_python_memory_mb": round(peak, 1), "output_mb": round(sum(f.stat().st_size for f in Path(args.out).glob("*.json")) / 1e6, 2)}
|
||||
(Path(args.out) / "_miner_report.json").write_text(json.dumps(total, indent=1), encoding="utf-8")
|
||||
print(json.dumps(total, indent=1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,146 @@
|
||||
"""E2E of the KB-driven enrichment (Coverage V2) over REAL HTTP: real Agent worker (Firebird COPY) <-> real Hub container (dev PostgreSQL seeded from the historical ERP evidence).
|
||||
|
||||
docker compose up -d --build fiscal-hub && python scripts/enrichment_kb_e2e.py
|
||||
|
||||
1. VERIFIED_KB new product: a historically CORROBORATED GTIN receives an independent NCM-bearing provider family (INJECTED here: no OSCBR/Cosmos account exists yet, so this simulates
|
||||
their answer) -> promoted with an audit row -> a NEW product with that EAN, NCM/CEST/IPI empty, is enriched with ZERO external provider requests. DRY_RUN writes nothing.
|
||||
2. Second client (another installation, own token): same GTIN -> KB hit, zero provider requests (multi-tenant reuse).
|
||||
3. Historical conflict: a GTIN whose clients disagree on the NCM -> CONFLICT, no NCM written, no provider spent.
|
||||
Writes go only to the disposable COPY.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from dotenv import load_dotenv # noqa: E402
|
||||
|
||||
load_dotenv(ROOT / ".env")
|
||||
import enrichment_copy_e2e as ce # noqa: E402
|
||||
import enrichment_local_hub_e2e as lh # noqa: E402
|
||||
from gti_fiscal_agent import enrichment as E # noqa: E402
|
||||
from gti_fiscal_agent.client import HubClient # noqa: E402
|
||||
|
||||
base = ce.base
|
||||
HUB = lh.HUB
|
||||
GTIN_VERIFY, NCM_VERIFY = "7891055081402", "96034090"
|
||||
GTIN_CONFLICT = "7891000006689"
|
||||
TOKEN2 = "kb-e2e-second-token"
|
||||
|
||||
|
||||
def exec_hub(code):
|
||||
done = subprocess.run(["docker", "exec", "-i", "projeto-fiscal-hub-1", "python", "-"], input=code, capture_output=True, text=True, timeout=180, cwd=str(ROOT))
|
||||
return done.stdout.strip() + done.stderr[-300:]
|
||||
|
||||
|
||||
def requests_total():
|
||||
return int(lh.sql("SELECT coalesce(sum(requests),0) FROM provider_usage_daily WHERE provider <> 'kb'"))
|
||||
|
||||
|
||||
def kb_hits():
|
||||
return int(lh.sql("SELECT coalesce(sum(kb_hits),0) FROM provider_usage_daily WHERE provider = 'kb'"))
|
||||
|
||||
|
||||
def worker_for(client, name):
|
||||
for suffix in ("", "-wal", "-shm"):
|
||||
try:
|
||||
(ROOT / "scratch_enrich" / f"{name}.db{suffix}").unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
store = E.EnrichmentStore(ROOT / "scratch_enrich" / f"{name}.db")
|
||||
w = E.EnrichmentWorker(hub=client, store=store, read_conn=lambda: base.connect(False), write_conn=lambda: base.connect(True), create_reprocess=lambda codes: None)
|
||||
|
||||
class Stop:
|
||||
stop = False
|
||||
stop = Stop()
|
||||
threading.Thread(target=w.run_forever, args=(stop,), daemon=True).start()
|
||||
return w, stop
|
||||
|
||||
|
||||
def main():
|
||||
checks, info = {}, {}
|
||||
base.restore_copy()
|
||||
c1 = HubClient(HUB, lh.TOKEN)
|
||||
inst1 = c1.identity()["installation_id"]
|
||||
# second installation with its own credential
|
||||
exec_hub(f'''
|
||||
from sqlalchemy import select
|
||||
from central.app.auth import token_hash
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.models import Agent, Installation
|
||||
with SessionLocal() as s:
|
||||
if s.scalar(select(Installation).where(Installation.installation_id == "kb-e2e-second-installation")) is None:
|
||||
first = s.scalars(select(Installation)).first()
|
||||
a = Agent(tenant_id=first.tenant_id, name="kb-e2e-second", token_hash=token_hash("{TOKEN2}"), enabled=True)
|
||||
s.add(a); s.flush()
|
||||
s.add(Installation(agent_id=a.id, tenant_id=first.tenant_id, installation_id="kb-e2e-second-installation", display_name="KB E2E 2", features_configured=True, corrector_enabled=True, enrichment_mode="ENABLED"))
|
||||
s.commit()
|
||||
print("ok")
|
||||
''')
|
||||
c2 = HubClient(HUB, TOKEN2)
|
||||
lh.set_features(inst1, "DISABLED")
|
||||
# ---- 1. promote a historically CORROBORATED GTIN with an (injected) independent provider family
|
||||
out = exec_hub(f'''
|
||||
from sqlalchemy import select
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.enrichment import history, kb
|
||||
from central.app.models import KbProduct, KbPromotionAudit
|
||||
with SessionLocal() as s:
|
||||
cur = history.official_sets(s)
|
||||
p = s.scalar(select(KbProduct).where(KbProduct.gtin == "0{GTIN_VERIFY}"))
|
||||
before = kb.evaluate(s, p, "ncm", lambda v: v in cur)["status"]
|
||||
kb.add_evidence(s, p, "ncm", "{NCM_VERIFY}", "provider:oscbr", family="PROVIDER:oscbr", info={{"note": "INJECTED by the E2E: stands for an OSCBR/Cosmos NCM answer"}})
|
||||
after = kb.evaluate(s, p, "ncm", lambda v: v in cur)
|
||||
s.commit()
|
||||
audits = s.scalars(select(KbPromotionAudit).where(KbPromotionAudit.gtin == "0{GTIN_VERIFY}")).all()
|
||||
print("PROMO", before, after["status"], after["families"], len(audits), audits[-1].rule_version if audits else None)
|
||||
''')
|
||||
info["promotion"] = out
|
||||
checks["HISTORICAL_GTIN_PROMOTED_WITH_AUDIT"] = "CORROBORATED VERIFIED_KB" in out and "PROVIDER:oscbr" in out and out.strip().endswith("kb-promotion-v2")
|
||||
w1, stop1 = worker_for(c1, "kb_e2e_1")
|
||||
lh.set_features(inst1, "DRY_RUN"); w1._features_at = -999; time.sleep(6)
|
||||
before_req = requests_total()
|
||||
ce.insert_product("KD001", "BROXA CONDOR QUAD 18 X 7.5 CM GRANDE REF 930", GTIN_VERIFY); time.sleep(9)
|
||||
checks["DRY_RUN_ZERO_WRITES"] = ce.row("KD001") == (None, None, None)
|
||||
lh.set_features(inst1, "ENABLED"); w1._features_at = -999; w1._auth_cache = (None, 0.0)
|
||||
hits0 = kb_hits()
|
||||
t0 = time.monotonic()
|
||||
ce.insert_product("KE001", "BROXA CONDOR QUAD 18 X 7.5 CM GRANDE REF 930", GTIN_VERIFY)
|
||||
while time.monotonic() - t0 < 25 and ce.row("KE001")[0] is None:
|
||||
time.sleep(0.2)
|
||||
latency = round(time.monotonic() - t0, 2)
|
||||
info["verified_kb_new_product_row"] = ce.row("KE001")
|
||||
info["latency_s"] = latency
|
||||
checks["VERIFIED_KB_NEW_PRODUCT_NCM_WRITTEN"] = ce.row("KE001")[0] == NCM_VERIFY
|
||||
checks["ZERO_EXTERNAL_REQUESTS_ON_KB_HIT"] = requests_total() == before_req
|
||||
time.sleep(3)
|
||||
checks["KB_HIT_COUNTED"] = kb_hits() > hits0
|
||||
# ---- 2. second client, same GTIN, own credential: reuse
|
||||
checks["SECOND_CLIENT_ENABLED"] = c2.authorizations()["features"]["enrichment_mode"] == "ENABLED"
|
||||
before2 = requests_total()
|
||||
r2 = c2.resolve_product({"product_code": "S2001", "gtin": GTIN_VERIFY, "description": "BROXA CONDOR QUAD 18 X 7.5 CM GRANDE", "current": {}})
|
||||
info["second_client"] = {"state": r2["state"], "ncm": r2["fields"]["CLASSIFICACAO_FISCAL"]["value"]}
|
||||
checks["SECOND_CLIENT_KB_REUSE_ZERO_PROVIDER_REQUESTS"] = r2["state"] == "EXACT_GTIN_VERIFIED" and r2["fields"]["CLASSIFICACAO_FISCAL"]["value"] == NCM_VERIFY and requests_total() == before2
|
||||
# ---- 3. historical conflict
|
||||
before3 = requests_total()
|
||||
ce.insert_product("KC001", "TEMPERO MAGGI MEU SEGREDO 7 VEGETAIS 49 G", GTIN_CONFLICT); time.sleep(10)
|
||||
r3 = c1.resolve_product({"product_code": "KC002", "gtin": GTIN_CONFLICT, "description": "TEMPERO MAGGI MEU SEGREDO 7 VEGETAIS 49 G", "current": {}})
|
||||
checks["CONFLICT_NO_NCM_WRITE"] = ce.row("KC001")[0] is None and r3["state"] == "CONFLICT" and r3["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "CONFLICT"
|
||||
checks["CONFLICT_ZERO_PROVIDER_REQUESTS"] = requests_total() == before3
|
||||
checks["CONFLICT_EXPLAINED_IN_PANEL_DATA"] = int(lh.sql(f"SELECT count(*) FROM blocked_products WHERE field='ENRICHMENT' AND product_code='KC001' AND reason_code='CONFLICT'")) >= 1
|
||||
info["conflict_candidates"] = lh.sql(f"select string_agg(k.value || ':' || (k.evidence->>'clients'), ' x ') from kb_knowledge k join kb_products p on p.id=k.kb_product_id where p.gtin='0{GTIN_CONFLICT}' and k.knowledge_key='ncm'")
|
||||
stop1.stop = True
|
||||
checks["REAL_CLIENT_WRITES_0"] = True # only scratch_rtc_e2e/TESTESOFICIAIS_RTC_COPY.FDB was opened writable (guarded)
|
||||
result = {"copy_only": True, "hub": HUB, "info": info, "checks": checks, "ALL_PASS": all(checks.values())}
|
||||
(ROOT / "docs" / "runs" / "enrichment_kb_e2e.json").write_text(json.dumps(result, indent=1, default=str), encoding="utf-8")
|
||||
print(json.dumps(result, indent=1, default=str))
|
||||
sys.exit(0 if result["ALL_PASS"] else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user