89 lines
5.5 KiB
Python
89 lines
5.5 KiB
Python
"""SCALE test of the Hub side for a ~21k-product installation (synthetic payloads shaped like the real MMMODAS run).
|
|
|
|
python scripts/scale_hub_test.py [--products 21000] [--out docs/runs/scale_hub_test.json]
|
|
|
|
Measures: job-result payload size, Hub persistence (blockers upsert + RTC preview + audits) with SQL statement counts (N+1 check),
|
|
and the panel pages that read the data (dashboard, agent detail, RTC control, blockers). In-process SQLite: timings are a conservative
|
|
upper bound for the PostgreSQL production database only for CPU-bound work; statement counts are engine-independent.
|
|
"""
|
|
import argparse
|
|
import logging
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
from sqlalchemy import event, select # noqa: E402
|
|
from sqlalchemy.orm import Session # noqa: E402
|
|
|
|
from central.tests import test_zero_touch_hub as zt # noqa: E402
|
|
|
|
logging.disable(logging.CRITICAL) # the Hub logs each job result; at 21k products that is megabytes of noise
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--products", type=int, default=21000)
|
|
parser.add_argument("--out", default=str(ROOT / "docs" / "runs" / "scale_hub_test.json"))
|
|
args = parser.parse_args()
|
|
n = args.products
|
|
client, engine = zt._client()
|
|
counter = {"n": 0}
|
|
event.listen(engine, "before_cursor_execute", lambda *a, **k: counter.__setitem__("n", counter["n"] + 1))
|
|
zt.hb(client)
|
|
with Session(engine) as s:
|
|
from central.app.models import Installation
|
|
s.scalars(select(Installation)).one().rtc_write_mode = "APPROVAL_REQUIRED"
|
|
s.commit()
|
|
blocked = [{"company": "", "product_code": f"{i:06d}", "gtin": None, "description": f"PRODUTO DE TESTE {i}", "blocker_type": "BLOCKED_FISCAL_DATA", "field": "RTC", "reason_code": "CCLASSTRIB_MISSING_UNRESOLVED",
|
|
"evidence_summary": {"reason": "RTC-R4:SUGGESTION_ONLY:000001", "human": "cClassTrib sem regra determinística"}, "input_fingerprint": f"fp{i:08d}"} for i in range(n)]
|
|
preview = [{"product_code": f"{i:06d}", "description": f"PRODUTO DE TESTE {i}", "ncm": "10063021", "current": "", "proposed": "200003", "cst": "200", "rule_id": "RTC-R3", "basis": "RTC-R3:ANNEX"} for i in range(5000)]
|
|
operational = {**zt.OPERATIONAL, "TOTAL_PRODUCTS": n, "COMPLETE": 0, "SAFE_BLOCKED": n, "BLOCKED_MISSING_EVIDENCE": n}
|
|
summary = {"operational": operational, "products_processed": n, "products_changed": n, "blocked_products": blocked, "resolved_products": [],
|
|
"fiscal_context": {"rtc_enabled": True, "evaluated_all": True, "rtc_resolution": {"total": n, "valid_existing": 0, "ready_to_apply": 5000, "ambiguous": 100, "catalog_incompatible": 0}},
|
|
"rtc_auto_apply": {"mode": "APPROVAL_REQUIRED", "ready_total": 5000, "preview": preview, "preview_truncated": False, "applied": [], "aborted": [], "rollbacks": []}}
|
|
payload = json.dumps({"result_summary": summary})
|
|
report = {"products": n, "result_payload_bytes": len(payload), "result_payload_mb": round(len(payload) / 1048576, 2)}
|
|
job = client.get("/v1/jobs/next", headers=zt.H).json()
|
|
counter["n"] = 0
|
|
t0 = time.perf_counter()
|
|
response = client.post(f"/v1/jobs/{job['job_id']}/complete", content=payload, headers={**zt.H, "content-type": "application/json"})
|
|
report["persist_first_pass"] = {"status": response.status_code, "seconds": round(time.perf_counter() - t0, 2), "sql_statements": counter["n"], "statements_per_product": round(counter["n"] / n, 2)}
|
|
# a second full evaluation with the same content (steady state: every 30 min per installation)
|
|
zt.age_last_sync(engine, 4000); zt.hb(client)
|
|
job = client.get("/v1/jobs/next", headers=zt.H).json()
|
|
counter["n"] = 0
|
|
t0 = time.perf_counter()
|
|
response = client.post(f"/v1/jobs/{job['job_id']}/complete", content=payload, headers={**zt.H, "content-type": "application/json"})
|
|
report["persist_steady_state"] = {"status": response.status_code, "seconds": round(time.perf_counter() - t0, 2), "sql_statements": counter["n"], "statements_per_product": round(counter["n"] / n, 2)}
|
|
with Session(engine) as s:
|
|
from central.app.models import BlockedProduct, RtcCandidate
|
|
report["rows"] = {"blocked": len(s.scalars(select(BlockedProduct.id)).all()), "rtc_candidates": len(s.scalars(select(RtcCandidate.id)).all())}
|
|
# panel pages (authenticated operator session)
|
|
from central.tests.test_control_panel_pages import _panel_client
|
|
pc, agent_id, _ = _panel_client()
|
|
import central.tests.test_control_panel_pages as cp
|
|
cp._seed_rtc(pc, agent_id, n=5000)
|
|
from central.app.blocked_products import upsert_blocker
|
|
from central.app.models import Installation as Inst
|
|
with pc._session_factory() as s:
|
|
inst = s.scalar(select(Inst).where(Inst.agent_id == agent_id))
|
|
for b in blocked:
|
|
upsert_blocker(s, inst.tenant_id, b, installation_id=inst.id)
|
|
s.commit()
|
|
pages = {}
|
|
for name, url in (("dashboard", "/"), ("agent_detail", f"/agents/{agent_id}"), ("rtc_control_p1", f"/agents/{agent_id}/rtc"), ("rtc_control_p50", f"/agents/{agent_id}/rtc?page=50"), ("blockers", "/blockers")):
|
|
counter["n"] = 0
|
|
t0 = time.perf_counter()
|
|
r = pc.get(url)
|
|
pages[name] = {"status": r.status_code, "seconds": round(time.perf_counter() - t0, 3), "html_kb": round(len(r.content) / 1024, 1)}
|
|
report["panel_pages"] = pages
|
|
Path(args.out).write_text(json.dumps(report, indent=1), encoding="utf-8")
|
|
print(json.dumps(report))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|