"""SCALE test of the Hub side for a ~21k-product installation (synthetic payloads shaped like the real MMMODAS run). python scripts/scale_hub_test.py [--products 21000] [--out docs/runs/scale_hub_test.json] Measures: job-result payload size, Hub persistence (blockers upsert + RTC preview + audits) with SQL statement counts (N+1 check), and the panel pages that read the data (dashboard, agent detail, RTC control, blockers). In-process SQLite: timings are a conservative upper bound for the PostgreSQL production database only for CPU-bound work; statement counts are engine-independent. """ import argparse import logging import json import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from sqlalchemy import event, select # noqa: E402 from sqlalchemy.orm import Session # noqa: E402 from central.tests import test_zero_touch_hub as zt # noqa: E402 logging.disable(logging.CRITICAL) # the Hub logs each job result; at 21k products that is megabytes of noise def main(): parser = argparse.ArgumentParser() parser.add_argument("--products", type=int, default=21000) parser.add_argument("--out", default=str(ROOT / "docs" / "runs" / "scale_hub_test.json")) args = parser.parse_args() n = args.products client, engine = zt._client() counter = {"n": 0} event.listen(engine, "before_cursor_execute", lambda *a, **k: counter.__setitem__("n", counter["n"] + 1)) zt.hb(client) with Session(engine) as s: from central.app.models import Installation s.scalars(select(Installation)).one().rtc_write_mode = "APPROVAL_REQUIRED" s.commit() blocked = [{"company": "", "product_code": f"{i:06d}", "gtin": None, "description": f"PRODUTO DE TESTE {i}", "blocker_type": "BLOCKED_FISCAL_DATA", "field": "RTC", "reason_code": "CCLASSTRIB_MISSING_UNRESOLVED", "evidence_summary": {"reason": "RTC-R4:SUGGESTION_ONLY:000001", "human": "cClassTrib sem regra determinística"}, "input_fingerprint": f"fp{i:08d}"} for i in range(n)] preview = [{"product_code": f"{i:06d}", "description": f"PRODUTO DE TESTE {i}", "ncm": "10063021", "current": "", "proposed": "200003", "cst": "200", "rule_id": "RTC-R3", "basis": "RTC-R3:ANNEX"} for i in range(5000)] operational = {**zt.OPERATIONAL, "TOTAL_PRODUCTS": n, "COMPLETE": 0, "SAFE_BLOCKED": n, "BLOCKED_MISSING_EVIDENCE": n} summary = {"operational": operational, "products_processed": n, "products_changed": n, "blocked_products": blocked, "resolved_products": [], "fiscal_context": {"rtc_enabled": True, "evaluated_all": True, "rtc_resolution": {"total": n, "valid_existing": 0, "ready_to_apply": 5000, "ambiguous": 100, "catalog_incompatible": 0}}, "rtc_auto_apply": {"mode": "APPROVAL_REQUIRED", "ready_total": 5000, "preview": preview, "preview_truncated": False, "applied": [], "aborted": [], "rollbacks": []}} payload = json.dumps({"result_summary": summary}) report = {"products": n, "result_payload_bytes": len(payload), "result_payload_mb": round(len(payload) / 1048576, 2)} job = client.get("/v1/jobs/next", headers=zt.H).json() counter["n"] = 0 t0 = time.perf_counter() response = client.post(f"/v1/jobs/{job['job_id']}/complete", content=payload, headers={**zt.H, "content-type": "application/json"}) report["persist_first_pass"] = {"status": response.status_code, "seconds": round(time.perf_counter() - t0, 2), "sql_statements": counter["n"], "statements_per_product": round(counter["n"] / n, 2)} # a second full evaluation with the same content (steady state: every 30 min per installation) zt.age_last_sync(engine, 4000); zt.hb(client) job = client.get("/v1/jobs/next", headers=zt.H).json() counter["n"] = 0 t0 = time.perf_counter() response = client.post(f"/v1/jobs/{job['job_id']}/complete", content=payload, headers={**zt.H, "content-type": "application/json"}) report["persist_steady_state"] = {"status": response.status_code, "seconds": round(time.perf_counter() - t0, 2), "sql_statements": counter["n"], "statements_per_product": round(counter["n"] / n, 2)} with Session(engine) as s: from central.app.models import BlockedProduct, RtcCandidate report["rows"] = {"blocked": len(s.scalars(select(BlockedProduct.id)).all()), "rtc_candidates": len(s.scalars(select(RtcCandidate.id)).all())} # panel pages (authenticated operator session) from central.tests.test_control_panel_pages import _panel_client pc, agent_id, _ = _panel_client() import central.tests.test_control_panel_pages as cp cp._seed_rtc(pc, agent_id, n=5000) from central.app.blocked_products import upsert_blocker from central.app.models import Installation as Inst with pc._session_factory() as s: inst = s.scalar(select(Inst).where(Inst.agent_id == agent_id)) for b in blocked: upsert_blocker(s, inst.tenant_id, b, installation_id=inst.id) s.commit() pages = {} for name, url in (("dashboard", "/"), ("agent_detail", f"/agents/{agent_id}"), ("rtc_control_p1", f"/agents/{agent_id}/rtc"), ("rtc_control_p50", f"/agents/{agent_id}/rtc?page=50"), ("blockers", "/blockers")): counter["n"] = 0 t0 = time.perf_counter() r = pc.get(url) pages[name] = {"status": r.status_code, "seconds": round(time.perf_counter() - t0, 3), "html_kb": round(len(r.content) / 1024, 1)} report["panel_pages"] = pages Path(args.out).write_text(json.dumps(report, indent=1), encoding="utf-8") print(json.dumps(report)) if __name__ == "__main__": main()