"""Controlled concurrency simulation of whole-database classification against the LOCAL dev Hub. N simulated Agents each classify M products (read-only Firebird sample + Hub /classify, exactly the Agent's HubClient path). UNLIMITED = all N at once; LIMITED = only K at a time (what HEAVY_JOB_CONCURRENCY_LIMIT enforces at job claim). Samples the Hub container CPU/RAM (docker stats). python scripts/concurrency_sim.py [--agents 8] [--products 2084] [--limit 3]""" import argparse import json import os import subprocess import sys import threading import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path[:0] = [str(ROOT), str(ROOT / "scripts"), str(ROOT / "agent" / "src")] from dotenv import load_dotenv # noqa: E402 load_dotenv(ROOT / ".env") import local_completion as lc # noqa: E402 from gti_fiscal_agent import firebird_reader # noqa: E402 from gti_fiscal_agent.client import HubClient # noqa: E402 from gti_fiscal_agent.runner import to_hub_product # noqa: E402 def sampler(stop, samples): while not stop.is_set(): out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.CPUPerc}}|{{.MemUsage}}", "projeto-fiscal-hub-1"], capture_output=True, text=True).stdout.strip() try: cpu, mem = out.split("|"); samples.append((float(cpu.strip("%")), float(mem.split("/")[0].strip().replace("MiB", "").replace("GiB", "")) )) except Exception: # noqa: BLE001 pass time.sleep(1) def run(agents, products, limit): gate = threading.Semaphore(limit if limit else agents) results, samples, stop = [], [], threading.Event() threading.Thread(target=sampler, args=(stop, samples), daemon=True).start() def one(i): queued = time.time() with gate: started = time.time() try: client = HubClient(os.environ.get("HUB_URL", lc.HUB), os.environ["AGENT_DEV_TOKEN"]) client.classify_batch(products) return {"agent": i, "queue_wait_s": round(started - queued, 1), "run_s": round(time.time() - started, 1), "ok": True} except Exception as error: # noqa: BLE001 return {"agent": i, "queue_wait_s": round(started - queued, 1), "run_s": round(time.time() - started, 1), "ok": False, "error": type(error).__name__} t0 = time.time() with ThreadPoolExecutor(agents) as pool: results = list(pool.map(one, range(agents))) wall = time.time() - t0 stop.set() cpus = [c for c, _ in samples] or [0]; mems = [m for _, m in samples] or [0] return {"limit": limit or "none", "wall_s": round(wall, 1), "products_per_s": round(agents * len(products) / wall, 1), "failures": sum(1 for r in results if not r["ok"]), "max_run_s": max(r["run_s"] for r in results), "max_queue_wait_s": max(r["queue_wait_s"] for r in results), "hub_cpu_peak_pct": max(cpus), "hub_cpu_avg_pct": round(sum(cpus) / len(cpus), 1), "hub_mem_peak_mib": max(mems)} def main(): ap = argparse.ArgumentParser(); ap.add_argument("--agents", type=int, default=8); ap.add_argument("--products", type=int, default=2084); ap.add_argument("--limit", type=int, default=3) a = ap.parse_args() env = lc.env_for(lc.allowlist()["TESTESOFICIAIS"]) base = firebird_reader.read_products(env) sample = [to_hub_product(p) for p in (base * (a.products // len(base) + 1))[:a.products]] out = {"agents": a.agents, "products_per_agent": a.products, "runs": [run(a.agents, sample, 0), run(a.agents, sample, a.limit)]} print(json.dumps(out, indent=1)) (ROOT / "docs" / "runs" / "final_corpus" / "concurrency_sim.json").write_text(json.dumps(out, indent=1), encoding="utf-8") if __name__ == "__main__": main()