"""Backpressure: at most HEAVY_JOB_CONCURRENCY_LIMIT whole-database jobs run at once across all Agents; the rest queue as PENDING.""" from types import SimpleNamespace from sqlalchemy import select from central.app import jobs from central.app.models import Agent, Job, Tenant from central.tests.test_control_plane import setup_db def _agents(session, n): tenant = Tenant(name="Backpressure") session.add(tenant) session.flush() out = [] for i in range(n): agent = Agent(tenant_id=tenant.id, name=f"a{i}", token_hash=f"t{i}".ljust(64, "x"), agent_version="1.2.1") session.add(agent) out.append(agent) session.flush() return out def _limit(monkeypatch, value): monkeypatch.setattr(jobs, "settings", SimpleNamespace(heavy_job_concurrency_limit=value)) def test_only_the_configured_number_of_full_jobs_run_at_once_and_the_rest_wait(monkeypatch): _limit(monkeypatch, 2) with setup_db()() as session: agents = _agents(session, 4) queued = [jobs.create_job(session, a, "INITIAL_SYNC") for a in agents] claimed = [jobs.claim_next_job(session, a) for a in agents] assert [c is not None for c in claimed] == [True, True, False, False] # 2 running, 2 queued (backpressure) assert [session.get(Job, q.id).status for q in queued] == ["RUNNING", "RUNNING", "PENDING", "PENDING"] jobs.complete_job(session, session.get(Job, queued[0].id), {}) # a slot frees up assert jobs.claim_next_job(session, agents[2]) is not None and jobs.claim_next_job(session, agents[3]) is None def test_a_light_incremental_job_is_never_held_behind_a_queued_full_job(monkeypatch): _limit(monkeypatch, 1) with setup_db()() as session: busy, waiting = _agents(session, 2) jobs.create_job(session, busy, "REPROCESS_ALL") assert jobs.claim_next_job(session, busy) is not None full = jobs.create_job(session, waiting, "REPROCESS_ALL") light = jobs.create_job(session, waiting, "REPROCESS_PRODUCTS", {"codigos": ["1"]}) claimed = jobs.claim_next_job(session, waiting) assert claimed is not None and claimed.id == light.id and session.get(Job, full.id).status == "PENDING" def test_limit_zero_disables_backpressure_and_a_stale_running_job_does_not_hold_a_slot(monkeypatch): from datetime import datetime, timedelta, timezone _limit(monkeypatch, 0) with setup_db()() as session: agents = _agents(session, 3) for a in agents: jobs.create_job(session, a, "INITIAL_SYNC") assert all(jobs.claim_next_job(session, a) is not None for a in agents) _limit(monkeypatch, 1) with setup_db()() as session: dead, fresh = _agents(session, 2) stale = jobs.create_job(session, dead, "INITIAL_SYNC") session.execute(Job.__table__.update().where(Job.id == stale.id).values(status="RUNNING", started_at=datetime.now(timezone.utc) - timedelta(hours=3))) session.commit() jobs.create_job(session, fresh, "INITIAL_SYNC") assert jobs.claim_next_job(session, fresh) is not None # the dead Agent's 3 h old RUNNING job does not block the queue