68 lines
3.2 KiB
Python
68 lines
3.2 KiB
Python
"""Backpressure: at most HEAVY_JOB_CONCURRENCY_LIMIT whole-database jobs run at once across all Agents; the rest queue as PENDING."""
|
|
from types import SimpleNamespace
|
|
|
|
from sqlalchemy import select
|
|
|
|
from central.app import jobs
|
|
from central.app.models import Agent, Job, Tenant
|
|
from central.tests.test_control_plane import setup_db
|
|
|
|
|
|
def _agents(session, n):
|
|
tenant = Tenant(name="Backpressure")
|
|
session.add(tenant)
|
|
session.flush()
|
|
out = []
|
|
for i in range(n):
|
|
agent = Agent(tenant_id=tenant.id, name=f"a{i}", token_hash=f"t{i}".ljust(64, "x"), agent_version="1.2.1")
|
|
session.add(agent)
|
|
out.append(agent)
|
|
session.flush()
|
|
return out
|
|
|
|
|
|
def _limit(monkeypatch, value):
|
|
monkeypatch.setattr(jobs, "settings", SimpleNamespace(heavy_job_concurrency_limit=value))
|
|
|
|
|
|
def test_only_the_configured_number_of_full_jobs_run_at_once_and_the_rest_wait(monkeypatch):
|
|
_limit(monkeypatch, 2)
|
|
with setup_db()() as session:
|
|
agents = _agents(session, 4)
|
|
queued = [jobs.create_job(session, a, "INITIAL_SYNC") for a in agents]
|
|
claimed = [jobs.claim_next_job(session, a) for a in agents]
|
|
assert [c is not None for c in claimed] == [True, True, False, False] # 2 running, 2 queued (backpressure)
|
|
assert [session.get(Job, q.id).status for q in queued] == ["RUNNING", "RUNNING", "PENDING", "PENDING"]
|
|
jobs.complete_job(session, session.get(Job, queued[0].id), {}) # a slot frees up
|
|
assert jobs.claim_next_job(session, agents[2]) is not None and jobs.claim_next_job(session, agents[3]) is None
|
|
|
|
|
|
def test_a_light_incremental_job_is_never_held_behind_a_queued_full_job(monkeypatch):
|
|
_limit(monkeypatch, 1)
|
|
with setup_db()() as session:
|
|
busy, waiting = _agents(session, 2)
|
|
jobs.create_job(session, busy, "REPROCESS_ALL")
|
|
assert jobs.claim_next_job(session, busy) is not None
|
|
full = jobs.create_job(session, waiting, "REPROCESS_ALL")
|
|
light = jobs.create_job(session, waiting, "REPROCESS_PRODUCTS", {"codigos": ["1"]})
|
|
claimed = jobs.claim_next_job(session, waiting)
|
|
assert claimed is not None and claimed.id == light.id and session.get(Job, full.id).status == "PENDING"
|
|
|
|
|
|
def test_limit_zero_disables_backpressure_and_a_stale_running_job_does_not_hold_a_slot(monkeypatch):
|
|
from datetime import datetime, timedelta, timezone
|
|
_limit(monkeypatch, 0)
|
|
with setup_db()() as session:
|
|
agents = _agents(session, 3)
|
|
for a in agents:
|
|
jobs.create_job(session, a, "INITIAL_SYNC")
|
|
assert all(jobs.claim_next_job(session, a) is not None for a in agents)
|
|
_limit(monkeypatch, 1)
|
|
with setup_db()() as session:
|
|
dead, fresh = _agents(session, 2)
|
|
stale = jobs.create_job(session, dead, "INITIAL_SYNC")
|
|
session.execute(Job.__table__.update().where(Job.id == stale.id).values(status="RUNNING", started_at=datetime.now(timezone.utc) - timedelta(hours=3)))
|
|
session.commit()
|
|
jobs.create_job(session, fresh, "INITIAL_SYNC")
|
|
assert jobs.claim_next_job(session, fresh) is not None # the dead Agent's 3 h old RUNNING job does not block the queue
|