feat: zero-touch Agent — central scheduler, central pause, delta sync, discovery-driven installer

Hub: installations.processing_enabled / sync_interval_seconds / last_sync_* (migration 0026). The first heartbeat of an installation without coverage queues exactly one INITIAL_SYNC; afterwards one SYNC_INCREMENTAL per interval (idempotent, backs off on failure). One fiscal job at a time per Agent (FIFO, stale RUNNING recovery). Job completion now updates the coverage snapshot, blockers and JOB_STARTED/COMPLETED/FAILED events. Panel: Sincronizar agora / Reprocessar tudo / Pausar / Retomar + status; pause is enforced Hub-side. Agent: local AGENT_JOBS_ENABLED is ignored (central control); SYNC_INCREMENTAL re-classifies only new/changed products via a per-product fingerprint+decision cache and rebuilds coverage from it; results carry operational summary and blockers; Firebird credentials come from a DPAPI firebird.bin. Setup: read-only SGV Firebird discovery (auto-select only when unambiguous, wizard chooser otherwise), starts the service, waits for the first heartbeat, upgrade-in-place mode. All jobs stay read-only (FIREBIRD_WRITES=0). Version 1.1.3. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-24 08:47:00 -03:00
parent 9f9e06e9e9
commit bb663a979c
36 changed files with 1409 additions and 118 deletions
@@ -0,0 +1,196 @@
+"""Zero-touch control plane: central scheduler (INITIAL_SYNC once, SYNC_INCREMENTAL
+per interval), single-job concurrency, central pause/resume, coverage snapshot from
+the job result, idempotency. Everything is Hub-side — no local toggle involved."""
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from central.app.jobs import claim_next_job, create_job, recover_stale_running
+from central.app.models import Agent, AgentSnapshot, AuditEvent, Installation, Job
+from central.tests.test_jobs import _client
+
+H = {"Authorization": "Bearer test-token"}
+INST = "00000000-0000-4000-8000-0000000000aa"
+OPERATIONAL = {"COMPLETE": 480, "CORRECTED": 0, "BLOCKED_MISSING_EVIDENCE": 31, "BLOCKED_CONFLICT": 10, "BLOCKED_COMPANY_CONTEXT": 0, "BLOCKED_SCHEMA": 0,
+               "FAILED_TECHNICAL": 0, "TOTAL_PRODUCTS": 521, "SAFE_BLOCKED": 41}
+
+
+def hb(client):
+    return client.post("/v1/agents/heartbeat", json={"installation_id": INST, "status": "IDLE", "agent_version": "1.1.3"}, headers=H)
+
+
+def jobs(engine, kind=None):
+    with Session(engine) as s:
+        q = select(Job).order_by(Job.id)
+        if kind: q = q.where(Job.kind == kind)
+        return s.scalars(q).all()
+
+
+def finish(client, engine, summary=None):
+    job = client.get("/v1/jobs/next", headers=H).json()
+    assert job, "expected a claimable job"
+    body = {"result_summary": summary or {"operational": OPERATIONAL, "products_processed": 521, "products_changed": 521, "blocked_products": [], "resolved_products": []}}
+    assert client.post(f"/v1/jobs/{job['job_id']}/complete", json=body, headers=H).status_code == 200
+    return job
+
+
+def age_last_sync(engine, seconds):
+    with Session(engine) as s:
+        for job in s.scalars(select(Job)).all():
+            job.finished_at = datetime.now(timezone.utc) - timedelta(seconds=seconds)
+            job.created_at = datetime.now(timezone.utc) - timedelta(seconds=seconds)
+        s.commit()
+
+
+def test_first_heartbeat_creates_exactly_one_initial_sync():
+    client, engine = _client()
+    body = hb(client).json()
+    assert body["processing_enabled"] is True and body["sync_interval_seconds"] == 1800
+    for _ in range(4):
+        hb(client)
+    initial = jobs(engine, "INITIAL_SYNC")
+    assert len(initial) == 1 and initial[0].status == "PENDING" and initial[0].payload["reason"] == "INITIAL_SYNC"
+
+
+def test_initial_sync_completion_sets_coverage_and_invariant():
+    client, engine = _client()
+    hb(client); finish(client, engine)
+    with Session(engine) as s:
+        snap = s.scalar(select(AgentSnapshot)); inst = s.scalar(select(Installation))
+        assert (snap.total_products, snap.fiscally_complete, snap.safe_blocked, snap.failed_products) == (521, 480, 41, 0)
+        assert snap.fiscally_complete + snap.safe_blocked + snap.failed_products == snap.total_products  # coverage invariant closes
+        assert snap.blockers == {"BLOCKED_MISSING_EVIDENCE": 31, "BLOCKED_CONFLICT": 10}
+        assert inst.last_sync_at is not None and inst.last_sync_changed == 521
+    for _ in range(3):
+        hb(client)
+    assert len(jobs(engine)) == 1  # no second initial sync, and not due for an incremental yet
+
+
+def test_incremental_sync_scheduled_once_per_interval():
+    client, engine = _client()
+    hb(client); finish(client, engine)
+    age_last_sync(engine, 1801)
+    hb(client); hb(client); hb(client)
+    inc = jobs(engine, "SYNC_INCREMENTAL")
+    assert len(inc) == 1 and inc[0].payload["reason"] == "SCHEDULED"
+    assert not jobs(engine, "REPROCESS_ALL")  # full reprocess never runs automatically
+
+
+def test_central_pause_stops_scheduling_and_claiming_but_heartbeat_continues():
+    client, engine = _client()
+    hb(client)
+    with Session(engine) as s:
+        s.scalar(select(Installation)).processing_enabled = False; s.commit()
+    body = hb(client).json()
+    assert body["processing_enabled"] is False and body["enabled"] is True  # heartbeat keeps working
+    assert client.get("/v1/jobs/next", headers=H).json() is None  # the PENDING initial sync is NOT handed out while paused
+    with Session(engine) as s:
+        s.scalar(select(Installation)).processing_enabled = True; s.commit()
+    assert client.get("/v1/jobs/next", headers=H).json()["kind"] == "INITIAL_SYNC"  # resume: jobs flow again
+
+
+def test_only_one_job_runs_at_a_time_and_queue_is_fifo():
+    client, engine = _client()
+    hb(client)
+    with Session(engine) as s:
+        agent = s.scalar(select(Agent))
+        create_job(s, agent, "REPROCESS_PRODUCTS", {"codigos": ["1"]})
+    first = client.get("/v1/jobs/next", headers=H).json()
+    assert first["kind"] == "INITIAL_SYNC"
+    assert client.get("/v1/jobs/next", headers=H).json() is None  # second job waits: one fiscal job per Firebird
+    client.post(f"/v1/jobs/{first['job_id']}/complete", json={"result_summary": {}}, headers=H)
+    assert client.get("/v1/jobs/next", headers=H).json()["kind"] == "REPROCESS_PRODUCTS"
+
+
+def test_reprocess_all_and_sync_now_are_not_duplicated():
+    client, engine = _client()
+    hb(client); finish(client, engine)
+    with Session(engine) as s:
+        agent = s.scalar(select(Agent))
+        a = create_job(s, agent, "SYNC_INCREMENTAL", {"reason": "MANUAL_SYNC_NOW"}); b = create_job(s, agent, "SYNC_INCREMENTAL")
+        assert a.id == b.id  # double click = one job
+        full1 = create_job(s, agent, "REPROCESS_ALL"); full2 = create_job(s, agent, "REPROCESS_ALL")
+        assert full1.id == full2.id and full1.id != a.id  # a full reprocess still queues behind the incremental, never twice
+        assert create_job(s, agent, "SYNC_INCREMENTAL").id == a.id
+
+
+def test_stale_running_job_does_not_block_the_queue_forever():
+    client, engine = _client()
+    hb(client)
+    job = client.get("/v1/jobs/next", headers=H).json()
+    with Session(engine) as s:
+        row = s.get(Job, job["job_id"]); row.started_at = datetime.now(timezone.utc) - timedelta(hours=2); s.commit()
+        create_job(s, s.scalar(select(Agent)), "REPROCESS_PRODUCTS", {"codigos": ["1"]})
+    assert client.get("/v1/jobs/next", headers=H).json()["kind"] == "REPROCESS_PRODUCTS"
+    with Session(engine) as s:
+        assert s.get(Job, job["job_id"]).status == "FAILED"
+
+
+def test_job_events_recorded_without_secrets():
+    client, engine = _client()
+    hb(client); finish(client, engine)
+    with Session(engine) as s:
+        from central.app.models import AgentEvent
+        types = {e.event_type for e in s.scalars(select(AgentEvent)).all()}
+        assert {"HEARTBEAT", "JOB_STARTED", "JOB_COMPLETED"} <= types
+
+
+def test_failed_job_records_failure_and_does_not_reschedule_immediately():
+    client, engine = _client()
+    hb(client)
+    job = client.get("/v1/jobs/next", headers=H).json()
+    client.post(f"/v1/jobs/{job['job_id']}/fail", json={"error": "Firebird offline"}, headers=H)
+    for _ in range(3):
+        hb(client)
+    assert len(jobs(engine)) == 1  # backoff = the interval; no error loop against a broken Firebird
+    with Session(engine) as s:
+        assert s.scalar(select(AgentSnapshot)).last_job_status == "FAILED"
+
+
+# ---------- panel: central control buttons + status ----------
+
+def _panel():
+    from central.tests.test_control_panel_pages import _panel_client
+    client, agent_id, _ = _panel_client()
+    return client, agent_id, client._session_factory
+
+
+def test_panel_shows_processing_status_and_the_operational_buttons():
+    client, agent_id, _ = _panel()
+    body = client.get(f"/agents/{agent_id}").text
+    for label in ("Sincronizar agora", "Reprocessar tudo", "Pausar processamento", "Última sincronização", "Sincronização agendada"):
+        assert label in body
+    assert "Ativo" in body and "a cada 30 min" in body and "Retomar processamento" not in body
+
+
+def test_panel_sync_now_pause_resume_are_csrf_protected_audited_and_idempotent():
+    client, agent_id, factory = _panel()
+    csrf = client.cookies.get("gti_csrf")
+    assert client.post(f"/agents/{agent_id}/sync-now", data={"csrf_token": "wrong"}).status_code == 403
+    assert client.post(f"/agents/{agent_id}/sync-now", data={"csrf_token": csrf}, follow_redirects=False).status_code == 303
+    assert client.post(f"/agents/{agent_id}/sync-now", data={"csrf_token": csrf}, follow_redirects=False).status_code == 303  # double click
+    with factory() as s:
+        incs = s.scalars(select(Job).where(Job.kind == "SYNC_INCREMENTAL")).all()
+        assert len(incs) == 1 and incs[0].payload["reason"] == "MANUAL_SYNC_NOW"
+
+    client.post(f"/agents/{agent_id}/pause", data={"csrf_token": csrf})
+    with factory() as s:
+        assert s.scalar(select(Installation).where(Installation.agent_id == agent_id)).processing_enabled is False
+    paused = client.get(f"/agents/{agent_id}").text
+    assert "Pausado" in paused and "Retomar processamento" in paused and "disabled" in paused  # Sync-now disabled while paused
+    client.post(f"/agents/{agent_id}/resume", data={"csrf_token": csrf})
+    with factory() as s:
+        assert s.scalar(select(Installation).where(Installation.agent_id == agent_id)).processing_enabled is True
+        actions = [a.action for a in s.scalars(select(AuditEvent)).all()]
+        assert {"SYNC_NOW", "PROCESSING_PAUSE", "PROCESSING_RESUME"} <= set(actions)
+
+
+def test_panel_controls_require_login():
+    from fastapi.testclient import TestClient
+    from central.app.main import create_app
+    from central.tests.test_control_plane import seed, setup_db
+    factory = setup_db(); agent_id, _ = seed(factory)
+    anon = TestClient(create_app(database_probe=lambda: True, session_factory=factory, bootstrap_token=None))
+    for action in ("sync-now", "pause", "resume"):
+        assert anon.post(f"/agents/{agent_id}/{action}", data={}, follow_redirects=False).status_code == 303  # bounced to login, nothing executed