feat: zero-touch Agent — central scheduler, central pause, delta sync, discovery-driven installer
Hub: installations.processing_enabled / sync_interval_seconds / last_sync_* (migration 0026). The first heartbeat of an installation without coverage queues exactly one INITIAL_SYNC; afterwards one SYNC_INCREMENTAL per interval (idempotent, backs off on failure). One fiscal job at a time per Agent (FIFO, stale RUNNING recovery). Job completion now updates the coverage snapshot, blockers and JOB_STARTED/COMPLETED/FAILED events. Panel: Sincronizar agora / Reprocessar tudo / Pausar / Retomar + status; pause is enforced Hub-side. Agent: local AGENT_JOBS_ENABLED is ignored (central control); SYNC_INCREMENTAL re-classifies only new/changed products via a per-product fingerprint+decision cache and rebuilds coverage from it; results carry operational summary and blockers; Firebird credentials come from a DPAPI firebird.bin. Setup: read-only SGV Firebird discovery (auto-select only when unambiguous, wizard chooser otherwise), starts the service, waits for the first heartbeat, upgrade-in-place mode. All jobs stay read-only (FIREBIRD_WRITES=0). Version 1.1.3. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
Claude Sonnet 5
+46
-2
@@ -10,7 +10,7 @@ from .auth import authenticate, bootstrap
|
||||
from .classification import ENGINE_VERSION, Classifier
|
||||
from .config import settings
|
||||
from .datasets.service import DatasetService
|
||||
from .jobs import claim_next_job, complete_job, create_job, fail_job
|
||||
from .jobs import SYNC_KINDS, claim_next_job, complete_job, create_job, fail_job, schedule_automatic_sync
|
||||
from .blocked_products import list_blockers, resolve_blocker, resolve_product_blockers, upsert_blocker
|
||||
from .context_keys import validate_entry
|
||||
from .models import AgentSnapshot, BlockedProduct, FiscalContextEntry, ClassificationCache, ClassificationRequest, ClassificationResult, Dataset, Installation, Job
|
||||
@@ -206,8 +206,13 @@ def router(session_factory, bootstrap_token):
|
||||
session.add(agent); session.flush()
|
||||
record_event(session, tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, severity="INFO", event_type="HEARTBEAT", message="Agent heartbeat", context={"status": agent.last_status, "agent_version": agent.agent_version})
|
||||
session.commit()
|
||||
# Central scheduler: exactly one INITIAL_SYNC after the first heartbeat, then
|
||||
# one SYNC_INCREMENTAL per interval — idempotent, no local cron/toggle needed.
|
||||
if agent.enabled:
|
||||
schedule_automatic_sync(session, agent, installation, snapshot)
|
||||
logger.info("heartbeat agent_id=%s status=%s last_job_id=%s", agent.id, agent.last_status, payload.get("last_job_id"))
|
||||
return {"agent_id": agent.id, "installation_id": installation.installation_id, "last_seen_at": agent.last_seen_at.isoformat(), "enabled": agent.enabled}
|
||||
return {"agent_id": agent.id, "installation_id": installation.installation_id, "last_seen_at": agent.last_seen_at.isoformat(), "enabled": agent.enabled,
|
||||
"processing_enabled": bool(installation.processing_enabled), "sync_interval_seconds": installation.sync_interval_seconds}
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/register")
|
||||
@@ -332,6 +337,36 @@ def router(session_factory, bootstrap_token):
|
||||
"attempts": job.attempts, "result_summary": job.result_summary, "error": job.error,
|
||||
}
|
||||
|
||||
def _apply_job_outcome(session, agent, installation, job, result_summary, *, failed):
|
||||
"""After a job finishes: coverage snapshot (from the Agent's operational
|
||||
summary), sync bookkeeping and an audit-friendly event. Never stores secrets."""
|
||||
now = datetime.now(timezone.utc)
|
||||
snapshot = session.scalar(select(AgentSnapshot).where(AgentSnapshot.installation_id == installation.id))
|
||||
if snapshot is None:
|
||||
snapshot = AgentSnapshot(installation_id=installation.id); session.add(snapshot)
|
||||
snapshot.last_job_status = "FAILED" if failed else "COMPLETED"
|
||||
snapshot.last_job_at = now
|
||||
context = {"kind": job.kind}
|
||||
operational = result_summary.get("operational") if not failed else None
|
||||
if operational and job.kind in SYNC_KINDS:
|
||||
total = int(operational.get("TOTAL_PRODUCTS", 0))
|
||||
complete = int(operational.get("COMPLETE", 0)) + int(operational.get("CORRECTED", 0))
|
||||
snapshot.total_products = total
|
||||
snapshot.fiscally_complete = complete
|
||||
snapshot.safe_blocked = int(operational.get("SAFE_BLOCKED", 0))
|
||||
snapshot.failed_products = int(operational.get("FAILED_TECHNICAL", 0))
|
||||
snapshot.coverage_percent = round(100.0 * complete / total, 2) if total else 0.0
|
||||
snapshot.blockers = {state: int(count) for state, count in operational.items() if state.startswith("BLOCKED_") and count}
|
||||
snapshot.last_successful_job_at = now
|
||||
installation.last_sync_at = now
|
||||
installation.last_sync_changed = int(result_summary.get("products_changed", result_summary.get("products_processed", 0)) or 0)
|
||||
context.update({"total_products": total, "products_changed": installation.last_sync_changed, "products_processed": result_summary.get("products_processed")})
|
||||
if failed:
|
||||
context["error"] = result_summary.get("error")
|
||||
record_event(session, tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, severity="ERROR" if failed else "INFO",
|
||||
event_type="JOB_FAILED" if failed else "JOB_COMPLETED", message=f"Job {job.kind} {'failed' if failed else 'completed'}", context=context, job_id=job.id)
|
||||
session.commit()
|
||||
|
||||
@routes.post("/jobs")
|
||||
def create_job_route(payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
@@ -353,6 +388,10 @@ def router(session_factory, bootstrap_token):
|
||||
try:
|
||||
job = claim_next_job(session, agent)
|
||||
if job is None: return None
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is not None:
|
||||
record_event(session, tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, severity="INFO", event_type="JOB_STARTED", message=f"Job {job.kind} started", context={"kind": job.kind}, job_id=job.id)
|
||||
session.commit()
|
||||
logger.info("job claimed job_id=%s agent_id=%s kind=%s", job.id, agent.id, job.kind)
|
||||
return _job_view(job)
|
||||
finally: session.close()
|
||||
@@ -388,6 +427,8 @@ def router(session_factory, bootstrap_token):
|
||||
for product_code in result_summary.get("resolved_products", []):
|
||||
resolve_product_blockers(session, agent.tenant_id, product_code, installation_id=installation.id if installation else None)
|
||||
if not complete_job(session, job, result_summary): raise HTTPException(409, "job is not RUNNING")
|
||||
if installation is not None:
|
||||
_apply_job_outcome(session, agent, installation, job, result_summary, failed=False)
|
||||
logger.info("job completed job_id=%s agent_id=%s result=%s", job_id, agent.id, payload.get("result_summary", {}))
|
||||
return _job_view(session.get(Job, job_id))
|
||||
finally: session.close()
|
||||
@@ -399,6 +440,9 @@ def router(session_factory, bootstrap_token):
|
||||
job = session.get(Job, job_id)
|
||||
if job is None or job.agent_id != agent.id: raise HTTPException(404, "job not found")
|
||||
if not fail_job(session, job, payload.get("error", "unknown error")): raise HTTPException(409, "job is not RUNNING")
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is not None:
|
||||
_apply_job_outcome(session, agent, installation, job, {"error": str(payload.get("error", "unknown error"))[:500]}, failed=True)
|
||||
logger.warning("job failed job_id=%s agent_id=%s error=%s", job_id, agent.id, payload.get("error", "unknown error"))
|
||||
return _job_view(session.get(Job, job_id))
|
||||
finally: session.close()
|
||||
|
||||
+73
-8
@@ -1,31 +1,74 @@
|
||||
"""Minimal job queue for Agent orchestration: create, atomically claim, complete, fail."""
|
||||
from datetime import datetime, timezone
|
||||
"""Job queue for Agent orchestration: create (idempotent per kind), claim one job at
|
||||
a time per Agent, complete, fail. Also the central sync scheduler helpers.
|
||||
|
||||
Every fiscal job is READ-ONLY on the ERP (INITIAL_SYNC, SYNC_INCREMENTAL,
|
||||
REPROCESS_ALL, REPROCESS_PRODUCTS); automation never authorizes Firebird writes.
|
||||
"""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from .models import Job
|
||||
from .models import Installation, Job
|
||||
|
||||
_VALID_KINDS = {"REPROCESS_ALL", "REPROCESS_PRODUCTS"}
|
||||
FULL_KINDS = ("INITIAL_SYNC", "REPROCESS_ALL") # read every product, reclassify all
|
||||
SYNC_KINDS = FULL_KINDS + ("SYNC_INCREMENTAL",) # jobs that count as "a sync happened"
|
||||
_VALID_KINDS = set(SYNC_KINDS) | {"REPROCESS_PRODUCTS"}
|
||||
_MAX_ERROR_LENGTH = 2000
|
||||
STALE_RUNNING_AFTER = timedelta(hours=1) # an Agent that died mid-job must not block the queue forever
|
||||
|
||||
|
||||
def _aware(value):
|
||||
if value is not None and value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value
|
||||
|
||||
|
||||
def create_job(session, agent, kind, payload=None):
|
||||
"""Create a job. Sync-type jobs are idempotent: if this Agent already has a
|
||||
PENDING/RUNNING job that covers the request, that job is returned instead
|
||||
(no duplicates, never two full jobs at once)."""
|
||||
if kind not in _VALID_KINDS:
|
||||
raise ValueError(f"unsupported job kind: {kind}")
|
||||
if kind in SYNC_KINDS:
|
||||
# A full job subsumes an incremental one, not the other way round: asking for a
|
||||
# full reprocess while only an incremental is queued must still queue the full.
|
||||
blocking = FULL_KINDS if kind in FULL_KINDS else SYNC_KINDS
|
||||
existing = session.scalar(select(Job).where(Job.agent_id == agent.id, Job.kind.in_(blocking), Job.status.in_(("PENDING", "RUNNING"))).order_by(Job.id).limit(1))
|
||||
if existing is not None:
|
||||
return existing
|
||||
job = Job(agent_id=agent.id, kind=kind, status="PENDING", payload=payload or {})
|
||||
session.add(job)
|
||||
session.commit()
|
||||
return job
|
||||
|
||||
|
||||
def recover_stale_running(session, agent, now=None):
|
||||
now = now or datetime.now(timezone.utc)
|
||||
cutoff = now - STALE_RUNNING_AFTER
|
||||
stale = session.scalars(select(Job).where(Job.agent_id == agent.id, Job.status == "RUNNING")).all()
|
||||
changed = False
|
||||
for job in stale:
|
||||
if job.started_at is not None and _aware(job.started_at) < cutoff:
|
||||
job.status, job.finished_at, job.error = "FAILED", now, "stale: Agent stopped reporting while RUNNING"
|
||||
changed = True
|
||||
if changed:
|
||||
session.commit()
|
||||
|
||||
|
||||
def claim_next_job(session, agent):
|
||||
"""Atomically claim the oldest PENDING job for this agent only.
|
||||
A single UPDATE...WHERE status='PENDING' guards against two concurrent
|
||||
pollers (this agent or any other) claiming the same job twice."""
|
||||
"""Atomically claim the oldest PENDING job for this agent only — and only when
|
||||
processing is enabled centrally and no other job of this Agent is RUNNING
|
||||
(one fiscal job at a time per Firebird)."""
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is not None and not installation.processing_enabled:
|
||||
return None
|
||||
recover_stale_running(session, agent)
|
||||
if session.scalar(select(Job.id).where(Job.agent_id == agent.id, Job.status == "RUNNING").limit(1)) is not None:
|
||||
return None
|
||||
candidate = session.scalar(
|
||||
select(Job.id)
|
||||
.where(Job.agent_id == agent.id, Job.status == "PENDING")
|
||||
.order_by(Job.created_at.asc())
|
||||
.order_by(Job.created_at.asc(), Job.id.asc())
|
||||
.limit(1)
|
||||
)
|
||||
if candidate is None:
|
||||
@@ -61,3 +104,25 @@ def fail_job(session, job, error):
|
||||
)
|
||||
session.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
def schedule_automatic_sync(session, agent, installation, snapshot, now=None):
|
||||
"""Called from the Agent heartbeat. Central scheduler, idempotent:
|
||||
- first heartbeat of an installation with no initial coverage and no sync job
|
||||
ever created -> exactly ONE INITIAL_SYNC;
|
||||
- otherwise, if processing is enabled and the last sync attempt is older than
|
||||
the installation's interval, and nothing is queued/running -> ONE SYNC_INCREMENTAL.
|
||||
Returns the created job or None."""
|
||||
if installation is None or not installation.processing_enabled:
|
||||
return None
|
||||
now = now or datetime.now(timezone.utc)
|
||||
sync_jobs = session.scalars(select(Job).where(Job.agent_id == agent.id, Job.kind.in_(SYNC_KINDS)).order_by(Job.id.desc())).all()
|
||||
if any(job.status in ("PENDING", "RUNNING") for job in sync_jobs):
|
||||
return None
|
||||
never_synced = not any(job.kind == "INITIAL_SYNC" or job.kind == "REPROCESS_ALL" for job in sync_jobs) and not int(getattr(snapshot, "total_products", 0) or 0)
|
||||
if never_synced and not sync_jobs:
|
||||
return create_job(session, agent, "INITIAL_SYNC", {"reason": "INITIAL_SYNC"})
|
||||
last_attempt = max((_aware(job.finished_at or job.created_at) for job in sync_jobs if (job.finished_at or job.created_at)), default=None)
|
||||
if last_attempt is not None and (now - last_attempt).total_seconds() >= installation.sync_interval_seconds:
|
||||
return create_job(session, agent, "SYNC_INCREMENTAL", {"reason": "SCHEDULED"})
|
||||
return None
|
||||
|
||||
@@ -276,6 +276,12 @@ class Installation(Base):
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
||||
last_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
# Central execution control (no local .env toggle): the Hub decides whether this
|
||||
# installation's Agent executes jobs, and how often the automatic incremental sync runs.
|
||||
processing_enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||
sync_interval_seconds: Mapped[int] = mapped_column(Integer, default=1800, server_default="1800")
|
||||
last_sync_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_sync_changed: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
|
||||
once published — never a second, independent version number. Fiscal semantics
|
||||
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
|
||||
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
|
||||
GTI_FISCAL_VERSION = "1.1.2"
|
||||
GTI_FISCAL_VERSION = "1.1.3"
|
||||
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
|
||||
|
||||
+43
-1
@@ -279,9 +279,51 @@ def router(session_factory):
|
||||
for event in events:
|
||||
event["occurred_display"] = format_ui_datetime(datetime.fromisoformat(event["occurred_at"]) if event["occurred_at"] else None)
|
||||
agent_blockers = list_blockers(session, tenant_id=installation.tenant_id, agent_id=agent_id, page=1, page_size=25)
|
||||
return render(request, "agent_detail.html", {"detail": detail, "tab": tab, "blockers": agent_blockers, "blocker_breakdown": blocker_filter_summary(session, tenant_id=installation.tenant_id, agent_id=agent_id, status="OPEN")["blocker_breakdown"], "jobs": list_jobs(session, agent_id=agent_id, page=1, page_size=25), "events": events, "event_page": max(1, event_page), "has_more": has_more, "severity": severity or "", "event_type": event_type or "", "job_id": job_id or "", "product_code": product_code or "", "since": since or "", "until": until or "", "event_query_base": query_base(tab=tab, severity=severity, event_type=event_type, job_id=job_id, product_code=product_code, since=since, until=until), "auth": auth_data, "csrf_token": request.cookies.get("gti_csrf", "")})
|
||||
return render(request, "agent_detail.html", {"detail": detail, "tab": tab, "blockers": agent_blockers, "blocker_breakdown": blocker_filter_summary(session, tenant_id=installation.tenant_id, agent_id=agent_id, status="OPEN")["blocker_breakdown"], "jobs": list_jobs(session, agent_id=agent_id, page=1, page_size=25), "events": events, "event_page": max(1, event_page), "has_more": has_more, "severity": severity or "", "event_type": event_type or "", "job_id": job_id or "", "product_code": product_code or "", "since": since or "", "until": until or "", "event_query_base": query_base(tab=tab, severity=severity, event_type=event_type, job_id=job_id, product_code=product_code, since=since, until=until), "auth": auth_data, "csrf_token": request.cookies.get("gti_csrf", ""), "ops": _ops_view(session, installation, agent)})
|
||||
finally: session.close()
|
||||
|
||||
def _ops_view(session, installation, agent):
|
||||
"""Processing status shown on the Agent detail (no clutter: 5 facts)."""
|
||||
last_job = session.scalar(select(Job).where(Job.agent_id == agent.id).order_by(Job.id.desc()).limit(1))
|
||||
last_sync = _aware(installation.last_sync_at)
|
||||
next_sync = None
|
||||
if installation.processing_enabled and last_sync is not None:
|
||||
next_sync = last_sync + timedelta(seconds=installation.sync_interval_seconds)
|
||||
return {"enabled": bool(installation.processing_enabled), "interval_minutes": installation.sync_interval_seconds // 60,
|
||||
"last_sync": format_ui_datetime(last_sync) if last_sync else None, "next_sync": format_ui_datetime(next_sync) if next_sync else None,
|
||||
"last_changed": installation.last_sync_changed, "last_job": {"kind": last_job.kind, "status": last_job.status, "id": last_job.id} if last_job else None}
|
||||
|
||||
async def _installation_action(request, agent_id, action):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
agent = session.get(Agent, agent_id)
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent_id)) if agent else None
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
if action == "pause": installation.processing_enabled = False
|
||||
elif action == "resume": installation.processing_enabled = True
|
||||
elif action == "sync-now":
|
||||
job = create_job(session, agent, "SYNC_INCREMENTAL", {"reason": "MANUAL_SYNC_NOW", "requested_by": admin.id})
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="SYNC_NOW", target_type="installation", target_id=installation.installation_id, context={"job_id": job.id, "kind": job.kind})
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action=f"PROCESSING_{action.upper().replace('-', '_')}", target_type="installation", target_id=installation.installation_id, context={"processing_enabled": installation.processing_enabled}) if action in ("pause", "resume") else None
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}?tab=jobs", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/{agent_id}/sync-now")
|
||||
async def sync_now(request: Request, agent_id: int):
|
||||
return await _installation_action(request, agent_id, "sync-now")
|
||||
|
||||
@routes.post("/agents/{agent_id}/pause")
|
||||
async def pause_processing(request: Request, agent_id: int):
|
||||
return await _installation_action(request, agent_id, "pause")
|
||||
|
||||
@routes.post("/agents/{agent_id}/resume")
|
||||
async def resume_processing(request: Request, agent_id: int):
|
||||
return await _installation_action(request, agent_id, "resume")
|
||||
|
||||
@routes.get("/agents/{agent_id}/reprocess-confirmation", response_class=HTMLResponse)
|
||||
def reprocess_confirmation(request: Request, agent_id: int):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
|
||||
Reference in New Issue
Block a user