From deb71363c91af8663be999a70c0f0ce4c84cd1e5 Mon Sep 17 00:00:00 2001 From: 2GT-Rich <107219997+2GT-Rich@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:20:56 -0700 Subject: [PATCH] fix(poller): cap Redis memory, and stop retrying periodic polls (v0.23.18-beta) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two follow-ups left open by v0.23.17. One is a real fix; the other turned out to need explaining rather than fixing. **Redis had no memory ceiling.** v0.23.17 stopped the poller from growing Redis without bound, but nothing stopped anything else from doing so, and the host had no protection either way. Redis now runs with a 512 MB cap (REDIS_MAXMEMORY). The eviction policy is the part worth care. `allkeys-lru` would be actively dangerous here: BullMQ job data carries no TTL, so an LRU sweep across all keys could discard queued work with no trace. `volatile-lru` evicts only keys that set an expiry — which is exactly the poller's caches and in-flight claims, all of which are safe to lose because losing one merely causes an earlier re-poll. Verified rather than assumed: sampled BullMQ job keys report ttl=-1 and poller cache keys 528-3408s. If pressure ever exhausts the volatile keys Redis refuses writes instead of evicting queue state, which is loud and survivable. **The 14% fast-poll failure rate is not a bug.** Current rate across all devices and job kinds is 0% over 1,462 attempts. The historical 230,707 failures were genuine device downtime: device_availability records 324 outages totalling ~1,003 hours across four lab devices, which at a 30-second cadence and two attempts per cycle predicts ~240,700 failures — within 4% of what was recorded. Nothing to fix; retention had simply been hoarding an accurate count. It did expose real waste, though. Periodic polls no longer retry in-job, because the schedule already is the retry: a device that did not answer is unlikely to answer a second later, and each extra attempt occupies a worker for up to the job timeout. On a fleet with several devices down that doubled the capacity they consumed — the same capacity the rest of the fleet was short of in #114. It also doubled the recorded failure count, which is how ~1,000 hours of downtime came to look like 231k distinct faults. User-initiated full syncs keep their retries. Co-Authored-By: Claude Opus 5 --- .env.example | 13 +++++++++++++ README.md | 2 +- backend/package.json | 2 +- backend/src/services/PollerService.ts | 13 ++++++++++--- docker-compose.yml | 11 ++++++++++- frontend/package.json | 2 +- frontend/src/version.ts | 2 +- 7 files changed, 37 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 31f1caf..e6ffdd6 100644 --- a/.env.example +++ b/.env.example @@ -21,3 +21,16 @@ INFLUXDB_TOKEN=mytoken123456789 INFLUXDB_ORG=mikrotik-manager INFLUXDB_BUCKET=metrics INFLUXDB_ADMIN_PASSWORD=admin_password_123 + +# ─── Poller tuning (see GET /api/system/poller) ────────────────────────────── +# Raise concurrency if `headroom` in the poller health endpoint drops near 1.0 — +# below that the queue grows every cycle and devices go stale. +# POLLER_CONCURRENCY=12 +# POLLER_JOB_TIMEOUT_MS=45000 +# POLLER_INTERVAL_MS=30000 +# POLLER_JOB_RETENTION_SEC=3600 +# POLLER_JOB_RETENTION_COUNT=5000 + +# Redis memory ceiling. Only TTL-bearing keys are evicted (volatile-lru), so +# queued jobs are never discarded; the cap exists to protect the host. +# REDIS_MAXMEMORY=512mb diff --git a/README.md b/README.md index f131f26..3da4552 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@

- Version + Version License Docker TypeScript diff --git a/backend/package.json b/backend/package.json index fcb9841..d2636fa 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "mikrotik-manager-backend", - "version": "0.23.17-beta", + "version": "0.23.18-beta", "description": "Mikrotik Network Management Platform - Backend API", "main": "dist/index.js", "scripts": { diff --git a/backend/src/services/PollerService.ts b/backend/src/services/PollerService.ts index e36f86f..76ac9ba 100644 --- a/backend/src/services/PollerService.ts +++ b/backend/src/services/PollerService.ts @@ -159,12 +159,19 @@ export class PollerService { attempts: 3, backoff: { type: 'exponential', delay: 5000 }, }); + // Periodic polls are not retried in-job: the schedule is the retry. A device + // that just failed to answer is unlikely to answer a second later, and each + // extra attempt occupies a worker for up to the job timeout. On a fleet with + // several devices down that doubles the capacity they waste, which is exactly + // the capacity the rest of the fleet is short of (#114). Historically this + // also doubled the recorded failure count: 231k failed jobs on a four-device + // fleet turned out to be ~1,000 hours of genuine downtime, counted twice. } else if (type === 'fast') { - await this.fastQueue.add('device-fast-poll', jobData, { attempts: 2 }); + await this.fastQueue.add('device-fast-poll', jobData, { attempts: 1 }); } else if (type === 'slow') { - await this.slowQueue.add('device-slow-poll', jobData, { attempts: 2 }); + await this.slowQueue.add('device-slow-poll', jobData, { attempts: 1 }); } else if (type === 'logs') { - await this.logsQueue.add('device-logs-poll', jobData, { attempts: 2 }); + await this.logsQueue.add('device-logs-poll', jobData, { attempts: 1 }); } else if (type === 'macscan') { await this.fastQueue.add('device-macscan', jobData, { attempts: 1 }); } else if (type === 'spectral') { diff --git a/docker-compose.yml b/docker-compose.yml index 6fe9683..c48f867 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,7 +85,16 @@ services: redis: image: redis:7-alpine - command: redis-server --appendonly yes + # A memory ceiling so a runaway producer can never take the host down. The + # eviction policy is deliberately volatile-lru rather than allkeys-*: BullMQ + # job data carries no TTL and must never be evicted (that would silently + # destroy queued work), while the poller's caches and in-flight claims all + # set one and are safe to drop. If pressure ever exhausts the volatile keys + # Redis refuses writes instead of evicting queue state — loud, and survivable. + command: > + redis-server --appendonly yes + --maxmemory ${REDIS_MAXMEMORY:-512mb} + --maxmemory-policy volatile-lru volumes: - redis_data:/data restart: unless-stopped diff --git a/frontend/package.json b/frontend/package.json index 2d064da..822ff00 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mikrotik-manager-frontend", - "version": "0.23.17-beta", + "version": "0.23.18-beta", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/version.ts b/frontend/src/version.ts index 2252b75..a457959 100644 --- a/frontend/src/version.ts +++ b/frontend/src/version.ts @@ -1 +1 @@ -export const APP_VERSION = 'v0.23.17 Beta'; +export const APP_VERSION = 'v0.23.18 Beta';