From 37887d38c25189e2534fb66159f74882d6b5103f Mon Sep 17 00:00:00 2001 From: 2GT-Rich <107219997+2GT-Rich@users.noreply.github.com> Date: Sat, 22 Aug 2026 20:42:16 -0700 Subject: [PATCH] feat: RouterBOOT in staged rollouts, and snoozeable insights (v0.23.10-beta) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #113, #102. #113 — RouterBOOT as a follow-on rollout step Firmware rollouts upgraded RouterOS and stopped, leaving the bootloader to be done by hand on every device — the exact task staged rollouts exist to remove. The data was already collected and the collector already had the two methods; the orchestrator simply never called them. An optional "Then RouterBOOT" step now runs per device after the RouterOS upgrade has been verified, rides out the second reboot, and confirms the firmware version actually moved. Ordering is deliberate: the bootloader ships inside the RouterOS package, so upgrading it first would apply the old one. It runs only once the OS half is confirmed good, and off by default, because it is a second flash and a second reboot per device. A device with nothing pending is a success rather than a failure — most of a fleet is already current, and treating "nothing to do" as an error would halt rollouts for no reason. When RouterBOOT does fail after a successful RouterOS upgrade the error says so explicitly, rather than reporting the device as simply failed and sending someone to re-run an upgrade it already has. #102 — snooze an item in "Things to handle" Insights are recomputed from live state on every request and have no id, so a dismissal is keyed on a fingerprint of category, target and title. That survives a reworded title without swallowing a different device's version of the same issue. Dismissals expire, defaulting to 24 hours. "Handled" and "not now" look identical to the code, and an item that is genuinely resolved stops being generated anyway — so a permanent dismissal could only ever hide something still true. Expired rows are cleared on read, so a lapse needs no scheduled job, and the panel reports how many items are currently hidden rather than quietly shrinking. Verified against the fleet: snoozing took the list from 8 items to 7 with dismissedCount 1, and removing the dismissal restored it. Co-Authored-By: Claude Opus 5 --- README.md | 2 +- backend/package.json | 2 +- backend/src/db/migrate.ts | 22 ++++++ backend/src/routes/firmware.ts | 10 +-- backend/src/routes/operations.ts | 66 +++++++++++++++++- backend/src/services/FirmwareOrchestrator.ts | 72 ++++++++++++++++++++ frontend/package.json | 2 +- frontend/src/pages/DashboardPage.tsx | 30 +++++++- frontend/src/pages/FirmwarePage.tsx | 13 +++- frontend/src/services/api.ts | 15 +++- frontend/src/version.ts | 2 +- 11 files changed, 221 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index d13a7ff..3e74a07 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@

- Version + Version License Docker TypeScript diff --git a/backend/package.json b/backend/package.json index 97ae2dc..67c70bf 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "mikrotik-manager-backend", - "version": "0.23.9-beta", + "version": "0.23.10-beta", "description": "Mikrotik Network Management Platform - Backend API", "main": "dist/index.js", "scripts": { diff --git a/backend/src/db/migrate.ts b/backend/src/db/migrate.ts index b64980b..e9f6c5e 100644 --- a/backend/src/db/migrate.ts +++ b/backend/src/db/migrate.ts @@ -603,6 +603,28 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_seccheck_supp_device -- (github issue #96). ALTER TABLE devices ADD COLUMN IF NOT EXISTS wifi_package VARCHAR(16); +-- RouterBOOT as a follow-on step in a staged rollout (github issue #113). +-- MikroTik's own guidance is to upgrade the bootloader after RouterOS, which until +-- now meant visiting every device by hand — the exact task staged rollouts exist to +-- remove. +ALTER TABLE firmware_rollouts ADD COLUMN IF NOT EXISTS routerboot_after BOOLEAN NOT NULL DEFAULT FALSE; + +-- Temporarily dismissed "Things to handle" items (github issue #102). +-- +-- Insights are recomputed from live state on every request and have no natural id, +-- so a dismissal is keyed on a fingerprint of what the item is about. Dismissals +-- expire: the point is "not now", not "never" — an item that is genuinely handled +-- stops being produced, and one that is not should come back and ask again. +CREATE TABLE IF NOT EXISTS insight_dismissals ( + fingerprint VARCHAR(160) PRIMARY KEY, + category VARCHAR(40), + title TEXT, + dismissed_by VARCHAR(50), + dismissed_at TIMESTAMPTZ DEFAULT NOW(), + expires_at TIMESTAMPTZ NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_insight_dismissals_expiry ON insight_dismissals(expires_at); + -- ntfy.sh notification channel (github issue #93). The channel type is a CHECK -- constraint rather than a lookup table, so widening it means replacing it. ALTER TABLE alert_channels DROP CONSTRAINT IF EXISTS alert_channels_type_check; diff --git a/backend/src/routes/firmware.ts b/backend/src/routes/firmware.ts index 3ba2a90..638488a 100644 --- a/backend/src/routes/firmware.ts +++ b/backend/src/routes/firmware.ts @@ -50,8 +50,8 @@ router.post('/check-all', requireWrite, async (_req: Request, res: Response) => // POST /api/firmware/rollouts — create a rollout (optionally scheduled) router.post('/rollouts', requireWrite, async (req: Request, res: Response) => { - const { name, halt_on_failure, pre_backup, scheduled_at, devices, start } = req.body as { - name?: string; halt_on_failure?: boolean; pre_backup?: boolean; scheduled_at?: string | null; + const { name, halt_on_failure, pre_backup, routerboot_after, scheduled_at, devices, start } = req.body as { + name?: string; halt_on_failure?: boolean; pre_backup?: boolean; routerboot_after?: boolean; scheduled_at?: string | null; devices?: { device_id: number; wave: number }[]; start?: boolean; }; if (!name || !name.trim()) return res.status(400).json({ error: 'name is required' }); @@ -64,9 +64,9 @@ router.post('/rollouts', requireWrite, async (req: Request, res: Response) => { if (scheduled_at && isNaN(Date.parse(scheduled_at))) return res.status(400).json({ error: 'scheduled_at must be a valid timestamp' }); const rollout = await queryOne<{ id: number }>( - `INSERT INTO firmware_rollouts (name, halt_on_failure, pre_backup, scheduled_at) - VALUES ($1,$2,$3,$4) RETURNING id`, - [name.trim().slice(0, 100), halt_on_failure !== false, pre_backup !== false, scheduled_at || null]); + `INSERT INTO firmware_rollouts (name, halt_on_failure, pre_backup, routerboot_after, scheduled_at) + VALUES ($1,$2,$3,$4,$5) RETURNING id`, + [name.trim().slice(0, 100), halt_on_failure !== false, pre_backup !== false, routerboot_after === true, scheduled_at || null]); for (const d of devices) { await query( `INSERT INTO firmware_rollout_devices (rollout_id, device_id, wave) VALUES ($1,$2,$3)`, diff --git a/backend/src/routes/operations.ts b/backend/src/routes/operations.ts index 23968d7..a4ca577 100644 --- a/backend/src/routes/operations.ts +++ b/backend/src/routes/operations.ts @@ -15,6 +15,20 @@ const bucket = process.env.INFLUXDB_BUCKET || 'mikrotik'; type Sev = 'error' | 'warn' | 'info'; interface AttentionItem { sev: Sev; category: string; title: string; body: string; action: string; path: string; + /** Stable identity for an item that is otherwise recomputed each request (#102). */ + fingerprint?: string; +} + +/** + * Identify an insight by what it is about rather than by its wording. + * + * Insights are derived from live state, so they have no id to dismiss. Keying on + * category plus the target path survives a reworded title — dismissing "sw1 is + * unreachable" should stay dismissed if the copy changes, but must NOT swallow a + * different device's outage. + */ +function fingerprintOf(item: AttentionItem): string { + return `${item.category}|${item.path}|${item.title}`.slice(0, 160); } // Latest value per device for a set of device_resources fields. @@ -314,14 +328,62 @@ router.get('/insights', async (_req: Request, res: Response) => { const anomalies = await detectAnomalies(); for (const a of anomalies) attention.push(a); + // Dismissals (#102). Expired rows are cleared on read, so a dismissal lapses + // without needing a scheduled job, and an item still true when it lapses simply + // reappears — which is the intended behaviour of "remind me later". + await query(`DELETE FROM insight_dismissals WHERE expires_at < NOW()`).catch(() => []); + const dismissed = new Set( + (await query<{ fingerprint: string }>(`SELECT fingerprint FROM insight_dismissals`) + .catch(() => [])).map((r) => r.fingerprint) + ); + + for (const item of attention) item.fingerprint = fingerprintOf(item); + const visible = attention.filter((i) => !dismissed.has(i.fingerprint!)); + // Severity ordering: error → warn → info const sevRank: Record = { error: 0, warn: 1, info: 2 }; - attention.sort((a, b) => sevRank[a.sev] - sevRank[b.sev]); + visible.sort((a, b) => sevRank[a.sev] - sevRank[b.sev]); // Activity feed — recent config changes, user actions, and notable events const activity = await buildActivity(); - res.json({ attention, capacity, activity }); + res.json({ attention: visible, dismissedCount: attention.length - visible.length, capacity, activity }); +}); + +/** + * POST /api/operations/insights/dismiss — hide one item for a while. + * + * Time-boxed on purpose. "Handled" and "not now" look the same to the code, and an + * item that is genuinely resolved stops being generated anyway — so a dismissal that + * never lapsed would only ever hide things that are still true. + */ +router.post('/insights/dismiss', requireWrite, async (req: Request, res: Response) => { + const { fingerprint, category, title, hours } = req.body as + { fingerprint?: string; category?: string; title?: string; hours?: number }; + if (!fingerprint) return res.status(400).json({ error: 'fingerprint is required' }); + + const h = Math.min(720, Math.max(1, Number(hours) || 24)); + await query( + `INSERT INTO insight_dismissals (fingerprint, category, title, dismissed_by, expires_at) + VALUES ($1, $2, $3, $4, NOW() + ($5 || ' hours')::interval) + ON CONFLICT (fingerprint) DO UPDATE SET + expires_at = EXCLUDED.expires_at, dismissed_at = NOW(), dismissed_by = EXCLUDED.dismissed_by`, + [fingerprint.slice(0, 160), category ?? null, title ?? null, + (req as unknown as { user?: { username?: string } }).user?.username ?? null, String(h)] + ); + res.status(201).json({ fingerprint, hours: h }); +}); + +/** GET /api/operations/insights/dismissed — what is currently hidden, and until when. */ +router.get('/insights/dismissed', async (_req: Request, res: Response) => { + await query(`DELETE FROM insight_dismissals WHERE expires_at < NOW()`).catch(() => []); + res.json(await query(`SELECT * FROM insight_dismissals ORDER BY expires_at ASC`)); +}); + +/** DELETE — bring an item back before its dismissal lapses. */ +router.delete('/insights/dismiss/:fingerprint', requireWrite, async (req: Request, res: Response) => { + await query(`DELETE FROM insight_dismissals WHERE fingerprint = $1`, [req.params.fingerprint]); + res.status(204).end(); }); // ─── Baseline anomaly detection ──────────────────────────────────────────────── diff --git a/backend/src/services/FirmwareOrchestrator.ts b/backend/src/services/FirmwareOrchestrator.ts index 35a67fd..7a1dbfb 100644 --- a/backend/src/services/FirmwareOrchestrator.ts +++ b/backend/src/services/FirmwareOrchestrator.ts @@ -21,6 +21,8 @@ const sleep = (ms: number) => new Promise(r => setTimeout(r, ms)); interface RolloutRow { id: number; name: string; status: string; halt_on_failure: boolean; pre_backup: boolean; + /** Follow the RouterOS upgrade with the pending RouterBOOT upgrade (issue #113). */ + routerboot_after: boolean; } interface RolloutDeviceRow { id: number; rollout_id: number; device_id: number; wave: number; status: string; @@ -199,6 +201,21 @@ export class FirmwareOrchestrator { if (newVersion && fromVersion && newVersion === fromVersion) { return fail(`Device rebooted but still reports ${newVersion} — the update did not apply`); } + // 5. RouterBOOT, if asked for (issue #113). + // + // Deliberately after the RouterOS upgrade and its verification: the bootloader + // ships inside the RouterOS package, so upgrading it first would apply the old + // one. This is a second flash and a second reboot, so it only runs once the OS + // half is confirmed good. + if (rollout.routerboot_after) { + const rbResult = await this.upgradeRouterboot(device, item); + if (!rbResult.ok) { + // The RouterOS upgrade did land. Say so, rather than reporting the device as + // simply failed and sending someone to re-run an upgrade it already has. + return fail(`RouterOS upgraded to ${newVersion || 'unknown'} successfully, but the RouterBOOT upgrade failed: ${rbResult.error}`); + } + } + await this.setItem(item.id, { status: 'success', to_version: newVersion || null }); await query(`UPDATE firmware_rollout_devices SET finished_at=NOW() WHERE id=$1`, [item.id]); await query(`UPDATE devices SET ros_version=COALESCE(NULLIF($2,''), ros_version), firmware_update_available=FALSE, status='online', last_seen=NOW() WHERE id=$1`, @@ -206,6 +223,61 @@ export class FirmwareOrchestrator { console.log(`[Firmware] ${device.name}: upgraded to ${newVersion || 'unknown'}`); return true; } + + /** + * Apply a pending RouterBOOT upgrade and ride out the reboot it causes. + * + * A device with nothing pending is a success, not a failure — most of a fleet will + * already be current, and treating "nothing to do" as an error would halt rollouts + * for no reason. + */ + private async upgradeRouterboot( + device: DeviceRow, + item: RolloutDeviceRow + ): Promise<{ ok: true } | { ok: false; error: string }> { + let before: string; + try { + const probe = new DeviceCollector(device); + await probe.connect(); + const status = await probe.checkRouterboardUpgrade(); + before = status.currentFirmware; + if (!status.upgradeAvailable) { + probe.disconnect(); + console.log(`[Firmware] ${device.name}: RouterBOOT already current (${before || 'unknown'})`); + return { ok: true }; + } + await this.setItem(item.id, { status: 'routerboot' }); + await probe.installRouterboardUpgrade(); // upgrades, then reboots + probe.disconnect(); + } catch (e) { + return { ok: false, error: (e as Error).message }; + } + + // Second reboot of this upgrade. Same budget as the first. + const deadline = Date.now() + REBOOT_TIMEOUT_MS; + while (Date.now() < deadline) { + if (this.cancelRequested) return { ok: false, error: 'cancelled while rebooting' }; + const probe = new DeviceCollector(device); + try { + await probe.connect(); + const after = await probe.checkRouterboardUpgrade(); + probe.disconnect(); + if (after.currentFirmware && before && after.currentFirmware === before) { + return { ok: false, error: `device rebooted but RouterBOOT still reports ${after.currentFirmware}` }; + } + await query( + `UPDATE devices SET routerboard_upgrade_available = FALSE, firmware_version = COALESCE(NULLIF($2,''), firmware_version) WHERE id = $1`, + [device.id, after.currentFirmware] + ); + console.log(`[Firmware] ${device.name}: RouterBOOT ${before || '?'} → ${after.currentFirmware || '?'}`); + return { ok: true }; + } catch { + probe.disconnect(); + await sleep(REBOOT_POLL_MS); + } + } + return { ok: false, error: `device did not come back within ${Math.round(REBOOT_TIMEOUT_MS / 60000)} minutes after the RouterBOOT upgrade` }; + } } export const firmwareOrchestrator = new FirmwareOrchestrator(); diff --git a/frontend/package.json b/frontend/package.json index d01bc8e..a1ebcdb 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mikrotik-manager-frontend", - "version": "0.23.9-beta", + "version": "0.23.10-beta", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/pages/DashboardPage.tsx b/frontend/src/pages/DashboardPage.tsx index a01041e..59cbd54 100644 --- a/frontend/src/pages/DashboardPage.tsx +++ b/frontend/src/pages/DashboardPage.tsx @@ -16,7 +16,7 @@ import type { OpsAttentionItem, OpsCapacityRow, OpsActivityItem } from '../servi import TerminalModal from '../components/TerminalModal'; import { useSocket } from '../hooks/useSocket'; import { useCanWrite } from '../hooks/useCanWrite'; -import { useQueryClient } from '@tanstack/react-query'; +import { useQueryClient, useMutation } from '@tanstack/react-query'; import { format, formatDistanceToNow } from 'date-fns'; import type { Device, DeviceEvent } from '../types'; import clsx from 'clsx'; @@ -608,6 +608,11 @@ function OperationsView({ refetchInterval: 60_000, }); const thingsToHandle: OpsAttentionItem[] = insights?.attention ?? []; + const dismissInsight = useMutation({ + mutationFn: (v: { fingerprint: string; hours: number; category?: string; title?: string }) => + operationsApi.dismissInsight(v.fingerprint, v.hours, v.category, v.title), + onSuccess: () => qc.invalidateQueries({ queryKey: ['ops-insights'] }), + }); const capacity: OpsCapacityRow[] = insights?.capacity ?? []; const activityAll: OpsActivityItem[] = insights?.activity ?? []; // The app's own write requests are audit-log noise from an operations point of @@ -840,6 +845,24 @@ function OperationsView({ > {item.action} → + {/* "Not now" rather than "never": an item that is genuinely resolved + stops being generated, so a dismissal is time-boxed (#102). */} + {item.fingerprint && ( + + )} )) : (

@@ -847,6 +870,11 @@ function OperationsView({ Nothing needs attention
)} + {(insights?.dismissedCount ?? 0) > 0 && ( +
+ {insights!.dismissedCount} snoozed — they return automatically if still true. +
+ )} diff --git a/frontend/src/pages/FirmwarePage.tsx b/frontend/src/pages/FirmwarePage.tsx index ce29c9b..cd1544c 100644 --- a/frontend/src/pages/FirmwarePage.tsx +++ b/frontend/src/pages/FirmwarePage.tsx @@ -2,7 +2,7 @@ import { useMemo, useState } from 'react'; import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import { ArrowUpCircle, RefreshCw, CheckCircle, XCircle, AlertTriangle, Clock, - HardDrive, ShieldAlert, Rocket, Ban, ChevronRight, + HardDrive, ShieldAlert, Rocket, Ban, ChevronRight, Cpu, } from 'lucide-react'; import clsx from 'clsx'; import { firmwareApi } from '../services/api'; @@ -73,6 +73,7 @@ function RolloutPanel({ rolloutId, canWrite }: { rolloutId: number; canWrite: bo {rollout.pre_backup && pre-backup} {rollout.halt_on_failure && halt on failure} + {rollout.routerboot_after && + RouterBOOT} {canWrite && active && (