feat: RouterBOOT in staged rollouts, and snoozeable insights (v0.23.10-beta)

Closes #113, #102.

#113 — RouterBOOT as a follow-on rollout step
  Firmware rollouts upgraded RouterOS and stopped, leaving the bootloader to be
  done by hand on every device — the exact task staged rollouts exist to remove.
  The data was already collected and the collector already had the two methods; the
  orchestrator simply never called them.

  An optional "Then RouterBOOT" step now runs per device after the RouterOS upgrade
  has been verified, rides out the second reboot, and confirms the firmware version
  actually moved.

  Ordering is deliberate: the bootloader ships inside the RouterOS package, so
  upgrading it first would apply the old one. It runs only once the OS half is
  confirmed good, and off by default, because it is a second flash and a second
  reboot per device.

  A device with nothing pending is a success rather than a failure — most of a fleet
  is already current, and treating "nothing to do" as an error would halt rollouts
  for no reason. When RouterBOOT does fail after a successful RouterOS upgrade the
  error says so explicitly, rather than reporting the device as simply failed and
  sending someone to re-run an upgrade it already has.

#102 — snooze an item in "Things to handle"
  Insights are recomputed from live state on every request and have no id, so a
  dismissal is keyed on a fingerprint of category, target and title. That survives a
  reworded title without swallowing a different device's version of the same issue.

  Dismissals expire, defaulting to 24 hours. "Handled" and "not now" look identical
  to the code, and an item that is genuinely resolved stops being generated anyway —
  so a permanent dismissal could only ever hide something still true. Expired rows
  are cleared on read, so a lapse needs no scheduled job, and the panel reports how
  many items are currently hidden rather than quietly shrinking.

Verified against the fleet: snoozing took the list from 8 items to 7 with
dismissedCount 1, and removing the dismissal restored it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2GT-Rich
2026-08-22 20:42:16 -07:00
co-authored by Claude Opus 5
parent 60d7d13bd3
commit 37887d38c2
11 changed files with 221 additions and 15 deletions
+1 -1
View File
@@ -6,7 +6,7 @@
</p>
<p align="center">
<img src="https://img.shields.io/badge/version-0.23.9_Beta-blue" alt="Version" />
<img src="https://img.shields.io/badge/version-0.23.10_Beta-blue" alt="Version" />
<img src="https://img.shields.io/badge/license-AGPLv3-blue" alt="License" />
<img src="https://img.shields.io/badge/docker-compose-2496ED?logo=docker&logoColor=white" alt="Docker" />
<img src="https://img.shields.io/badge/TypeScript-5.3-3178C6?logo=typescript&logoColor=white" alt="TypeScript" />
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "mikrotik-manager-backend",
"version": "0.23.9-beta",
"version": "0.23.10-beta",
"description": "Mikrotik Network Management Platform - Backend API",
"main": "dist/index.js",
"scripts": {
+22
View File
@@ -603,6 +603,28 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_seccheck_supp_device
-- (github issue #96).
ALTER TABLE devices ADD COLUMN IF NOT EXISTS wifi_package VARCHAR(16);
-- RouterBOOT as a follow-on step in a staged rollout (github issue #113).
-- MikroTik's own guidance is to upgrade the bootloader after RouterOS, which until
-- now meant visiting every device by hand — the exact task staged rollouts exist to
-- remove.
ALTER TABLE firmware_rollouts ADD COLUMN IF NOT EXISTS routerboot_after BOOLEAN NOT NULL DEFAULT FALSE;
-- Temporarily dismissed "Things to handle" items (github issue #102).
--
-- Insights are recomputed from live state on every request and have no natural id,
-- so a dismissal is keyed on a fingerprint of what the item is about. Dismissals
-- expire: the point is "not now", not "never" — an item that is genuinely handled
-- stops being produced, and one that is not should come back and ask again.
CREATE TABLE IF NOT EXISTS insight_dismissals (
fingerprint VARCHAR(160) PRIMARY KEY,
category VARCHAR(40),
title TEXT,
dismissed_by VARCHAR(50),
dismissed_at TIMESTAMPTZ DEFAULT NOW(),
expires_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_insight_dismissals_expiry ON insight_dismissals(expires_at);
-- ntfy.sh notification channel (github issue #93). The channel type is a CHECK
-- constraint rather than a lookup table, so widening it means replacing it.
ALTER TABLE alert_channels DROP CONSTRAINT IF EXISTS alert_channels_type_check;
+5 -5
View File
@@ -50,8 +50,8 @@ router.post('/check-all', requireWrite, async (_req: Request, res: Response) =>
// POST /api/firmware/rollouts — create a rollout (optionally scheduled)
router.post('/rollouts', requireWrite, async (req: Request, res: Response) => {
const { name, halt_on_failure, pre_backup, scheduled_at, devices, start } = req.body as {
name?: string; halt_on_failure?: boolean; pre_backup?: boolean; scheduled_at?: string | null;
const { name, halt_on_failure, pre_backup, routerboot_after, scheduled_at, devices, start } = req.body as {
name?: string; halt_on_failure?: boolean; pre_backup?: boolean; routerboot_after?: boolean; scheduled_at?: string | null;
devices?: { device_id: number; wave: number }[]; start?: boolean;
};
if (!name || !name.trim()) return res.status(400).json({ error: 'name is required' });
@@ -64,9 +64,9 @@ router.post('/rollouts', requireWrite, async (req: Request, res: Response) => {
if (scheduled_at && isNaN(Date.parse(scheduled_at))) return res.status(400).json({ error: 'scheduled_at must be a valid timestamp' });
const rollout = await queryOne<{ id: number }>(
`INSERT INTO firmware_rollouts (name, halt_on_failure, pre_backup, scheduled_at)
VALUES ($1,$2,$3,$4) RETURNING id`,
[name.trim().slice(0, 100), halt_on_failure !== false, pre_backup !== false, scheduled_at || null]);
`INSERT INTO firmware_rollouts (name, halt_on_failure, pre_backup, routerboot_after, scheduled_at)
VALUES ($1,$2,$3,$4,$5) RETURNING id`,
[name.trim().slice(0, 100), halt_on_failure !== false, pre_backup !== false, routerboot_after === true, scheduled_at || null]);
for (const d of devices) {
await query(
`INSERT INTO firmware_rollout_devices (rollout_id, device_id, wave) VALUES ($1,$2,$3)`,
+64 -2
View File
@@ -15,6 +15,20 @@ const bucket = process.env.INFLUXDB_BUCKET || 'mikrotik';
type Sev = 'error' | 'warn' | 'info';
interface AttentionItem {
sev: Sev; category: string; title: string; body: string; action: string; path: string;
/** Stable identity for an item that is otherwise recomputed each request (#102). */
fingerprint?: string;
}
/**
* Identify an insight by what it is about rather than by its wording.
*
* Insights are derived from live state, so they have no id to dismiss. Keying on
* category plus the target path survives a reworded title — dismissing "sw1 is
* unreachable" should stay dismissed if the copy changes, but must NOT swallow a
* different device's outage.
*/
function fingerprintOf(item: AttentionItem): string {
return `${item.category}|${item.path}|${item.title}`.slice(0, 160);
}
// Latest value per device for a set of device_resources fields.
@@ -314,14 +328,62 @@ router.get('/insights', async (_req: Request, res: Response) => {
const anomalies = await detectAnomalies();
for (const a of anomalies) attention.push(a);
// Dismissals (#102). Expired rows are cleared on read, so a dismissal lapses
// without needing a scheduled job, and an item still true when it lapses simply
// reappears — which is the intended behaviour of "remind me later".
await query(`DELETE FROM insight_dismissals WHERE expires_at < NOW()`).catch(() => []);
const dismissed = new Set(
(await query<{ fingerprint: string }>(`SELECT fingerprint FROM insight_dismissals`)
.catch(() => [])).map((r) => r.fingerprint)
);
for (const item of attention) item.fingerprint = fingerprintOf(item);
const visible = attention.filter((i) => !dismissed.has(i.fingerprint!));
// Severity ordering: error → warn → info
const sevRank: Record<Sev, number> = { error: 0, warn: 1, info: 2 };
attention.sort((a, b) => sevRank[a.sev] - sevRank[b.sev]);
visible.sort((a, b) => sevRank[a.sev] - sevRank[b.sev]);
// Activity feed — recent config changes, user actions, and notable events
const activity = await buildActivity();
res.json({ attention, capacity, activity });
res.json({ attention: visible, dismissedCount: attention.length - visible.length, capacity, activity });
});
/**
* POST /api/operations/insights/dismiss — hide one item for a while.
*
* Time-boxed on purpose. "Handled" and "not now" look the same to the code, and an
* item that is genuinely resolved stops being generated anyway — so a dismissal that
* never lapsed would only ever hide things that are still true.
*/
router.post('/insights/dismiss', requireWrite, async (req: Request, res: Response) => {
const { fingerprint, category, title, hours } = req.body as
{ fingerprint?: string; category?: string; title?: string; hours?: number };
if (!fingerprint) return res.status(400).json({ error: 'fingerprint is required' });
const h = Math.min(720, Math.max(1, Number(hours) || 24));
await query(
`INSERT INTO insight_dismissals (fingerprint, category, title, dismissed_by, expires_at)
VALUES ($1, $2, $3, $4, NOW() + ($5 || ' hours')::interval)
ON CONFLICT (fingerprint) DO UPDATE SET
expires_at = EXCLUDED.expires_at, dismissed_at = NOW(), dismissed_by = EXCLUDED.dismissed_by`,
[fingerprint.slice(0, 160), category ?? null, title ?? null,
(req as unknown as { user?: { username?: string } }).user?.username ?? null, String(h)]
);
res.status(201).json({ fingerprint, hours: h });
});
/** GET /api/operations/insights/dismissed — what is currently hidden, and until when. */
router.get('/insights/dismissed', async (_req: Request, res: Response) => {
await query(`DELETE FROM insight_dismissals WHERE expires_at < NOW()`).catch(() => []);
res.json(await query(`SELECT * FROM insight_dismissals ORDER BY expires_at ASC`));
});
/** DELETE — bring an item back before its dismissal lapses. */
router.delete('/insights/dismiss/:fingerprint', requireWrite, async (req: Request, res: Response) => {
await query(`DELETE FROM insight_dismissals WHERE fingerprint = $1`, [req.params.fingerprint]);
res.status(204).end();
});
// ─── Baseline anomaly detection ────────────────────────────────────────────────
@@ -21,6 +21,8 @@ const sleep = (ms: number) => new Promise<void>(r => setTimeout(r, ms));
interface RolloutRow {
id: number; name: string; status: string;
halt_on_failure: boolean; pre_backup: boolean;
/** Follow the RouterOS upgrade with the pending RouterBOOT upgrade (issue #113). */
routerboot_after: boolean;
}
interface RolloutDeviceRow {
id: number; rollout_id: number; device_id: number; wave: number; status: string;
@@ -199,6 +201,21 @@ export class FirmwareOrchestrator {
if (newVersion && fromVersion && newVersion === fromVersion) {
return fail(`Device rebooted but still reports ${newVersion} — the update did not apply`);
}
// 5. RouterBOOT, if asked for (issue #113).
//
// Deliberately after the RouterOS upgrade and its verification: the bootloader
// ships inside the RouterOS package, so upgrading it first would apply the old
// one. This is a second flash and a second reboot, so it only runs once the OS
// half is confirmed good.
if (rollout.routerboot_after) {
const rbResult = await this.upgradeRouterboot(device, item);
if (!rbResult.ok) {
// The RouterOS upgrade did land. Say so, rather than reporting the device as
// simply failed and sending someone to re-run an upgrade it already has.
return fail(`RouterOS upgraded to ${newVersion || 'unknown'} successfully, but the RouterBOOT upgrade failed: ${rbResult.error}`);
}
}
await this.setItem(item.id, { status: 'success', to_version: newVersion || null });
await query(`UPDATE firmware_rollout_devices SET finished_at=NOW() WHERE id=$1`, [item.id]);
await query(`UPDATE devices SET ros_version=COALESCE(NULLIF($2,''), ros_version), firmware_update_available=FALSE, status='online', last_seen=NOW() WHERE id=$1`,
@@ -206,6 +223,61 @@ export class FirmwareOrchestrator {
console.log(`[Firmware] ${device.name}: upgraded to ${newVersion || 'unknown'}`);
return true;
}
/**
* Apply a pending RouterBOOT upgrade and ride out the reboot it causes.
*
* A device with nothing pending is a success, not a failure — most of a fleet will
* already be current, and treating "nothing to do" as an error would halt rollouts
* for no reason.
*/
private async upgradeRouterboot(
device: DeviceRow,
item: RolloutDeviceRow
): Promise<{ ok: true } | { ok: false; error: string }> {
let before: string;
try {
const probe = new DeviceCollector(device);
await probe.connect();
const status = await probe.checkRouterboardUpgrade();
before = status.currentFirmware;
if (!status.upgradeAvailable) {
probe.disconnect();
console.log(`[Firmware] ${device.name}: RouterBOOT already current (${before || 'unknown'})`);
return { ok: true };
}
await this.setItem(item.id, { status: 'routerboot' });
await probe.installRouterboardUpgrade(); // upgrades, then reboots
probe.disconnect();
} catch (e) {
return { ok: false, error: (e as Error).message };
}
// Second reboot of this upgrade. Same budget as the first.
const deadline = Date.now() + REBOOT_TIMEOUT_MS;
while (Date.now() < deadline) {
if (this.cancelRequested) return { ok: false, error: 'cancelled while rebooting' };
const probe = new DeviceCollector(device);
try {
await probe.connect();
const after = await probe.checkRouterboardUpgrade();
probe.disconnect();
if (after.currentFirmware && before && after.currentFirmware === before) {
return { ok: false, error: `device rebooted but RouterBOOT still reports ${after.currentFirmware}` };
}
await query(
`UPDATE devices SET routerboard_upgrade_available = FALSE, firmware_version = COALESCE(NULLIF($2,''), firmware_version) WHERE id = $1`,
[device.id, after.currentFirmware]
);
console.log(`[Firmware] ${device.name}: RouterBOOT ${before || '?'} → ${after.currentFirmware || '?'}`);
return { ok: true };
} catch {
probe.disconnect();
await sleep(REBOOT_POLL_MS);
}
}
return { ok: false, error: `device did not come back within ${Math.round(REBOOT_TIMEOUT_MS / 60000)} minutes after the RouterBOOT upgrade` };
}
}
export const firmwareOrchestrator = new FirmwareOrchestrator();
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "mikrotik-manager-frontend",
"version": "0.23.9-beta",
"version": "0.23.10-beta",
"private": true,
"type": "module",
"scripts": {
+29 -1
View File
@@ -16,7 +16,7 @@ import type { OpsAttentionItem, OpsCapacityRow, OpsActivityItem } from '../servi
import TerminalModal from '../components/TerminalModal';
import { useSocket } from '../hooks/useSocket';
import { useCanWrite } from '../hooks/useCanWrite';
import { useQueryClient } from '@tanstack/react-query';
import { useQueryClient, useMutation } from '@tanstack/react-query';
import { format, formatDistanceToNow } from 'date-fns';
import type { Device, DeviceEvent } from '../types';
import clsx from 'clsx';
@@ -608,6 +608,11 @@ function OperationsView({
refetchInterval: 60_000,
});
const thingsToHandle: OpsAttentionItem[] = insights?.attention ?? [];
const dismissInsight = useMutation({
mutationFn: (v: { fingerprint: string; hours: number; category?: string; title?: string }) =>
operationsApi.dismissInsight(v.fingerprint, v.hours, v.category, v.title),
onSuccess: () => qc.invalidateQueries({ queryKey: ['ops-insights'] }),
});
const capacity: OpsCapacityRow[] = insights?.capacity ?? [];
const activityAll: OpsActivityItem[] = insights?.activity ?? [];
// The app's own write requests are audit-log noise from an operations point of
@@ -840,6 +845,24 @@ function OperationsView({
>
{item.action} →
</button>
{/* "Not now" rather than "never": an item that is genuinely resolved
stops being generated, so a dismissal is time-boxed (#102). */}
{item.fingerprint && (
<button
onClick={() => dismissInsight.mutate({
fingerprint: item.fingerprint!, hours: 24,
category: item.category, title: item.title,
})}
disabled={dismissInsight.isPending}
title="Hide this for 24 hours. It returns if it is still true."
className="text-[11.5px] rounded-[5px] px-2 py-[6px] flex-shrink-0 transition-colors"
style={{ background: 'transparent', border: '1px solid var(--line)', color: 'var(--ink-3)' }}
onMouseEnter={(e) => (e.currentTarget.style.background = 'var(--surface-3)')}
onMouseLeave={(e) => (e.currentTarget.style.background = 'transparent')}
>
Snooze
</button>
)}
</div>
)) : (
<div className="flex items-center justify-center gap-2 py-8" style={{ color: 'var(--ink-3)' }}>
@@ -847,6 +870,11 @@ function OperationsView({
<span className="text-[13px]">Nothing needs attention</span>
</div>
)}
{(insights?.dismissedCount ?? 0) > 0 && (
<div className="px-5 py-2 text-[11.5px]" style={{ color: 'var(--ink-3)', borderTop: '1px solid var(--line-soft)' }}>
{insights!.dismissedCount} snoozed — they return automatically if still true.
</div>
)}
</div>
</div>
+12 -1
View File
@@ -2,7 +2,7 @@ import { useMemo, useState } from 'react';
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
import {
ArrowUpCircle, RefreshCw, CheckCircle, XCircle, AlertTriangle, Clock,
HardDrive, ShieldAlert, Rocket, Ban, ChevronRight,
HardDrive, ShieldAlert, Rocket, Ban, ChevronRight, Cpu,
} from 'lucide-react';
import clsx from 'clsx';
import { firmwareApi } from '../services/api';
@@ -73,6 +73,7 @@ function RolloutPanel({ rolloutId, canWrite }: { rolloutId: number; canWrite: bo
<span className="ml-auto flex items-center gap-3 text-xs text-gray-400 dark:text-slate-500">
{rollout.pre_backup && <span className="flex items-center gap-1"><HardDrive className="w-3 h-3" />pre-backup</span>}
{rollout.halt_on_failure && <span className="flex items-center gap-1"><ShieldAlert className="w-3 h-3" />halt on failure</span>}
{rollout.routerboot_after && <span className="flex items-center gap-1"><Cpu className="w-3 h-3" />+ RouterBOOT</span>}
{canWrite && active && (
<button onClick={() => cancelMutation.mutate()} disabled={cancelMutation.isPending}
className="flex items-center gap-1 px-2 py-1 rounded-lg text-red-600 dark:text-red-400 border border-red-200 dark:border-red-800 hover:bg-red-50 dark:hover:bg-red-900/20 transition-colors">
@@ -124,6 +125,8 @@ export default function FirmwarePage() {
const [rolloutName, setRolloutName] = useState('');
const [preBackup, setPreBackup] = useState(true);
const [haltOnFailure, setHaltOnFailure] = useState(true);
// Off by default: it is a second flash and a second reboot per device (#113).
const [routerbootAfter, setRouterbootAfter] = useState(false);
const [scheduleAt, setScheduleAt] = useState('');
const [viewRolloutId, setViewRolloutId] = useState<number | null>(null);
const [checkResult, setCheckResult] = useState<string | null>(null);
@@ -170,6 +173,7 @@ export default function FirmwarePage() {
name: rolloutName.trim() || `RouterOS upgrade ${new Date().toISOString().slice(0, 10)}`,
halt_on_failure: haltOnFailure,
pre_backup: preBackup,
routerboot_after: routerbootAfter,
scheduled_at: scheduleAt ? new Date(scheduleAt).toISOString() : null,
start: !scheduleAt,
devices: [...selected.entries()].map(([device_id, wave]) => ({ device_id, wave })),
@@ -345,6 +349,13 @@ export default function FirmwarePage() {
<input type="checkbox" className="w-4 h-4 rounded" checked={haltOnFailure} onChange={e => setHaltOnFailure(e.target.checked)} />
Halt on failure
</label>
<label
className="flex items-center gap-1.5 text-sm text-gray-700 dark:text-slate-300 cursor-pointer"
title="After each RouterOS upgrade verifies, apply any pending RouterBOOT upgrade. This is a second flash and a second reboot per device."
>
<input type="checkbox" className="w-4 h-4 rounded" checked={routerbootAfter} onChange={e => setRouterbootAfter(e.target.checked)} />
Then RouterBOOT
</label>
<label className="flex items-center gap-1.5 text-sm text-gray-700 dark:text-slate-300">
<Clock className="w-4 h-4 text-gray-400" />
<input type="datetime-local" className="input py-1 text-xs w-auto" value={scheduleAt} onChange={e => setScheduleAt(e.target.value)} />
+13 -2
View File
@@ -597,7 +597,7 @@ export interface FirmwareDeviceRow {
}
export interface FirmwareRollout {
id: number; name: string; status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
halt_on_failure: boolean; pre_backup: boolean;
halt_on_failure: boolean; pre_backup: boolean; routerboot_after: boolean;
scheduled_at: string | null; started_at: string | null; finished_at: string | null; created_at: string;
device_count?: number; success_count?: number; failed_count?: number;
}
@@ -616,7 +616,7 @@ export const firmwareApi = {
api.post<{ results: { name: string; ok: boolean; installed?: string; latest?: string; available?: boolean; error?: string }[] }>(
'/firmware/check-all', undefined, { timeout: 180_000 }),
createRollout: (data: {
name: string; halt_on_failure: boolean; pre_backup: boolean;
name: string; halt_on_failure: boolean; pre_backup: boolean; routerboot_after?: boolean;
scheduled_at?: string | null; start?: boolean;
devices: { device_id: number; wave: number }[];
}) => api.post<{ id: number }>('/firmware/rollouts', data),
@@ -637,6 +637,8 @@ export interface OpsAttentionItem {
body: string;
action: string;
path: string;
/** Stable identity used to dismiss an item that is otherwise recomputed. */
fingerprint?: string;
}
export interface OpsCapacityRow { id: number; name: string; cpu: number; mem_pct: number }
export interface OpsActivityItem {
@@ -648,6 +650,8 @@ export interface OpsActivityItem {
}
export interface OpsInsights {
attention: OpsAttentionItem[];
/** How many items are currently hidden by a dismissal. */
dismissedCount?: number;
capacity: OpsCapacityRow[];
activity: OpsActivityItem[];
}
@@ -655,6 +659,13 @@ export interface OpsActionResult { total: number; results: { name: string; ok: b
export const operationsApi = {
insights: () => api.get<OpsInsights>('/operations/insights'),
dismissInsight: (fingerprint: string, hours: number, category?: string, title?: string) =>
api.post('/operations/insights/dismiss', { fingerprint, hours, category, title }),
listDismissed: () =>
api.get<{ fingerprint: string; category: string | null; title: string | null; expires_at: string }[]>(
'/operations/insights/dismissed'),
undismissInsight: (fingerprint: string) =>
api.delete(`/operations/insights/dismiss/${encodeURIComponent(fingerprint)}`),
backupAll: () => api.post<OpsActionResult>('/operations/backup-all', undefined, { timeout: 180_000 }),
syncAll: () => api.post<OpsActionResult>('/operations/sync-all', undefined, { timeout: 180_000 }),
};
+1 -1
View File
@@ -1 +1 @@
export const APP_VERSION = 'v0.23.9 Beta';
export const APP_VERSION = 'v0.23.10 Beta';