feat: RouterBOOT in staged rollouts, and snoozeable insights (v0.23.10-beta)

Closes #113, #102.

#113 — RouterBOOT as a follow-on rollout step
  Firmware rollouts upgraded RouterOS and stopped, leaving the bootloader to be
  done by hand on every device — the exact task staged rollouts exist to remove.
  The data was already collected and the collector already had the two methods; the
  orchestrator simply never called them.

  An optional "Then RouterBOOT" step now runs per device after the RouterOS upgrade
  has been verified, rides out the second reboot, and confirms the firmware version
  actually moved.

  Ordering is deliberate: the bootloader ships inside the RouterOS package, so
  upgrading it first would apply the old one. It runs only once the OS half is
  confirmed good, and off by default, because it is a second flash and a second
  reboot per device.

  A device with nothing pending is a success rather than a failure — most of a fleet
  is already current, and treating "nothing to do" as an error would halt rollouts
  for no reason. When RouterBOOT does fail after a successful RouterOS upgrade the
  error says so explicitly, rather than reporting the device as simply failed and
  sending someone to re-run an upgrade it already has.

#102 — snooze an item in "Things to handle"
  Insights are recomputed from live state on every request and have no id, so a
  dismissal is keyed on a fingerprint of category, target and title. That survives a
  reworded title without swallowing a different device's version of the same issue.

  Dismissals expire, defaulting to 24 hours. "Handled" and "not now" look identical
  to the code, and an item that is genuinely resolved stops being generated anyway —
  so a permanent dismissal could only ever hide something still true. Expired rows
  are cleared on read, so a lapse needs no scheduled job, and the panel reports how
  many items are currently hidden rather than quietly shrinking.

Verified against the fleet: snoozing took the list from 8 items to 7 with
dismissedCount 1, and removing the dismissal restored it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/src/db/migrate.ts

@@ -603,6 +603,28 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_seccheck_supp_device
 -- (github issue #96).
 ALTER TABLE devices ADD COLUMN IF NOT EXISTS wifi_package VARCHAR(16);
 
+-- RouterBOOT as a follow-on step in a staged rollout (github issue #113).
+-- MikroTik's own guidance is to upgrade the bootloader after RouterOS, which until
+-- now meant visiting every device by hand — the exact task staged rollouts exist to
+-- remove.
+ALTER TABLE firmware_rollouts ADD COLUMN IF NOT EXISTS routerboot_after BOOLEAN NOT NULL DEFAULT FALSE;
+
+-- Temporarily dismissed "Things to handle" items (github issue #102).
+--
+-- Insights are recomputed from live state on every request and have no natural id,
+-- so a dismissal is keyed on a fingerprint of what the item is about. Dismissals
+-- expire: the point is "not now", not "never" — an item that is genuinely handled
+-- stops being produced, and one that is not should come back and ask again.
+CREATE TABLE IF NOT EXISTS insight_dismissals (
+  fingerprint  VARCHAR(160) PRIMARY KEY,
+  category     VARCHAR(40),
+  title        TEXT,
+  dismissed_by VARCHAR(50),
+  dismissed_at TIMESTAMPTZ DEFAULT NOW(),
+  expires_at   TIMESTAMPTZ NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_insight_dismissals_expiry ON insight_dismissals(expires_at);
+
 -- ntfy.sh notification channel (github issue #93). The channel type is a CHECK
 -- constraint rather than a lookup table, so widening it means replacing it.
 ALTER TABLE alert_channels DROP CONSTRAINT IF EXISTS alert_channels_type_check;

backend/src/routes/firmware.ts

@@ -50,8 +50,8 @@ router.post('/check-all', requireWrite, async (_req: Request, res: Response) =>
 
 // POST /api/firmware/rollouts — create a rollout (optionally scheduled)
 router.post('/rollouts', requireWrite, async (req: Request, res: Response) => {
-  const { name, halt_on_failure, pre_backup, scheduled_at, devices, start } = req.body as {
-    name?: string; halt_on_failure?: boolean; pre_backup?: boolean; scheduled_at?: string | null;
+  const { name, halt_on_failure, pre_backup, routerboot_after, scheduled_at, devices, start } = req.body as {
+    name?: string; halt_on_failure?: boolean; pre_backup?: boolean; routerboot_after?: boolean; scheduled_at?: string | null;
     devices?: { device_id: number; wave: number }[]; start?: boolean;
   };
   if (!name || !name.trim()) return res.status(400).json({ error: 'name is required' });
@@ -64,9 +64,9 @@ router.post('/rollouts', requireWrite, async (req: Request, res: Response) => {
   if (scheduled_at && isNaN(Date.parse(scheduled_at))) return res.status(400).json({ error: 'scheduled_at must be a valid timestamp' });
 
   const rollout = await queryOne<{ id: number }>(
-    `INSERT INTO firmware_rollouts (name, halt_on_failure, pre_backup, scheduled_at)
-     VALUES ($1,$2,$3,$4) RETURNING id`,
-    [name.trim().slice(0, 100), halt_on_failure !== false, pre_backup !== false, scheduled_at || null]);
+    `INSERT INTO firmware_rollouts (name, halt_on_failure, pre_backup, routerboot_after, scheduled_at)
+     VALUES ($1,$2,$3,$4,$5) RETURNING id`,
+    [name.trim().slice(0, 100), halt_on_failure !== false, pre_backup !== false, routerboot_after === true, scheduled_at || null]);
   for (const d of devices) {
     await query(
       `INSERT INTO firmware_rollout_devices (rollout_id, device_id, wave) VALUES ($1,$2,$3)`,

backend/src/routes/operations.ts

@@ -15,6 +15,20 @@ const bucket = process.env.INFLUXDB_BUCKET || 'mikrotik';
 type Sev = 'error' | 'warn' | 'info';
 interface AttentionItem {
   sev: Sev; category: string; title: string; body: string; action: string; path: string;
+  /** Stable identity for an item that is otherwise recomputed each request (#102). */
+  fingerprint?: string;
+}
+
+/**
+ * Identify an insight by what it is about rather than by its wording.
+ *
+ * Insights are derived from live state, so they have no id to dismiss. Keying on
+ * category plus the target path survives a reworded title — dismissing "sw1 is
+ * unreachable" should stay dismissed if the copy changes, but must NOT swallow a
+ * different device's outage.
+ */
+function fingerprintOf(item: AttentionItem): string {
+  return `${item.category}|${item.path}|${item.title}`.slice(0, 160);
 }
 
 // Latest value per device for a set of device_resources fields.
@@ -314,14 +328,62 @@ router.get('/insights', async (_req: Request, res: Response) => {
   const anomalies = await detectAnomalies();
   for (const a of anomalies) attention.push(a);
 
+  // Dismissals (#102). Expired rows are cleared on read, so a dismissal lapses
+  // without needing a scheduled job, and an item still true when it lapses simply
+  // reappears — which is the intended behaviour of "remind me later".
+  await query(`DELETE FROM insight_dismissals WHERE expires_at < NOW()`).catch(() => []);
+  const dismissed = new Set(
+    (await query<{ fingerprint: string }>(`SELECT fingerprint FROM insight_dismissals`)
+      .catch(() => [])).map((r) => r.fingerprint)
+  );
+
+  for (const item of attention) item.fingerprint = fingerprintOf(item);
+  const visible = attention.filter((i) => !dismissed.has(i.fingerprint!));
+
   // Severity ordering: error → warn → info
   const sevRank: Record<Sev, number> = { error: 0, warn: 1, info: 2 };
-  attention.sort((a, b) => sevRank[a.sev] - sevRank[b.sev]);
+  visible.sort((a, b) => sevRank[a.sev] - sevRank[b.sev]);
 
   // Activity feed — recent config changes, user actions, and notable events
   const activity = await buildActivity();
 
-  res.json({ attention, capacity, activity });
+  res.json({ attention: visible, dismissedCount: attention.length - visible.length, capacity, activity });
+});
+
+/**
+ * POST /api/operations/insights/dismiss — hide one item for a while.
+ *
+ * Time-boxed on purpose. "Handled" and "not now" look the same to the code, and an
+ * item that is genuinely resolved stops being generated anyway — so a dismissal that
+ * never lapsed would only ever hide things that are still true.
+ */
+router.post('/insights/dismiss', requireWrite, async (req: Request, res: Response) => {
+  const { fingerprint, category, title, hours } = req.body as
+    { fingerprint?: string; category?: string; title?: string; hours?: number };
+  if (!fingerprint) return res.status(400).json({ error: 'fingerprint is required' });
+
+  const h = Math.min(720, Math.max(1, Number(hours) || 24));
+  await query(
+    `INSERT INTO insight_dismissals (fingerprint, category, title, dismissed_by, expires_at)
+     VALUES ($1, $2, $3, $4, NOW() + ($5 || ' hours')::interval)
+     ON CONFLICT (fingerprint) DO UPDATE SET
+       expires_at = EXCLUDED.expires_at, dismissed_at = NOW(), dismissed_by = EXCLUDED.dismissed_by`,
+    [fingerprint.slice(0, 160), category ?? null, title ?? null,
+     (req as unknown as { user?: { username?: string } }).user?.username ?? null, String(h)]
+  );
+  res.status(201).json({ fingerprint, hours: h });
+});
+
+/** GET /api/operations/insights/dismissed — what is currently hidden, and until when. */
+router.get('/insights/dismissed', async (_req: Request, res: Response) => {
+  await query(`DELETE FROM insight_dismissals WHERE expires_at < NOW()`).catch(() => []);
+  res.json(await query(`SELECT * FROM insight_dismissals ORDER BY expires_at ASC`));
+});
+
+/** DELETE — bring an item back before its dismissal lapses. */
+router.delete('/insights/dismiss/:fingerprint', requireWrite, async (req: Request, res: Response) => {
+  await query(`DELETE FROM insight_dismissals WHERE fingerprint = $1`, [req.params.fingerprint]);
+  res.status(204).end();
 });
 
 // ─── Baseline anomaly detection ────────────────────────────────────────────────

backend/src/services/FirmwareOrchestrator.ts

@@ -21,6 +21,8 @@ const sleep = (ms: number) => new Promise<void>(r => setTimeout(r, ms));
 interface RolloutRow {
   id: number; name: string; status: string;
   halt_on_failure: boolean; pre_backup: boolean;
+  /** Follow the RouterOS upgrade with the pending RouterBOOT upgrade (issue #113). */
+  routerboot_after: boolean;
 }
 interface RolloutDeviceRow {
   id: number; rollout_id: number; device_id: number; wave: number; status: string;
@@ -199,6 +201,21 @@ export class FirmwareOrchestrator {
     if (newVersion && fromVersion && newVersion === fromVersion) {
       return fail(`Device rebooted but still reports ${newVersion} — the update did not apply`);
     }
+    // 5. RouterBOOT, if asked for (issue #113).
+    //
+    // Deliberately after the RouterOS upgrade and its verification: the bootloader
+    // ships inside the RouterOS package, so upgrading it first would apply the old
+    // one. This is a second flash and a second reboot, so it only runs once the OS
+    // half is confirmed good.
+    if (rollout.routerboot_after) {
+      const rbResult = await this.upgradeRouterboot(device, item);
+      if (!rbResult.ok) {
+        // The RouterOS upgrade did land. Say so, rather than reporting the device as
+        // simply failed and sending someone to re-run an upgrade it already has.
+        return fail(`RouterOS upgraded to ${newVersion || 'unknown'} successfully, but the RouterBOOT upgrade failed: ${rbResult.error}`);
+      }
+    }
+
     await this.setItem(item.id, { status: 'success', to_version: newVersion || null });
     await query(`UPDATE firmware_rollout_devices SET finished_at=NOW() WHERE id=$1`, [item.id]);
     await query(`UPDATE devices SET ros_version=COALESCE(NULLIF($2,''), ros_version), firmware_update_available=FALSE, status='online', last_seen=NOW() WHERE id=$1`,
@@ -206,6 +223,61 @@ export class FirmwareOrchestrator {
     console.log(`[Firmware] ${device.name}: upgraded to ${newVersion || 'unknown'}`);
     return true;
   }
+
+  /**
+   * Apply a pending RouterBOOT upgrade and ride out the reboot it causes.
+   *
+   * A device with nothing pending is a success, not a failure — most of a fleet will
+   * already be current, and treating "nothing to do" as an error would halt rollouts
+   * for no reason.
+   */
+  private async upgradeRouterboot(
+    device: DeviceRow,
+    item: RolloutDeviceRow
+  ): Promise<{ ok: true } | { ok: false; error: string }> {
+    let before: string;
+    try {
+      const probe = new DeviceCollector(device);
+      await probe.connect();
+      const status = await probe.checkRouterboardUpgrade();
+      before = status.currentFirmware;
+      if (!status.upgradeAvailable) {
+        probe.disconnect();
+        console.log(`[Firmware] ${device.name}: RouterBOOT already current (${before || 'unknown'})`);
+        return { ok: true };
+      }
+      await this.setItem(item.id, { status: 'routerboot' });
+      await probe.installRouterboardUpgrade();   // upgrades, then reboots
+      probe.disconnect();
+    } catch (e) {
+      return { ok: false, error: (e as Error).message };
+    }
+
+    // Second reboot of this upgrade. Same budget as the first.
+    const deadline = Date.now() + REBOOT_TIMEOUT_MS;
+    while (Date.now() < deadline) {
+      if (this.cancelRequested) return { ok: false, error: 'cancelled while rebooting' };
+      const probe = new DeviceCollector(device);
+      try {
+        await probe.connect();
+        const after = await probe.checkRouterboardUpgrade();
+        probe.disconnect();
+        if (after.currentFirmware && before && after.currentFirmware === before) {
+          return { ok: false, error: `device rebooted but RouterBOOT still reports ${after.currentFirmware}` };
+        }
+        await query(
+          `UPDATE devices SET routerboard_upgrade_available = FALSE, firmware_version = COALESCE(NULLIF($2,''), firmware_version) WHERE id = $1`,
+          [device.id, after.currentFirmware]
+        );
+        console.log(`[Firmware] ${device.name}: RouterBOOT ${before || '?'} → ${after.currentFirmware || '?'}`);
+        return { ok: true };
+      } catch {
+        probe.disconnect();
+        await sleep(REBOOT_POLL_MS);
+      }
+    }
+    return { ok: false, error: `device did not come back within ${Math.round(REBOOT_TIMEOUT_MS / 60000)} minutes after the RouterBOOT upgrade` };
+  }
 }
 
 export const firmwareOrchestrator = new FirmwareOrchestrator();