feat(commands): run one command across a fleet, in waves (v0.23.27-beta)
Closes #118. Unblocked by #110, which is what makes SSH on sixty devices practical. Arbitrary RouterOS console commands over SSH, because `:put`, `:foreach` and `/interface print` are console syntax that the binary API cannot express, and because a curated command library cannot cover "add a channel to LACP" or "rewrite a WireGuard endpoint" — the things people actually asked for. **Waves are the feature, not the safety rail around it.** A shell loop already runs a command on many hosts, costs nothing, and prints errors as they happen. What it cannot do is stop. Devices run together within a wave, waves run strictly in sequence, and halt-on-failure ends the run before a mistake becomes sixty mistakes. Without that, this would be a worse bash. Proved on hardware rather than argued: a bad command on two devices, one per wave, failed on the first and left the second `skipped`. RouterOS reports most errors in output text rather than an exit code, so output is read for failure markers — otherwise a rejected command and a successful one are indistinguishable and the run would sail on. Conservatively: only unambiguous markers count, since misreading ordinary output would halt a run that was working. Change Guard wraps each device by default, so one that stops answering after the command restores itself. It is one click to turn off, because a sharp tool that refuses to cut is not a tool — the requester's argument, and a fair one. The combination that gets an explicit acknowledgement is a command that can sever management *with the guard disabled*; that is the only case where nothing catches a mistake. A preview names devices, wave assignment, and the reasons a command looks capable of cutting management access. It warns, never blocks. Devices with no SSH credential are listed before the run rather than producing a wave of identical authentication failures. Per-device output is captured and kept, because "it failed" without the device's own words is not actionable, and because some commands exist purely for output. Verified end to end against real hardware: a read-only command across two devices, halt-on-failure stopping a bad command after one device, and a genuine write applied under Change Guard, confirmed, then reverted — with both devices returned to their original configuration afterwards. Also adds services/sshExec.ts, shared with the key work: it selects a verified key or a password, and throws rather than swallowing. That last part is deliberate — a swallowed error is exactly how the data-cap feature spent a week doing nothing while looking healthy. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2GT-Rich
co-authored by
Claude Opus 5
Claude Opus 5
parent
19e2d914f5
commit
0e364b1558
@@ -878,6 +878,47 @@ CREATE INDEX IF NOT EXISTS idx_data_cap_sends ON lte_data_cap_sends(device_id, a
|
||||
-- The password is retained alongside. Key auth falling back to a password is a
|
||||
-- degraded state worth knowing about; key auth failing with no fallback is an
|
||||
-- outage.
|
||||
-- Bulk command execution (#118).
|
||||
--
|
||||
-- Waves are not a nicety. Running one command across sixty devices in parallel
|
||||
-- means a mistake reaches all sixty before the first failure is visible, and the
|
||||
-- whole reason to do this here rather than in a shell loop is that this one can
|
||||
-- stop. wave_size and halt_on_failure are what make that possible.
|
||||
CREATE TABLE IF NOT EXISTS command_runs (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(120) NOT NULL,
|
||||
command TEXT NOT NULL,
|
||||
wave_size INTEGER NOT NULL DEFAULT 1,
|
||||
halt_on_failure BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
-- Change Guard restores a device that stops answering after the command.
|
||||
-- Defaults on; an operator may turn it off deliberately, per run.
|
||||
use_change_guard BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
-- pending | running | completed | failed | halted | cancelled
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'pending',
|
||||
created_by VARCHAR(50),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
started_at TIMESTAMPTZ,
|
||||
finished_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
-- Output is kept per device because "it failed" without the device's own words
|
||||
-- is not actionable, and because some commands exist purely for their output.
|
||||
CREATE TABLE IF NOT EXISTS command_run_devices (
|
||||
id SERIAL PRIMARY KEY,
|
||||
run_id INTEGER NOT NULL REFERENCES command_runs(id) ON DELETE CASCADE,
|
||||
device_id INTEGER NOT NULL REFERENCES devices(id) ON DELETE CASCADE,
|
||||
wave INTEGER NOT NULL DEFAULT 1,
|
||||
-- pending | running | success | failed | skipped | reverted
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'pending',
|
||||
output TEXT,
|
||||
error TEXT,
|
||||
guarded BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
auto_reverted BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
started_at TIMESTAMPTZ,
|
||||
finished_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_command_run_devices ON command_run_devices(run_id, wave);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS device_ssh_keys (
|
||||
device_id INTEGER PRIMARY KEY REFERENCES devices(id) ON DELETE CASCADE,
|
||||
key_type VARCHAR(24) NOT NULL DEFAULT 'ed25519',
|
||||
|
||||
@@ -62,6 +62,7 @@ import wirelessRoutes from './routes/wireless';
|
||||
import networkServicesRoutes from './routes/networkServices';
|
||||
import trafficAnalyticsRoutes from './routes/trafficAnalytics';
|
||||
import credentialPresetsRoutes from './routes/credentialPresets';
|
||||
import commandRoutes from './routes/commands';
|
||||
import systemRoutes, { setPollerService as setSystemPoller } from './routes/system';
|
||||
import { auditMiddleware } from './middleware/auditMiddleware';
|
||||
|
||||
@@ -327,6 +328,7 @@ app.use('/api/maintenance-windows', maintenanceWindowsRoutes);
|
||||
app.use('/api/config-templates', configTemplatesRoutes);
|
||||
app.use('/api/config-history', configHistoryRoutes);
|
||||
app.use('/api/system', systemRoutes);
|
||||
app.use('/api/commands', commandRoutes);
|
||||
|
||||
// ─── Error Handler ────────────────────────────────────────────────────────────
|
||||
app.use(errorHandler);
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
/**
|
||||
* Bulk command execution across devices (#118).
|
||||
*
|
||||
* The dry run is not decoration. This endpoint family can misconfigure an entire
|
||||
* fleet in one call, and the difference between a tool and an accident is being
|
||||
* able to see what will happen first.
|
||||
*/
|
||||
import { Router } from 'express';
|
||||
import type { Request, Response } from 'express';
|
||||
import { requireAuth, requireWrite } from '../middleware/auth';
|
||||
import { query, queryOne } from '../config/database';
|
||||
import { commandRunner } from '../services/CommandRunner';
|
||||
|
||||
const router = Router();
|
||||
router.use(requireAuth);
|
||||
|
||||
/** Commands that change how a device is reached, and so deserve a warning. */
|
||||
const RISKY_PATTERNS: { pattern: RegExp; why: string }[] = [
|
||||
{ pattern: /\/ip\/?\s*address\s+(remove|set)/i, why: 'changes IP addressing, which can remove the management address' },
|
||||
{ pattern: /\/ip\/?\s*route\s+(remove|set)/i, why: 'changes routing, which can cut the path back to this server' },
|
||||
{ pattern: /\/ip\/?\s*firewall/i, why: 'edits firewall rules, which can block management access' },
|
||||
{ pattern: /\/interface\/?\s*\w*\s*(disable|remove)/i, why: 'disables or removes an interface, possibly the one in use' },
|
||||
{ pattern: /\/system\/?\s*(reboot|shutdown|reset-configuration)/i, why: 'reboots, shuts down or resets the device' },
|
||||
{ pattern: /\/user\s+(remove|set|disable)/i, why: 'changes user accounts, which can revoke this server’s access' },
|
||||
{ pattern: /\/ip\/?\s*service\s+(disable|set|remove)/i, why: 'changes management services such as the API or SSH' },
|
||||
];
|
||||
|
||||
function assessCommand(command: string): { risky: boolean; reasons: string[] } {
|
||||
const reasons = RISKY_PATTERNS.filter((r) => r.pattern.test(command)).map((r) => r.why);
|
||||
return { risky: reasons.length > 0, reasons };
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/commands/preview — what would happen, without doing it.
|
||||
*
|
||||
* Returns the devices, their wave assignment, and any reason the command looks
|
||||
* capable of severing management. Warnings only: the operator decides.
|
||||
*/
|
||||
router.post('/preview', async (req: Request, res: Response) => {
|
||||
const { command, device_ids, wave_size } = req.body as
|
||||
{ command?: string; device_ids?: number[]; wave_size?: number };
|
||||
|
||||
if (!command?.trim()) return res.status(400).json({ error: 'command is required' });
|
||||
if (!Array.isArray(device_ids) || device_ids.length === 0) {
|
||||
return res.status(400).json({ error: 'device_ids array is required' });
|
||||
}
|
||||
|
||||
const size = Math.max(1, Math.min(50, Number(wave_size) || 1));
|
||||
const devices = await query<{ id: number; name: string; ip_address: string; status: string }>(
|
||||
`SELECT id, name, ip_address, status FROM devices WHERE id = ANY($1::int[]) ORDER BY name`,
|
||||
[device_ids]
|
||||
);
|
||||
|
||||
// Devices without any SSH credential cannot be reached at all; saying so now
|
||||
// is better than a wave of identical authentication failures.
|
||||
const unreachable = await query<{ id: number; name: string }>(
|
||||
`SELECT d.id, d.name FROM devices d
|
||||
WHERE d.id = ANY($1::int[])
|
||||
AND d.ssh_username IS NULL AND d.api_username IS NULL`,
|
||||
[device_ids]
|
||||
);
|
||||
|
||||
res.json({
|
||||
command,
|
||||
...assessCommand(command),
|
||||
wave_size: size,
|
||||
devices: devices.map((d, i) => ({ ...d, wave: Math.floor(i / size) + 1 })),
|
||||
waves: Math.ceil(devices.length / size),
|
||||
unreachable,
|
||||
});
|
||||
});
|
||||
|
||||
// POST /api/commands/runs — create a run, optionally starting it immediately
|
||||
router.post('/runs', requireWrite, async (req: Request, res: Response) => {
|
||||
const {
|
||||
name, command, device_ids, wave_size, halt_on_failure, use_change_guard, start,
|
||||
} = req.body as {
|
||||
name?: string; command?: string; device_ids?: number[]; wave_size?: number;
|
||||
halt_on_failure?: boolean; use_change_guard?: boolean; start?: boolean;
|
||||
};
|
||||
|
||||
if (!command?.trim()) return res.status(400).json({ error: 'command is required' });
|
||||
if (!Array.isArray(device_ids) || device_ids.length === 0) {
|
||||
return res.status(400).json({ error: 'device_ids array is required' });
|
||||
}
|
||||
|
||||
const size = Math.max(1, Math.min(50, Number(wave_size) || 1));
|
||||
const run = await queryOne<{ id: number }>(
|
||||
`INSERT INTO command_runs
|
||||
(name, command, wave_size, halt_on_failure, use_change_guard, created_by)
|
||||
VALUES ($1,$2,$3,$4,$5,$6) RETURNING id`,
|
||||
[
|
||||
(name?.trim() || command.trim()).slice(0, 120),
|
||||
command.trim(),
|
||||
size,
|
||||
halt_on_failure !== false,
|
||||
// Guards are opt-out, never opt-in. Someone who has not thought about it
|
||||
// gets the protected behaviour.
|
||||
use_change_guard !== false,
|
||||
(req as unknown as { user?: { username?: string } }).user?.username ?? null,
|
||||
]
|
||||
);
|
||||
|
||||
const ordered = await query<{ id: number }>(
|
||||
`SELECT id FROM devices WHERE id = ANY($1::int[]) ORDER BY name`, [device_ids]
|
||||
);
|
||||
for (const [i, d] of ordered.entries()) {
|
||||
await query(
|
||||
`INSERT INTO command_run_devices (run_id, device_id, wave) VALUES ($1,$2,$3)`,
|
||||
[run!.id, d.id, Math.floor(i / size) + 1]
|
||||
);
|
||||
}
|
||||
|
||||
if (start) {
|
||||
try {
|
||||
await commandRunner.start(run!.id);
|
||||
} catch (e) {
|
||||
return res.status(409).json({ error: (e as Error).message, id: run!.id });
|
||||
}
|
||||
}
|
||||
res.status(201).json({ id: run!.id });
|
||||
});
|
||||
|
||||
// GET /api/commands/runs — recent runs with progress
|
||||
router.get('/runs', async (_req: Request, res: Response) => {
|
||||
const runs = await query(
|
||||
`SELECT r.*,
|
||||
COUNT(d.*)::int AS total,
|
||||
COUNT(*) FILTER (WHERE d.status = 'success')::int AS succeeded,
|
||||
COUNT(*) FILTER (WHERE d.status IN ('failed','reverted'))::int AS failed
|
||||
FROM command_runs r
|
||||
LEFT JOIN command_run_devices d ON d.run_id = r.id
|
||||
GROUP BY r.id ORDER BY r.id DESC LIMIT 25`
|
||||
);
|
||||
res.json(runs);
|
||||
});
|
||||
|
||||
// GET /api/commands/runs/:id — one run with per-device output
|
||||
router.get('/runs/:id', async (req: Request, res: Response) => {
|
||||
const run = await queryOne(`SELECT * FROM command_runs WHERE id = $1`, [req.params.id]);
|
||||
if (!run) return res.status(404).json({ error: 'Run not found' });
|
||||
const devices = await query(
|
||||
`SELECT rd.*, d.name AS device_name, d.ip_address
|
||||
FROM command_run_devices rd
|
||||
JOIN devices d ON d.id = rd.device_id
|
||||
WHERE rd.run_id = $1 ORDER BY rd.wave, d.name`,
|
||||
[req.params.id]
|
||||
);
|
||||
res.json({ ...run, devices });
|
||||
});
|
||||
|
||||
// POST /api/commands/runs/:id/start
|
||||
router.post('/runs/:id/start', requireWrite, async (req: Request, res: Response) => {
|
||||
try {
|
||||
await commandRunner.start(Number(req.params.id));
|
||||
res.json({ message: 'Run started' });
|
||||
} catch (e) {
|
||||
res.status(409).json({ error: (e as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/commands/runs/:id/cancel — stop before the next wave
|
||||
router.post('/runs/:id/cancel', requireWrite, async (req: Request, res: Response) => {
|
||||
commandRunner.cancel(Number(req.params.id));
|
||||
// Devices already running finish; cancellation prevents the next wave, which
|
||||
// is the only point at which stopping is safe.
|
||||
res.json({ message: 'Cancellation requested. Waves already running will finish.' });
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,172 @@
|
||||
/**
|
||||
* Running one command across many devices, in waves.
|
||||
*
|
||||
* The waves are the point. `for h in $hosts; do ssh $h "$cmd"; done` already
|
||||
* exists, costs nothing, and does most of this — but it reaches every device
|
||||
* before the first error is visible. What this adds is the ability to stop: a
|
||||
* small first wave, a look at what happened, and a halt before the mistake
|
||||
* becomes sixty mistakes (#118).
|
||||
*
|
||||
* Within that, the operator is trusted. Arbitrary commands, because a curated
|
||||
* library cannot express "add a channel to LACP" or "rewrite a WireGuard
|
||||
* endpoint". Change Guard on by default because bulk shutdown and firewall edits
|
||||
* are on the list of things people actually run, and off in one click because a
|
||||
* sharp tool that refuses to cut is not a tool.
|
||||
*/
|
||||
import { query, queryOne } from '../config/database';
|
||||
import { runSshCommand, looksLikeFailure, type SshExecDevice } from './sshExec';
|
||||
import { withSafeApply, type GuardDevice } from './changeGuard/ChangeGuard';
|
||||
|
||||
interface RunRow {
|
||||
id: number;
|
||||
name: string;
|
||||
command: string;
|
||||
wave_size: number;
|
||||
halt_on_failure: boolean;
|
||||
use_change_guard: boolean;
|
||||
status: string;
|
||||
}
|
||||
|
||||
interface RunDeviceRow {
|
||||
id: number;
|
||||
device_id: number;
|
||||
wave: number;
|
||||
}
|
||||
|
||||
type DeviceRow = SshExecDevice & GuardDevice;
|
||||
|
||||
export class CommandRunner {
|
||||
private activeRunId: number | null = null;
|
||||
private cancelRequested = false;
|
||||
|
||||
get isRunning(): boolean {
|
||||
return this.activeRunId !== null;
|
||||
}
|
||||
|
||||
async start(runId: number): Promise<void> {
|
||||
if (this.activeRunId) throw new Error(`Run #${this.activeRunId} is already in progress`);
|
||||
const run = await queryOne<RunRow>(`SELECT * FROM command_runs WHERE id = $1`, [runId]);
|
||||
if (!run) throw new Error('Run not found');
|
||||
if (run.status !== 'pending') throw new Error(`Run is ${run.status} — only pending runs can start`);
|
||||
|
||||
this.activeRunId = runId;
|
||||
this.cancelRequested = false;
|
||||
await query(`UPDATE command_runs SET status='running', started_at=NOW() WHERE id=$1`, [runId]);
|
||||
|
||||
// Fire and forget; callers poll. A bulk run outlives any HTTP request.
|
||||
void this.run(run)
|
||||
.catch(async (e) => {
|
||||
console.error(`[Command] run #${runId} crashed:`, e);
|
||||
await query(`UPDATE command_runs SET status='failed', finished_at=NOW() WHERE id=$1`, [runId]);
|
||||
})
|
||||
.finally(() => { this.activeRunId = null; });
|
||||
}
|
||||
|
||||
cancel(runId: number): void {
|
||||
if (this.activeRunId === runId) this.cancelRequested = true;
|
||||
}
|
||||
|
||||
private async run(run: RunRow): Promise<void> {
|
||||
const items = await query<RunDeviceRow>(
|
||||
`SELECT id, device_id, wave FROM command_run_devices WHERE run_id = $1 ORDER BY wave, id`,
|
||||
[run.id]
|
||||
);
|
||||
|
||||
const waves = [...new Set(items.map((i) => i.wave))].sort((a, b) => a - b);
|
||||
let halted = false;
|
||||
|
||||
for (const wave of waves) {
|
||||
if (this.cancelRequested || halted) break;
|
||||
const inWave = items.filter((i) => i.wave === wave);
|
||||
|
||||
// Devices inside a wave run together; waves themselves are strictly
|
||||
// sequential. That is what gives the operator a decision point.
|
||||
const results = await Promise.all(inWave.map((item) => this.runOne(run, item)));
|
||||
|
||||
if (run.halt_on_failure && results.some((ok) => !ok)) {
|
||||
halted = true;
|
||||
console.warn(`[Command] run #${run.id}: halting after failures in wave ${wave}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Anything not reached is skipped, not failed — it never ran.
|
||||
await query(
|
||||
`UPDATE command_run_devices SET status='skipped', finished_at=NOW()
|
||||
WHERE run_id=$1 AND status='pending'`, [run.id]);
|
||||
|
||||
const status = this.cancelRequested ? 'cancelled' : halted ? 'halted' : 'completed';
|
||||
await query(`UPDATE command_runs SET status=$2, finished_at=NOW() WHERE id=$1`, [run.id, status]);
|
||||
console.log(`[Command] run #${run.id} ${status}`);
|
||||
}
|
||||
|
||||
/** Returns true when the device succeeded. */
|
||||
private async runOne(run: RunRow, item: RunDeviceRow): Promise<boolean> {
|
||||
const device = await queryOne<DeviceRow>(
|
||||
`SELECT id, name, ip_address, api_port, api_username, api_password_encrypted,
|
||||
ssh_port, ssh_username, ssh_password_encrypted
|
||||
FROM devices WHERE id = $1`,
|
||||
[item.device_id]
|
||||
);
|
||||
if (!device) {
|
||||
await this.finish(item.id, 'failed', null, 'Device no longer exists');
|
||||
return false;
|
||||
}
|
||||
|
||||
await query(
|
||||
`UPDATE command_run_devices SET status='running', started_at=NOW(), guarded=$2 WHERE id=$1`,
|
||||
[item.id, run.use_change_guard]
|
||||
);
|
||||
|
||||
const execute = async () => {
|
||||
const { output } = await runSshCommand(device, run.command);
|
||||
// RouterOS reports most errors in the text rather than an exit code, so an
|
||||
// unread output is an unnoticed failure.
|
||||
if (looksLikeFailure(output)) throw new Error(output.slice(0, 500));
|
||||
return output;
|
||||
};
|
||||
|
||||
try {
|
||||
if (!run.use_change_guard) {
|
||||
const output = await execute();
|
||||
await this.finish(item.id, 'success', output, null);
|
||||
return true;
|
||||
}
|
||||
|
||||
const outcome = await withSafeApply(device, {
|
||||
kind: 'command.bulk',
|
||||
summary: `Bulk command: ${run.command.slice(0, 80)}`,
|
||||
}, execute);
|
||||
|
||||
if (outcome.autoReverting) {
|
||||
// The device stopped answering and is restoring itself. That is the
|
||||
// guard working, and it must be reported as loudly as a failure.
|
||||
await this.finish(
|
||||
item.id, 'reverted', outcome.result ?? null,
|
||||
'Device stopped responding after the command and is restoring itself.',
|
||||
true
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
await this.finish(item.id, 'success', outcome.result ?? null,
|
||||
outcome.unprotectedReason ? `Ran unprotected: ${outcome.unprotectedReason}` : null);
|
||||
return true;
|
||||
} catch (e) {
|
||||
await this.finish(item.id, 'failed', null, (e as Error).message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async finish(
|
||||
id: number, status: string, output: string | null, error: string | null, reverted = false,
|
||||
): Promise<void> {
|
||||
await query(
|
||||
`UPDATE command_run_devices
|
||||
SET status=$2, output=$3, error=$4, auto_reverted=$5, finished_at=NOW()
|
||||
WHERE id=$1`,
|
||||
[id, status, output, error, reverted]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export const commandRunner = new CommandRunner();
|
||||
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* Running a command on a device over SSH, with whichever credential works.
|
||||
*
|
||||
* Console syntax — `:put`, `:foreach`, `/interface print` — is not reachable over
|
||||
* the binary API, which has its own command tree. Anything that runs what an
|
||||
* operator would type has to go over SSH (#118).
|
||||
*
|
||||
* Credential choice is not a preference. RouterOS refuses password
|
||||
* authentication for a user once a key is bound to them, so a device with an
|
||||
* installed key can *only* be reached by that key. Password is the path for
|
||||
* devices without one (#110).
|
||||
*/
|
||||
import { Client as SSHClient } from 'ssh2';
|
||||
import { queryOne } from '../config/database';
|
||||
import { decrypt } from '../utils/crypto';
|
||||
|
||||
export interface SshExecDevice {
|
||||
id: number;
|
||||
name: string;
|
||||
ip_address: string;
|
||||
ssh_port?: number | null;
|
||||
ssh_username?: string | null;
|
||||
ssh_password_encrypted?: string | null;
|
||||
api_username?: string | null;
|
||||
api_password_encrypted?: string | null;
|
||||
}
|
||||
|
||||
export interface SshExecResult {
|
||||
/** Combined stdout and stderr, as an operator would see at a console. */
|
||||
output: string;
|
||||
/** Which credential actually carried the session. */
|
||||
auth: 'key' | 'password';
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
|
||||
/**
|
||||
* Resolve the credential for a device.
|
||||
*
|
||||
* A key is used only when it has been *proved* — a key that was generated or
|
||||
* pushed but never authenticated is not evidence the device will accept it.
|
||||
*/
|
||||
async function resolveAuth(device: SshExecDevice): Promise<{
|
||||
username: string;
|
||||
auth: { password: string } | { privateKey: string };
|
||||
kind: 'key' | 'password';
|
||||
}> {
|
||||
const key = await queryOne<{ private_key_encrypted: string; ssh_username: string | null }>(
|
||||
`SELECT private_key_encrypted, ssh_username FROM device_ssh_keys
|
||||
WHERE device_id = $1 AND status = 'verified'`,
|
||||
[device.id]
|
||||
);
|
||||
|
||||
if (key) {
|
||||
const username = key.ssh_username || device.ssh_username || device.api_username;
|
||||
if (!username) throw new Error('No SSH username available for key authentication');
|
||||
return { username, auth: { privateKey: decrypt(key.private_key_encrypted) }, kind: 'key' };
|
||||
}
|
||||
|
||||
const username = device.ssh_username || device.api_username;
|
||||
if (!username) throw new Error('No SSH username configured');
|
||||
const encrypted = device.ssh_password_encrypted || device.api_password_encrypted;
|
||||
if (!encrypted) throw new Error('No SSH credential configured');
|
||||
return { username, auth: { password: decrypt(encrypted) }, kind: 'password' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one command and return everything it printed.
|
||||
*
|
||||
* Errors are thrown, never swallowed. A command runner that reports success
|
||||
* because it discarded the failure is worse than one that does nothing — and a
|
||||
* quietly discarded error is exactly how a shipped feature spent a week doing
|
||||
* nothing at all.
|
||||
*/
|
||||
export async function runSshCommand(
|
||||
device: SshExecDevice,
|
||||
command: string,
|
||||
timeoutMs = DEFAULT_TIMEOUT_MS,
|
||||
): Promise<SshExecResult> {
|
||||
const { username, auth, kind } = await resolveAuth(device);
|
||||
|
||||
return new Promise<SshExecResult>((resolve, reject) => {
|
||||
const conn = new SSHClient();
|
||||
let settled = false;
|
||||
const finish = (fn: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
try { conn.end(); } catch { /* already closing */ }
|
||||
fn();
|
||||
};
|
||||
|
||||
const timer = setTimeout(
|
||||
() => finish(() => reject(new Error(`Command timed out after ${timeoutMs}ms`))),
|
||||
timeoutMs,
|
||||
);
|
||||
|
||||
conn.on('ready', () => {
|
||||
conn.exec(command, (err, stream) => {
|
||||
if (err) return finish(() => reject(err));
|
||||
let output = '';
|
||||
stream.on('data', (d: Buffer) => { output += d.toString(); });
|
||||
stream.stderr.on('data', (d: Buffer) => { output += d.toString(); });
|
||||
stream.on('close', () => finish(() => resolve({ output: output.trim(), auth: kind })));
|
||||
});
|
||||
});
|
||||
conn.on('error', (err) => finish(() => reject(err)));
|
||||
|
||||
conn.connect({
|
||||
host: device.ip_address,
|
||||
port: device.ssh_port ?? 22,
|
||||
username,
|
||||
readyTimeout: Math.min(timeoutMs, 20_000),
|
||||
...auth,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Does RouterOS's output indicate the command failed?
|
||||
*
|
||||
* RouterOS reports most errors in the text rather than through an exit code, so
|
||||
* a command that "succeeded" and one that printed `syntax error` are
|
||||
* indistinguishable unless the output is read. Deliberately conservative: only
|
||||
* unambiguous markers count, because misreading ordinary output as failure would
|
||||
* halt a rollout that was working.
|
||||
*/
|
||||
export function looksLikeFailure(output: string): boolean {
|
||||
return /(^|\n)\s*(syntax error|bad command name|expected end of command|no such item|failure:|input does not match any value)/i
|
||||
.test(output);
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { looksLikeFailure } from '../../services/sshExec';
|
||||
|
||||
describe('looksLikeFailure', () => {
|
||||
it('catches the ways RouterOS reports a bad command', () => {
|
||||
// RouterOS signals most errors in the text, not an exit code — a command
|
||||
// that worked and one that was rejected are otherwise indistinguishable.
|
||||
expect(looksLikeFailure('syntax error (line 1 column 5)')).toBe(true);
|
||||
expect(looksLikeFailure('bad command name foo (line 1 column 1)')).toBe(true);
|
||||
expect(looksLikeFailure('expected end of command (line 1 column 20)')).toBe(true);
|
||||
expect(looksLikeFailure('no such item (4)')).toBe(true);
|
||||
expect(looksLikeFailure('failure: already have such address')).toBe(true);
|
||||
expect(looksLikeFailure('input does not match any value of interface')).toBe(true);
|
||||
});
|
||||
|
||||
it('treats ordinary output as success', () => {
|
||||
// Misreading normal output as failure would halt a rollout that was working,
|
||||
// which is worse than the error it was trying to catch.
|
||||
expect(looksLikeFailure('')).toBe(false);
|
||||
expect(looksLikeFailure(' name: MikroTik-Test')).toBe(false);
|
||||
expect(looksLikeFailure('0 ether1 ether 1500 enabled')).toBe(false);
|
||||
expect(looksLikeFailure('142.250.187.206')).toBe(false);
|
||||
});
|
||||
|
||||
it('does not fire on words merely containing an error term', () => {
|
||||
expect(looksLikeFailure('comment="no such item handling"')).toBe(false);
|
||||
expect(looksLikeFailure('name=syntax-error-test')).toBe(false);
|
||||
});
|
||||
|
||||
it('finds a failure on any line, not only the first', () => {
|
||||
expect(looksLikeFailure('0 ether1\n1 ether2\nno such item (4)')).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user