Closes #136. Also fixes #133, #137, #138, #139 (GitHub closes only the first). None of these came from multi-site; two came from me. #136 — waves failed on the second device. A regression from v0.23.31, the firmware rewrite that replaced fire-and-forget `install` with `download` plus polling. `/system/package/update/download` blocks for a minute or more, and the client's read timeout is 30 seconds. The timeout path was the real damage: it nulled the pending read but left the socket open, so when the reply finally arrived processBuffer() queued it and the *next* command took it as its own answer. Every later call on that connection returned the previous one's data, which is why the status polls could never see "Downloaded". - a timed-out read now abandons the connection instead of silently desynchronising it, so the failure is loud rather than wrong - long-running commands can state their own budget; check-for-updates, download and /log/print now do - a download whose command times out is re-checked on a fresh connection rather than assumed failed, because the image may well have landed #133 — the SSH terminal could not use a deployed key. It read only ssh_password_encrypted and refused to connect without one, so retiring the password after deploying a key -- the workflow the feature exists for -- broke the console. It now uses the same resolver as every other SSH consumer. preferredAuth(), written to make exactly this decision, had never run outside its own tests, and disagreed with the resolver that did: it accepted a 'deployed' key where the live path required 'verified'. Unified on the stricter rule and wired in. #139 — the rollout was written before start() was attempted, so a rejected start left the rows behind: "already running", press again, another rollout banked each time. Devices already in an unfinished rollout are now refused by name, before anything is persisted. #138 — cancel worked but the API's explanation that the in-flight device is never interrupted was discarded, so the button looked dead. It is shown. #137 — log overfetching. /log/print returns the whole ring buffer, and the count was incremented per attempted insert rather than per accepted row, so a poll that stored nothing still reported "Collected 1000 new log entries" after a thousand round trips. Inserts are batched and counted by RETURNING, and the id watermark is the maximum over recent rows rather than the last one inserted, which could move backwards and make the whole buffer look new. One self-inflicted wound worth recording. The first cut of the desync guard discarded sentences whenever `connected` was false -- but `connected` is only set *after* login succeeds, so it threw away every login reply and took the whole fleet offline until it was caught. "Not yet connected" and "abandoned" are different states and now have different flags. Every existing test pre-set connected = true, so none of them could see it; the test added for it fails against the bad version. Verified on hardware: devices back online, check-update 5.2s (was a 30s failure), full sync completes, and log collection now reports 298 / 22 / 12 / 12 across successive polls -- real decreasing counts rather than a constant 1000 -- with no failures. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
146 lines
5.2 KiB
TypeScript
146 lines
5.2 KiB
TypeScript
/**
|
|
* Running a command on a device over SSH, with whichever credential works.
|
|
*
|
|
* Console syntax — `:put`, `:foreach`, `/interface print` — is not reachable over
|
|
* the binary API, which has its own command tree. Anything that runs what an
|
|
* operator would type has to go over SSH (#118).
|
|
*
|
|
* Credential choice is not a preference. RouterOS refuses password
|
|
* authentication for a user once a key is bound to them, so a device with an
|
|
* installed key can *only* be reached by that key. Password is the path for
|
|
* devices without one (#110).
|
|
*/
|
|
import { Client as SSHClient } from 'ssh2';
|
|
import { queryOne } from '../config/database';
|
|
import { decrypt } from '../utils/crypto';
|
|
import { preferredAuth, type KeyStatus } from '../utils/sshKeys';
|
|
|
|
export interface SshExecDevice {
|
|
id: number;
|
|
name: string;
|
|
ip_address: string;
|
|
ssh_port?: number | null;
|
|
ssh_username?: string | null;
|
|
ssh_password_encrypted?: string | null;
|
|
api_username?: string | null;
|
|
api_password_encrypted?: string | null;
|
|
}
|
|
|
|
export interface SshExecResult {
|
|
/** Combined stdout and stderr, as an operator would see at a console. */
|
|
output: string;
|
|
/** Which credential actually carried the session. */
|
|
auth: 'key' | 'password';
|
|
}
|
|
|
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
|
|
/**
|
|
* Resolve the credential for a device.
|
|
*
|
|
* A key is used only when it has been *proved* — a key that was generated or
|
|
* pushed but never authenticated is not evidence the device will accept it.
|
|
*
|
|
* Exported because every SSH consumer must agree on this. The interactive
|
|
* terminal used to resolve credentials itself, looked only at the stored
|
|
* password, and so broke the moment a key was deployed and the password
|
|
* retired — the exact workflow the key feature exists to enable (#133).
|
|
*/
|
|
export async function resolveAuth(device: SshExecDevice): Promise<{
|
|
username: string;
|
|
auth: { password: string } | { privateKey: string };
|
|
kind: 'key' | 'password';
|
|
}> {
|
|
const key = await queryOne<{
|
|
private_key_encrypted: string; ssh_username: string | null; status: KeyStatus;
|
|
}>(
|
|
`SELECT private_key_encrypted, ssh_username, status FROM device_ssh_keys
|
|
WHERE device_id = $1`,
|
|
[device.id]
|
|
);
|
|
|
|
const encryptedPassword = device.ssh_password_encrypted || device.api_password_encrypted;
|
|
const choice = preferredAuth(
|
|
key?.status ?? null,
|
|
!!key?.private_key_encrypted,
|
|
!!encryptedPassword
|
|
);
|
|
|
|
if (choice === 'key' && key) {
|
|
const username = key.ssh_username || device.ssh_username || device.api_username;
|
|
if (!username) throw new Error('No SSH username available for key authentication');
|
|
return { username, auth: { privateKey: decrypt(key.private_key_encrypted) }, kind: 'key' };
|
|
}
|
|
|
|
const username = device.ssh_username || device.api_username;
|
|
if (!username) throw new Error('No SSH username configured');
|
|
if (choice === 'none' || !encryptedPassword) throw new Error('No SSH credential configured');
|
|
return { username, auth: { password: decrypt(encryptedPassword) }, kind: 'password' };
|
|
}
|
|
|
|
/**
|
|
* Run one command and return everything it printed.
|
|
*
|
|
* Errors are thrown, never swallowed. A command runner that reports success
|
|
* because it discarded the failure is worse than one that does nothing — and a
|
|
* quietly discarded error is exactly how a shipped feature spent a week doing
|
|
* nothing at all.
|
|
*/
|
|
export async function runSshCommand(
|
|
device: SshExecDevice,
|
|
command: string,
|
|
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
): Promise<SshExecResult> {
|
|
const { username, auth, kind } = await resolveAuth(device);
|
|
|
|
return new Promise<SshExecResult>((resolve, reject) => {
|
|
const conn = new SSHClient();
|
|
let settled = false;
|
|
const finish = (fn: () => void) => {
|
|
if (settled) return;
|
|
settled = true;
|
|
clearTimeout(timer);
|
|
try { conn.end(); } catch { /* already closing */ }
|
|
fn();
|
|
};
|
|
|
|
const timer = setTimeout(
|
|
() => finish(() => reject(new Error(`Command timed out after ${timeoutMs}ms`))),
|
|
timeoutMs,
|
|
);
|
|
|
|
conn.on('ready', () => {
|
|
conn.exec(command, (err, stream) => {
|
|
if (err) return finish(() => reject(err));
|
|
let output = '';
|
|
stream.on('data', (d: Buffer) => { output += d.toString(); });
|
|
stream.stderr.on('data', (d: Buffer) => { output += d.toString(); });
|
|
stream.on('close', () => finish(() => resolve({ output: output.trim(), auth: kind })));
|
|
});
|
|
});
|
|
conn.on('error', (err) => finish(() => reject(err)));
|
|
|
|
conn.connect({
|
|
host: device.ip_address,
|
|
port: device.ssh_port ?? 22,
|
|
username,
|
|
readyTimeout: Math.min(timeoutMs, 20_000),
|
|
...auth,
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Does RouterOS's output indicate the command failed?
|
|
*
|
|
* RouterOS reports most errors in the text rather than through an exit code, so
|
|
* a command that "succeeded" and one that printed `syntax error` are
|
|
* indistinguishable unless the output is read. Deliberately conservative: only
|
|
* unambiguous markers count, because misreading ordinary output as failure would
|
|
* halt a rollout that was working.
|
|
*/
|
|
export function looksLikeFailure(output: string): boolean {
|
|
return /(^|\n)\s*(syntax error|bad command name|expected end of command|no such item|failure:|input does not match any value)/i
|
|
.test(output);
|
|
}
|