feat: non-blocking bulk-add, CORS hardening, credential preset access control (#31)

* devices: hide discovered neighbors that are already managed Discovered Devices showed every unresolved MikroTik neighbor, even when the neighbor was obviously the same device a managed node is reporting through a different link / MAC / alias. Since we already know every managed device interface MAC, primary IP, and system identity, we can flag those rows. - /api/devices/discovered now annotates each row with duplicate_of_device_id and duplicate_of_device_name when the neighbor matches a managed device by interface MAC, primary IP, or system identity. - Add a Hide duplicates of managed devices toggle (default on) above the table, with a count of how many matches were found. When enabled, the list hides matched rows; when disabled, matches stay visible but are dimmed and tagged Already managed so the origin is clear. - Suppress the Add to Manager action for rows that are already a managed device, since adding them would create a second record for the same hardware. Co-authored-by: Denis <denis-ev@users.noreply.github.com> * devices: safer duplicate handling, try-all safeguards, and timezone/firmware fixes Co-authored-by: Denis <denis-ev@users.noreply.github.com> * Non-blocking follow-ups: parsePort util, tests, bulk-add queue, preset rate limit Co-authored-by: Denis <denis-ev@users.noreply.github.com> * fix(security): tighten CORS, Redis rate limits, and preset access controls Require explicit CORS origins in production, align Socket.IO CORS with HTTP, default Docker Compose to local HTTPS origins, and remove the in-memory mutation limiter in favor of Redis. Also add operator-use flags for credential presets in the UI and harden bulk-add job access and client error messages. Co-authored-by: Denis <denis-ev@users.noreply.github.com> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com>
2026-04-21 19:54:35 -07:00
parent 87a6d9161a
commit a12f52ebc7
28 changed files with 1912 additions and 252 deletions
@@ -0,0 +1,185 @@
+import { Queue, Worker, Job } from 'bullmq';
+import { createRedisConnection } from '../config/redis';
+import { redis } from '../config/redis';
+import { createDeviceFromBody, type CreateDeviceInput } from './deviceCreation';
+import type { PollerService } from './PollerService';
+
+const QUEUE_NAME = 'bulk-add-devices';
+const JOB_TTL_SEC = 86400;
+
+let pollerService: PollerService | null = null;
+let queue: Queue | null = null;
+let worker: Worker | null = null;
+
+export function setBulkAddPollerService(p: PollerService): void {
+  pollerService = p;
+}
+
+function metaKey(jobId: string): string {
+  return `device-bulk-add:${jobId}:meta`;
+}
+
+function resultsKey(jobId: string): string {
+  return `device-bulk-add:${jobId}:results`;
+}
+
+export type BulkAddResultRow = {
+  ip: string;
+  identity: string;
+  ok: boolean;
+  message: string;
+};
+
+interface BulkJobPayload {
+  jobId: string;
+  items: CreateDeviceInput[];
+}
+
+async function readMeta(jobId: string): Promise<Record<string, unknown>> {
+  const raw = await redis.get(metaKey(jobId));
+  if (!raw) return {};
+  try {
+    return JSON.parse(raw) as Record<string, unknown>;
+  } catch {
+    return {};
+  }
+}
+
+async function writeMeta(jobId: string, patch: Record<string, unknown>): Promise<void> {
+  const cur = await readMeta(jobId);
+  const next = { ...cur, ...patch };
+  await redis.set(metaKey(jobId), JSON.stringify(next), 'EX', JOB_TTL_SEC);
+}
+
+async function readResults(jobId: string): Promise<BulkAddResultRow[]> {
+  const raw = await redis.get(resultsKey(jobId));
+  if (!raw) return [];
+  try {
+    return JSON.parse(raw) as BulkAddResultRow[];
+  } catch {
+    return [];
+  }
+}
+
+async function appendResults(jobId: string, rows: BulkAddResultRow[]): Promise<void> {
+  const cur = await readResults(jobId);
+  const next = [...cur, ...rows];
+  await redis.set(resultsKey(jobId), JSON.stringify(next), 'EX', JOB_TTL_SEC);
+}
+
+async function processJob(job: Job<BulkJobPayload>): Promise<void> {
+  const { jobId, items } = job.data;
+  await writeMeta(jobId, { status: 'active', processed: 0 });
+  const poller = pollerService;
+
+  for (let i = 0; i < items.length; i++) {
+    const cancel = await redis.get(`device-bulk-add:${jobId}:cancel`);
+    if (cancel) {
+      await writeMeta(jobId, {
+        status: 'cancelled',
+        processed: i,
+        current_name: null,
+        cancelled_at: new Date().toISOString(),
+      });
+      return;
+    }
+
+    const item = items[i];
+    const identity = item.name || item.ip_address || '';
+    await writeMeta(jobId, {
+      processed: i,
+      current_name: identity,
+    });
+
+    const result = await createDeviceFromBody(
+      {
+        ...item,
+        device_type: item.device_type || 'router',
+      },
+      poller
+    );
+
+    let failMsg = 'Failed';
+    if (!result.ok) {
+      const b = result.body;
+      if (b.code === 'duplicate_serial') {
+        failMsg = 'Duplicate serial — review manually before combining';
+      } else if (typeof b.error === 'string') {
+        failMsg = b.error;
+      }
+    }
+    const row: BulkAddResultRow = {
+      ip: item.ip_address || '',
+      identity,
+      ok: result.ok,
+      message: result.ok
+        ? (result.body.merged_from_duplicate ? 'Updated (duplicate serial)' : 'Added')
+        : failMsg,
+    };
+    await appendResults(jobId, [row]);
+    await writeMeta(jobId, { processed: i + 1 });
+  }
+
+  await writeMeta(jobId, {
+    status: 'completed',
+    processed: items.length,
+    current_name: null,
+    completed_at: new Date().toISOString(),
+  });
+}
+
+export async function enqueueBulkAddJob(jobId: string, items: CreateDeviceInput[]): Promise<void> {
+  if (!queue) {
+    queue = new Queue(QUEUE_NAME, { connection: createRedisConnection() });
+  }
+  await queue.add('run', { jobId, items } satisfies BulkJobPayload, {
+    attempts: 1,
+    removeOnComplete: { age: 3600, count: 100 },
+    removeOnFail: { age: 86400 },
+  });
+}
+
+export async function getBulkAddJobState(jobId: string): Promise<{
+  found: boolean;
+  meta: Record<string, unknown>;
+  results: BulkAddResultRow[];
+}> {
+  const meta = await readMeta(jobId);
+  if (!meta || Object.keys(meta).length === 0) {
+    return { found: false, meta: {}, results: [] };
+  }
+  const results = await readResults(jobId);
+  return { found: true, meta, results };
+}
+
+export async function startBulkAddWorker(): Promise<void> {
+  if (worker) return;
+  worker = new Worker<BulkJobPayload>(
+    QUEUE_NAME,
+    async (job) => {
+      try {
+        await processJob(job);
+      } catch (err) {
+        const { jobId } = job.data;
+        console.error('[BulkAddWorker] job failed:', job?.id, err);
+        await writeMeta(jobId, {
+          status: 'failed',
+          error: 'Batch job failed. Check server logs for details.',
+          failed_at: new Date().toISOString(),
+        });
+        throw err;
+      }
+    },
+    { connection: createRedisConnection(), concurrency: 1 }
+  );
+  worker.on('failed', (job, err) => {
+    console.error('[BulkAddWorker] job failed:', job?.id, err);
+  });
+}
+
+export async function stopBulkAddWorker(): Promise<void> {
+  await worker?.close();
+  worker = null;
+  await queue?.close();
+  queue = null;
+}
@@ -0,0 +1,242 @@
+import { query, queryOne } from '../config/database';
+import { encrypt, decrypt } from '../utils/crypto';
+import { parsePort } from '../utils/parsePort';
+import { safeConnectionError } from '../utils/safeClientError';
+import { RouterOSClient } from './mikrotik/RouterOSClient';
+import type { PollerService } from './PollerService';
+import type { CredentialPresetRow } from '../routes/credentialPresets';
+
+export type CreateDeviceContext = {
+  /** JWT role of the caller ('admin' | 'operator' | 'viewer'). Preset use may be restricted for operators. */
+  requestingUserRole?: string;
+};
+
+async function loadCredentialPreset(
+  id: number | null | undefined,
+  ctx?: CreateDeviceContext
+): Promise<{
+  api_username: string;
+  api_password: string;
+  api_port: number | null;
+  ssh_username: string | null;
+  ssh_password: string | null;
+  ssh_port: number | null;
+} | null> {
+  if (id === null || id === undefined) return null;
+  const preset = await queryOne<CredentialPresetRow>(
+    `SELECT * FROM credential_presets WHERE id = $1`,
+    [id]
+  );
+  if (!preset) throw new Error(`Credential preset ${id} not found`);
+  const allowOp = (preset as CredentialPresetRow & { allow_operator_use?: boolean }).allow_operator_use !== false;
+  if (ctx?.requestingUserRole === 'operator' && !allowOp) {
+    const err = new Error('This credential preset is restricted to administrators');
+    (err as Error & { statusCode?: number }).statusCode = 403;
+    throw err;
+  }
+  return {
+    api_username: preset.api_username,
+    api_password: decrypt(preset.api_password_encrypted),
+    api_port: preset.api_port,
+    ssh_username: preset.ssh_username,
+    ssh_password: preset.ssh_password_encrypted ? decrypt(preset.ssh_password_encrypted) : null,
+    ssh_port: preset.ssh_port,
+  };
+}
+
+export interface CreateDeviceInput {
+  name?: string;
+  ip_address?: string;
+  device_type?: string;
+  notes?: string;
+  credential_preset_id?: number | null;
+  api_username?: string;
+  api_password?: string;
+  api_port?: unknown;
+  ssh_username?: string | null;
+  ssh_password?: string | null;
+  ssh_port?: unknown;
+  combine_with_device_id?: number;
+  force_replace_existing_by_serial?: boolean;
+}
+
+export type CreateDeviceResult =
+  | { ok: true; status: 200 | 201; body: Record<string, unknown> }
+  | { ok: false; status: number; body: Record<string, unknown> };
+
+/**
+ * Shared device creation logic used by POST /api/devices and bulk-add worker.
+ */
+export async function createDeviceFromBody(
+  input: CreateDeviceInput,
+  pollerService: PollerService | null,
+  ctx?: CreateDeviceContext
+): Promise<CreateDeviceResult> {
+  const {
+    name,
+    ip_address,
+    device_type = 'router',
+    notes,
+    credential_preset_id,
+  } = input;
+
+  let preset: Awaited<ReturnType<typeof loadCredentialPreset>> = null;
+  try {
+    preset = await loadCredentialPreset(credential_preset_id ?? null, ctx);
+  } catch (err) {
+    const status = (err as Error & { statusCode?: number }).statusCode ?? 400;
+    return { ok: false, status, body: { error: (err as Error).message } };
+  }
+
+  const api_username: string | undefined = preset?.api_username ?? input.api_username;
+  const api_password: string | undefined = preset?.api_password ?? input.api_password;
+  const api_port: number = preset?.api_port ?? parsePort(input.api_port, 8728);
+  const ssh_username: string | null = preset ? preset.ssh_username : (input.ssh_username ?? null);
+  const ssh_password: string | null = preset ? preset.ssh_password : (input.ssh_password ?? null);
+  const ssh_port: number = preset?.ssh_port ?? parsePort(input.ssh_port, 22);
+  const combineWithDeviceId =
+    typeof input.combine_with_device_id === 'number' ? input.combine_with_device_id : null;
+  const forceReplaceBySerial = input.force_replace_existing_by_serial === true;
+
+  if (!name || !ip_address || !api_username || !api_password) {
+    return {
+      ok: false,
+      status: 400,
+      body: { error: 'name, ip_address, api_username, api_password are required' },
+    };
+  }
+
+  const testClient = new RouterOSClient(ip_address, api_port, api_username, api_password, 10_000);
+  let detectedSerial: string | null = null;
+  try {
+    await testClient.connect();
+    const rb = await testClient.execute('/system/routerboard/print').catch(() => [] as Record<string, string>[]);
+    detectedSerial = (rb[0]?.['serial-number'] || '').trim() || null;
+  } catch (err) {
+    return {
+      ok: false,
+      status: 422,
+      body: { error: safeConnectionError('createDeviceFromBody', err) },
+    };
+  } finally {
+    testClient.disconnect();
+  }
+
+  if (detectedSerial) {
+    const existingBySerial = await queryOne<{
+      id: number;
+      name: string;
+      ip_address: string;
+      serial_number: string;
+    }>(
+      `SELECT id, name, ip_address, serial_number
+         FROM devices
+        WHERE serial_number = $1`,
+      [detectedSerial]
+    );
+
+    if (existingBySerial) {
+      const shouldCombine =
+        combineWithDeviceId != null && combineWithDeviceId === existingBySerial.id;
+      if (!shouldCombine && !forceReplaceBySerial) {
+        return {
+          ok: false,
+          status: 409,
+          body: {
+            error: 'duplicate_serial',
+            code: 'duplicate_serial',
+            existing_device: existingBySerial,
+            candidate: {
+              serial_number: detectedSerial,
+              identity: name,
+              ip_address,
+            },
+          },
+        };
+      }
+
+      const encryptedPass = encrypt(api_password);
+      const encryptedSshPass = ssh_password ? encrypt(ssh_password) : null;
+      await query(
+        `UPDATE devices SET
+           name=COALESCE($1,name),
+           ip_address=$2,
+           api_port=$3,
+           api_username=$4,
+           api_password_encrypted=$5,
+           ssh_port=$6,
+           ssh_username=COALESCE($7,ssh_username),
+           ssh_password_encrypted=COALESCE($8,ssh_password_encrypted),
+           device_type=COALESCE($9,device_type),
+           notes=COALESCE($10,notes),
+           updated_at=NOW()
+         WHERE id = $11`,
+        [
+          name,
+          ip_address,
+          api_port,
+          api_username,
+          encryptedPass,
+          ssh_port,
+          ssh_username,
+          encryptedSshPass,
+          device_type,
+          notes || null,
+          existingBySerial.id,
+        ]
+      );
+
+      if (pollerService) {
+        await pollerService.scheduleDeviceSync(existingBySerial.id, 'full');
+      }
+
+      const updatedExisting = await queryOne(
+        `SELECT id, name, ip_address, api_port, api_username, model, serial_number,
+                firmware_version, ros_version, device_type, status, last_seen, notes, created_at
+         FROM devices WHERE id = $1`,
+        [existingBySerial.id]
+      );
+      if (!updatedExisting) {
+        return {
+          ok: false,
+          status: 500,
+          body: {
+            error: 'Duplicate merge succeeded but device row could not be reloaded',
+            device_id: existingBySerial.id,
+          },
+        };
+      }
+      return {
+        ok: true,
+        status: 200,
+        body: { ...updatedExisting, merged_from_duplicate: true },
+      };
+    }
+  }
+
+  const encryptedPass = encrypt(api_password);
+  const encryptedSshPass = ssh_password ? encrypt(ssh_password) : null;
+
+  const rows = await query<{ id: number }>(
+    `INSERT INTO devices (name, ip_address, api_port, api_username, api_password_encrypted,
+                          ssh_port, ssh_username, ssh_password_encrypted, device_type, notes, status)
+     VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,'unknown') RETURNING id`,
+    [name, ip_address, api_port, api_username, encryptedPass,
+     ssh_port, ssh_username || null, encryptedSshPass, device_type, notes || null]
+  );
+
+  const newId = rows[0].id;
+
+  if (pollerService) {
+    await pollerService.scheduleDeviceSync(newId, 'full');
+  }
+
+  const device = await queryOne(
+    `SELECT id, name, ip_address, api_port, api_username, model, serial_number,
+            firmware_version, ros_version, device_type, status, last_seen, notes, created_at
+     FROM devices WHERE id = $1`,
+    [newId]
+  );
+
+  return { ok: true, status: 201, body: device as Record<string, unknown> };
+}