feat: NetFlow/IPFIX traffic analytics — per-client usage, app breakdown, built-in collector (v0.13.0-beta)
Adds Meraki-style traffic analytics powered by MikroTik Traffic Flow: Backend - Built-in NetFlow v9 + IPFIX collector (UDP 2055) with hand-rolled decoder, template caching, enterprise-field and IPv6 support - Flow attribution: client IP → MAC via the clients table, upload/download direction, NAT-duplicate records dropped - App classification by protocol/port (~25 categories: HTTPS, QUIC, DNS, SSH, Email, WireGuard, ...) - Deduplication: per-client best-exporter-per-window so flows crossing two managed routers are never double-counted - NAT-tolerant ingest: flows from sources matching no managed device (NAT / Tailscale subnet routers in path) are accepted as per-source unidentified exporters instead of silently dropped (netflow_accept_unknown setting) - Storage: InfluxDB client_traffic time-series (bounded cardinality via top-N fold) + Postgres client_traffic_daily rollups; daily retention purge - One-toggle device config push via RouterOS API (/ip traffic-flow + target) - New /api/traffic/* analytics endpoints + /api/network-services/netflow - docker-compose: NETFLOW_PORT (default 2055/udp) published on the backend Frontend - New Traffic page: fleet up/down chart, top talkers with client click-through, app-category donut, 1h–30d ranges - New NetFlow config page: collector settings, live ingest stats, per-device export toggles with health badges, NAT-source banner - Client detail: App Traffic card (auto-hides without data) - Clients page: sortable "Data (today)" column - Settings: NetFlow retention controls Includes 17 unit tests (decoder fixtures, dedup, attribution, classifier); verified end-to-end against live RouterOS devices. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2GT-Rich
co-authored by
Claude Fable 5
Claude Fable 5
parent
367034c2d6
commit
b841eb6e7b
@@ -42,7 +42,10 @@ router.get('/', async (req: Request, res: Response) => {
|
||||
// Deduplicate by MAC address across devices: prefer active rows, then most recently seen.
|
||||
// The inner DISTINCT ON picks the best row per MAC; the outer query applies final sort + pagination.
|
||||
const sql = `
|
||||
SELECT * FROM (
|
||||
SELECT deduped.*,
|
||||
-- float8 so node-pg returns a number (bigint would arrive as a string)
|
||||
(COALESCE(ctd.upload_bytes, 0) + COALESCE(ctd.download_bytes, 0))::float8 AS traffic_today_bytes
|
||||
FROM (
|
||||
SELECT DISTINCT ON (c.mac_address)
|
||||
c.*, d.name as device_name,
|
||||
wi.ssid as ssid
|
||||
@@ -53,6 +56,8 @@ router.get('/', async (req: Request, res: Response) => {
|
||||
${where}
|
||||
ORDER BY c.mac_address, (c.client_type = 'wireless') DESC, c.active DESC, c.last_seen DESC NULLS LAST
|
||||
) deduped
|
||||
LEFT JOIN client_traffic_daily ctd
|
||||
ON ctd.mac_address = LOWER(deduped.mac_address) AND ctd.day = CURRENT_DATE
|
||||
ORDER BY last_seen DESC NULLS LAST
|
||||
LIMIT $${idx++} OFFSET $${idx++}
|
||||
`;
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Router, Request, Response } from 'express';
|
||||
import { query } from '../config/database';
|
||||
import { requireAuth, requireWrite } from '../middleware/auth';
|
||||
import { DeviceCollector, DeviceRow } from '../services/mikrotik/DeviceCollector';
|
||||
import { netflowCollector } from '../services/netflow/NetflowCollector';
|
||||
|
||||
const router = Router();
|
||||
router.use(requireAuth);
|
||||
@@ -488,4 +489,166 @@ router.delete('/syslog/rule/:id', requireWrite, async (req: Request, res: Respon
|
||||
});
|
||||
});
|
||||
|
||||
// ─── NetFlow / Traffic Flow ───────────────────────────────────────────────────
|
||||
|
||||
async function getNetflowAppSettings(): Promise<{
|
||||
address: string; port: number; version: string; activeTimeout: string; inactiveTimeout: string;
|
||||
}> {
|
||||
const rows = await query<{ key: string; value: unknown }>(
|
||||
`SELECT key, value FROM app_settings
|
||||
WHERE key IN ('netflow_collector_address', 'netflow_collector_port', 'netflow_version',
|
||||
'netflow_active_timeout', 'netflow_inactive_timeout')`
|
||||
);
|
||||
const map: Record<string, unknown> = {};
|
||||
for (const row of rows) map[row.key] = row.value;
|
||||
return {
|
||||
address: String(map['netflow_collector_address'] || ''),
|
||||
port: Number(map['netflow_collector_port']) || 2055,
|
||||
version: String(map['netflow_version'] || '9'),
|
||||
activeTimeout: String(map['netflow_active_timeout'] || '1m'),
|
||||
inactiveTimeout: String(map['netflow_inactive_timeout'] || '15s'),
|
||||
};
|
||||
}
|
||||
|
||||
function targetMatchesCollector(
|
||||
targets: Record<string, string>[],
|
||||
collector: { address: string; port: number }
|
||||
): boolean {
|
||||
return targets.some(
|
||||
(t) => t['dst-address'] === collector.address && Number(t['port']) === collector.port && t['disabled'] !== 'true'
|
||||
);
|
||||
}
|
||||
|
||||
// GET /api/network-services/netflow/fleet — traffic-flow state for all online devices
|
||||
router.get('/netflow/fleet', async (_req: Request, res: Response) => {
|
||||
const settings = await getNetflowAppSettings();
|
||||
const stats = netflowCollector.getStats();
|
||||
const statsByDevice = new Map(stats.exporters.map((e) => [e.deviceId, e]));
|
||||
const devices = await query<DeviceRow>(`SELECT * FROM devices WHERE status = 'online' ORDER BY name`);
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
devices.map(async (device: DeviceRow) => {
|
||||
const collector = new DeviceCollector(device);
|
||||
try {
|
||||
await collector.connect();
|
||||
const [tfSettings, targets] = await Promise.all([
|
||||
collector.getTrafficFlowSettings(),
|
||||
collector.getTrafficFlowTargets(),
|
||||
]);
|
||||
const exporterStats = statsByDevice.get(device.id);
|
||||
return {
|
||||
id: device.id,
|
||||
name: device.name,
|
||||
ip_address: device.ip_address,
|
||||
enabled: tfSettings?.['enabled'] === 'true' || tfSettings?.['enabled'] === 'yes',
|
||||
interfaces: tfSettings?.['interfaces'] || '',
|
||||
targets: targets.map((t) => ({
|
||||
id: t['.id'],
|
||||
dst_address: t['dst-address'],
|
||||
port: Number(t['port']) || 0,
|
||||
version: t['version'] || '',
|
||||
disabled: t['disabled'] === 'true',
|
||||
})),
|
||||
target_matches_collector: targetMatchesCollector(targets, settings),
|
||||
flows_received: exporterStats?.flows || 0,
|
||||
last_flow_at: exporterStats?.lastSeen || null,
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
id: device.id, name: device.name, ip_address: device.ip_address,
|
||||
enabled: null, interfaces: '', targets: [], target_matches_collector: false,
|
||||
flows_received: 0, last_flow_at: null, error: (err as Error).message,
|
||||
};
|
||||
} finally {
|
||||
collector.disconnect();
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
res.json({
|
||||
collector: { ...settings, listening: stats.listening },
|
||||
devices: results.map((r) => (r.status === 'fulfilled' ? r.value : { error: (r.reason as Error).message })),
|
||||
});
|
||||
});
|
||||
|
||||
// GET /api/network-services/netflow?deviceId=X — single device traffic-flow state
|
||||
router.get('/netflow', async (req: Request, res: Response) => {
|
||||
const deviceId = deviceIdParam(req, res); if (!deviceId) return;
|
||||
const settings = await getNetflowAppSettings();
|
||||
await withDevice(deviceId, res, async (collector) => {
|
||||
const [tfSettings, targets] = await Promise.all([
|
||||
collector.getTrafficFlowSettings(),
|
||||
collector.getTrafficFlowTargets(),
|
||||
]);
|
||||
res.json({
|
||||
settings: tfSettings,
|
||||
targets,
|
||||
target_matches_collector: targetMatchesCollector(targets, settings),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// PUT /api/network-services/netflow?deviceId=X — body { enabled: boolean }
|
||||
// Enable: upsert our export target (matched by dst-address + port) and turn
|
||||
// traffic-flow on. Disable: remove our target; only turn traffic-flow off if
|
||||
// no other (user-managed) targets remain.
|
||||
router.put('/netflow', requireWrite, async (req: Request, res: Response) => {
|
||||
const deviceId = deviceIdParam(req, res); if (!deviceId) return;
|
||||
const { enabled } = req.body;
|
||||
if (typeof enabled !== 'boolean') return res.status(400).json({ error: 'enabled (boolean) is required' });
|
||||
|
||||
const settings = await getNetflowAppSettings();
|
||||
if (enabled && !settings.address) {
|
||||
return res.status(400).json({ error: 'Set the collector address in NetFlow settings before enabling devices' });
|
||||
}
|
||||
|
||||
await withDevice(deviceId, res, async (collector) => {
|
||||
const targets = await collector.getTrafficFlowTargets();
|
||||
const ours = targets.find(
|
||||
(t) => t['dst-address'] === settings.address && Number(t['port']) === settings.port
|
||||
);
|
||||
|
||||
if (enabled) {
|
||||
const targetParams: Record<string, string> = {
|
||||
'dst-address': settings.address,
|
||||
port: String(settings.port),
|
||||
version: settings.version === 'ipfix' ? 'ipfix' : '9',
|
||||
};
|
||||
if (settings.version !== 'ipfix') {
|
||||
// Re-send templates frequently so the collector can decode data
|
||||
// promptly after a backend restart clears its template cache.
|
||||
targetParams['v9-template-refresh'] = '20';
|
||||
targetParams['v9-template-timeout'] = '1m';
|
||||
}
|
||||
if (ours) {
|
||||
await collector.updateTrafficFlowTarget(ours['.id'], targetParams);
|
||||
} else {
|
||||
await collector.addTrafficFlowTarget(targetParams);
|
||||
}
|
||||
await collector.setTrafficFlowSettings({
|
||||
enabled: 'yes',
|
||||
interfaces: 'all',
|
||||
'active-flow-timeout': settings.activeTimeout,
|
||||
'inactive-flow-timeout': settings.inactiveTimeout,
|
||||
});
|
||||
} else {
|
||||
if (ours) await collector.removeTrafficFlowTarget(ours['.id']);
|
||||
const remaining = (await collector.getTrafficFlowTargets()).filter((t) => t['.id'] !== ours?.['.id']);
|
||||
if (remaining.length === 0) {
|
||||
await collector.setTrafficFlowSettings({ enabled: 'no' });
|
||||
}
|
||||
}
|
||||
|
||||
const [tfSettings, updatedTargets] = await Promise.all([
|
||||
collector.getTrafficFlowSettings(),
|
||||
collector.getTrafficFlowTargets(),
|
||||
]);
|
||||
res.json({
|
||||
settings: tfSettings,
|
||||
targets: updatedTargets,
|
||||
target_matches_collector: targetMatchesCollector(updatedTargets, settings),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Router, Request, Response } from 'express';
|
||||
import bcrypt from 'bcryptjs';
|
||||
import { query, queryOne } from '../config/database';
|
||||
import { requireAuth, requireAdmin } from '../middleware/auth';
|
||||
import { netflowCollector } from '../services/netflow/NetflowCollector';
|
||||
|
||||
const router = Router();
|
||||
router.use(requireAuth);
|
||||
@@ -26,6 +27,11 @@ router.put('/', requireAdmin, async (req: Request, res: Response) => {
|
||||
[key, JSON.stringify(value)]
|
||||
);
|
||||
}
|
||||
// Apply NetFlow listener changes immediately instead of waiting for the
|
||||
// collector's periodic settings reconcile.
|
||||
if (Object.keys(updates).some((k) => k.startsWith('netflow_'))) {
|
||||
netflowCollector.reconcile().catch(() => {});
|
||||
}
|
||||
res.json({ message: 'Settings updated' });
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { getQueryApi, bucket } from '../config/influxdb';
|
||||
import { query } from '../config/database';
|
||||
import { requireAuth } from '../middleware/auth';
|
||||
import { netflowCollector } from '../services/netflow/NetflowCollector';
|
||||
|
||||
const router = Router();
|
||||
router.use(requireAuth);
|
||||
|
||||
function rangeToFlux(range: string): string {
|
||||
const allowed = ['1h', '3h', '6h', '12h', '24h', '7d', '30d'];
|
||||
return allowed.includes(range) ? range : '24h';
|
||||
}
|
||||
|
||||
// Aggregation window sized to the range so charts stay readable (points are
|
||||
// written every 60s by the collector flush).
|
||||
function windowForRange(range: string): string {
|
||||
const map: Record<string, string> = {
|
||||
'1h': '1m', '3h': '5m', '6h': '5m', '12h': '10m',
|
||||
'24h': '15m', '7d': '1h', '30d': '6h',
|
||||
};
|
||||
return map[range] || '15m';
|
||||
}
|
||||
|
||||
// MACs are used inside Flux queries — restrict to safe charsets (real MACs
|
||||
// plus the 'unknown'/'other' pseudo-clients).
|
||||
function sanitizeMac(raw: string): string | null {
|
||||
const mac = raw.toLowerCase();
|
||||
if (mac === 'unknown' || mac === 'other') return mac;
|
||||
return /^[0-9a-f]{2}(:[0-9a-f]{2}){5}$/.test(mac) ? mac : null;
|
||||
}
|
||||
|
||||
// GET /api/traffic/status — collector state for the config page
|
||||
router.get('/status', async (_req: Request, res: Response) => {
|
||||
res.json(netflowCollector.getStats());
|
||||
});
|
||||
|
||||
// GET /api/traffic/timeseries?range=24h — fleet-wide upload/download over time
|
||||
router.get('/timeseries', async (req: Request, res: Response) => {
|
||||
const range = rangeToFlux(String(req.query.range || '24h'));
|
||||
const every = windowForRange(range);
|
||||
const queryApi = getQueryApi();
|
||||
|
||||
const fluxQuery = `
|
||||
from(bucket: "${bucket}")
|
||||
|> range(start: -${range})
|
||||
|> filter(fn: (r) => r._measurement == "client_traffic")
|
||||
|> filter(fn: (r) => r._field == "bytes")
|
||||
|> group(columns: ["direction"])
|
||||
|> aggregateWindow(every: ${every}, fn: sum, createEmpty: false)
|
||||
`;
|
||||
|
||||
const pivoted: Record<string, { time: string; upload: number; download: number }> = {};
|
||||
try {
|
||||
await queryApi.collectRows(fluxQuery, (row, tableMeta) => {
|
||||
const time = tableMeta.get(row, '_time') as string;
|
||||
const direction = tableMeta.get(row, 'direction') as string;
|
||||
const value = Number(tableMeta.get(row, '_value')) || 0;
|
||||
if (!pivoted[time]) pivoted[time] = { time, upload: 0, download: 0 };
|
||||
if (direction === 'upload') pivoted[time].upload += value;
|
||||
else if (direction === 'download') pivoted[time].download += value;
|
||||
});
|
||||
} catch {
|
||||
// No data yet
|
||||
}
|
||||
|
||||
res.json(Object.values(pivoted).sort((a, b) => a.time.localeCompare(b.time)));
|
||||
});
|
||||
|
||||
// GET /api/traffic/top-clients?range=24h&limit=10 — top talkers with client names
|
||||
router.get('/top-clients', async (req: Request, res: Response) => {
|
||||
const range = rangeToFlux(String(req.query.range || '24h'));
|
||||
const limit = Math.min(parseInt(String(req.query.limit || '10'), 10) || 10, 50);
|
||||
const queryApi = getQueryApi();
|
||||
|
||||
const fluxQuery = `
|
||||
from(bucket: "${bucket}")
|
||||
|> range(start: -${range})
|
||||
|> filter(fn: (r) => r._measurement == "client_traffic")
|
||||
|> filter(fn: (r) => r._field == "bytes")
|
||||
|> group(columns: ["mac", "direction"])
|
||||
|> sum()
|
||||
`;
|
||||
|
||||
const perMac = new Map<string, { upload: number; download: number }>();
|
||||
try {
|
||||
await queryApi.collectRows(fluxQuery, (row, tableMeta) => {
|
||||
const mac = tableMeta.get(row, 'mac') as string;
|
||||
const direction = tableMeta.get(row, 'direction') as string;
|
||||
const value = Number(tableMeta.get(row, '_value')) || 0;
|
||||
let entry = perMac.get(mac);
|
||||
if (!entry) {
|
||||
entry = { upload: 0, download: 0 };
|
||||
perMac.set(mac, entry);
|
||||
}
|
||||
if (direction === 'upload') entry.upload += value;
|
||||
else entry.download += value;
|
||||
});
|
||||
} catch {
|
||||
// No data yet
|
||||
}
|
||||
|
||||
const ranked = Array.from(perMac.entries())
|
||||
.map(([mac, e]) => ({ mac, upload_bytes: e.upload, download_bytes: e.download, total_bytes: e.upload + e.download }))
|
||||
.sort((a, b) => b.total_bytes - a.total_bytes)
|
||||
.slice(0, limit);
|
||||
|
||||
// Enrich real MACs with client identity from Postgres
|
||||
const realMacs = ranked.map((r) => r.mac).filter((m) => m !== 'unknown' && m !== 'other');
|
||||
const names = new Map<string, { hostname: string | null; custom_name: string | null; vendor: string | null; ip_address: string | null }>();
|
||||
if (realMacs.length > 0) {
|
||||
const rows = await query<{ mac_address: string; hostname: string | null; custom_name: string | null; vendor: string | null; ip_address: string | null }>(
|
||||
`SELECT DISTINCT ON (mac_address) mac_address, hostname, custom_name, vendor, ip_address
|
||||
FROM clients WHERE LOWER(mac_address) = ANY($1)
|
||||
ORDER BY mac_address, last_seen DESC NULLS LAST`,
|
||||
[realMacs]
|
||||
);
|
||||
for (const row of rows) names.set(row.mac_address.toLowerCase(), row);
|
||||
}
|
||||
|
||||
res.json(
|
||||
ranked.map((r) => {
|
||||
const info = names.get(r.mac);
|
||||
return {
|
||||
...r,
|
||||
hostname: info?.hostname || null,
|
||||
custom_name: info?.custom_name || null,
|
||||
vendor: info?.vendor || null,
|
||||
ip_address: info?.ip_address || null,
|
||||
};
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
// GET /api/traffic/apps?range=24h&mac=xx:xx:.. — app-category byte totals
|
||||
// (fleet-wide, or for a single client when mac is given)
|
||||
router.get('/apps', async (req: Request, res: Response) => {
|
||||
const range = rangeToFlux(String(req.query.range || '24h'));
|
||||
const queryApi = getQueryApi();
|
||||
|
||||
let macFilter = '';
|
||||
if (req.query.mac) {
|
||||
const mac = sanitizeMac(String(req.query.mac));
|
||||
if (!mac) return res.status(400).json({ error: 'Invalid mac' });
|
||||
macFilter = `|> filter(fn: (r) => r.mac == "${mac}")`;
|
||||
}
|
||||
|
||||
const fluxQuery = `
|
||||
from(bucket: "${bucket}")
|
||||
|> range(start: -${range})
|
||||
|> filter(fn: (r) => r._measurement == "client_traffic")
|
||||
|> filter(fn: (r) => r._field == "bytes")
|
||||
${macFilter}
|
||||
|> group(columns: ["app"])
|
||||
|> sum()
|
||||
`;
|
||||
|
||||
const apps: { app: string; bytes: number }[] = [];
|
||||
try {
|
||||
await queryApi.collectRows(fluxQuery, (row, tableMeta) => {
|
||||
apps.push({
|
||||
app: tableMeta.get(row, 'app') as string,
|
||||
bytes: Number(tableMeta.get(row, '_value')) || 0,
|
||||
});
|
||||
});
|
||||
} catch {
|
||||
// No data yet
|
||||
}
|
||||
|
||||
res.json(apps.sort((a, b) => b.bytes - a.bytes));
|
||||
});
|
||||
|
||||
// GET /api/traffic/client/:mac?range=24h — per-client time series + app breakdown
|
||||
router.get('/client/:mac', async (req: Request, res: Response) => {
|
||||
const mac = sanitizeMac(req.params.mac);
|
||||
if (!mac) return res.status(400).json({ error: 'Invalid mac' });
|
||||
const range = rangeToFlux(String(req.query.range || '24h'));
|
||||
const every = windowForRange(range);
|
||||
const queryApi = getQueryApi();
|
||||
|
||||
const seriesFlux = `
|
||||
from(bucket: "${bucket}")
|
||||
|> range(start: -${range})
|
||||
|> filter(fn: (r) => r._measurement == "client_traffic")
|
||||
|> filter(fn: (r) => r._field == "bytes")
|
||||
|> filter(fn: (r) => r.mac == "${mac}")
|
||||
|> group(columns: ["direction"])
|
||||
|> aggregateWindow(every: ${every}, fn: sum, createEmpty: false)
|
||||
`;
|
||||
|
||||
const pivoted: Record<string, { time: string; upload: number; download: number }> = {};
|
||||
try {
|
||||
await queryApi.collectRows(seriesFlux, (row, tableMeta) => {
|
||||
const time = tableMeta.get(row, '_time') as string;
|
||||
const direction = tableMeta.get(row, 'direction') as string;
|
||||
const value = Number(tableMeta.get(row, '_value')) || 0;
|
||||
if (!pivoted[time]) pivoted[time] = { time, upload: 0, download: 0 };
|
||||
if (direction === 'upload') pivoted[time].upload += value;
|
||||
else if (direction === 'download') pivoted[time].download += value;
|
||||
});
|
||||
} catch {
|
||||
// No data yet
|
||||
}
|
||||
|
||||
const appsFlux = `
|
||||
from(bucket: "${bucket}")
|
||||
|> range(start: -${range})
|
||||
|> filter(fn: (r) => r._measurement == "client_traffic")
|
||||
|> filter(fn: (r) => r._field == "bytes")
|
||||
|> filter(fn: (r) => r.mac == "${mac}")
|
||||
|> group(columns: ["app"])
|
||||
|> sum()
|
||||
`;
|
||||
|
||||
const apps: { app: string; bytes: number }[] = [];
|
||||
try {
|
||||
await queryApi.collectRows(appsFlux, (row, tableMeta) => {
|
||||
apps.push({
|
||||
app: tableMeta.get(row, 'app') as string,
|
||||
bytes: Number(tableMeta.get(row, '_value')) || 0,
|
||||
});
|
||||
});
|
||||
} catch {
|
||||
// No data yet
|
||||
}
|
||||
|
||||
res.json({
|
||||
series: Object.values(pivoted).sort((a, b) => a.time.localeCompare(b.time)),
|
||||
apps: apps.sort((a, b) => b.bytes - a.bytes),
|
||||
});
|
||||
});
|
||||
|
||||
export default router;
|
||||
Reference in New Issue
Block a user