feat: NetFlow/IPFIX traffic analytics — per-client usage, app breakdown, built-in collector (v0.13.0-beta)
Adds Meraki-style traffic analytics powered by MikroTik Traffic Flow: Backend - Built-in NetFlow v9 + IPFIX collector (UDP 2055) with hand-rolled decoder, template caching, enterprise-field and IPv6 support - Flow attribution: client IP → MAC via the clients table, upload/download direction, NAT-duplicate records dropped - App classification by protocol/port (~25 categories: HTTPS, QUIC, DNS, SSH, Email, WireGuard, ...) - Deduplication: per-client best-exporter-per-window so flows crossing two managed routers are never double-counted - NAT-tolerant ingest: flows from sources matching no managed device (NAT / Tailscale subnet routers in path) are accepted as per-source unidentified exporters instead of silently dropped (netflow_accept_unknown setting) - Storage: InfluxDB client_traffic time-series (bounded cardinality via top-N fold) + Postgres client_traffic_daily rollups; daily retention purge - One-toggle device config push via RouterOS API (/ip traffic-flow + target) - New /api/traffic/* analytics endpoints + /api/network-services/netflow - docker-compose: NETFLOW_PORT (default 2055/udp) published on the backend Frontend - New Traffic page: fleet up/down chart, top talkers with client click-through, app-category donut, 1h–30d ranges - New NetFlow config page: collector settings, live ingest stats, per-device export toggles with health badges, NAT-source banner - Client detail: App Traffic card (auto-hides without data) - Clients page: sortable "Data (today)" column - Settings: NetFlow retention controls Includes 17 unit tests (decoder fixtures, dedup, attribution, classifier); verified end-to-end against live RouterOS devices. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
367034c2d6
commit
b841eb6e7b
@@ -207,6 +207,17 @@ export class PollerService {
|
||||
);
|
||||
}
|
||||
|
||||
// NetFlow data retention — runs once per day. Purges old client_traffic
|
||||
// points from InfluxDB and old daily rollups from Postgres.
|
||||
const netflowPruneKey = 'task:netflow_retention';
|
||||
const lastNetflowPrune = await this.getTimestamp(netflowPruneKey);
|
||||
if (now - lastNetflowPrune > 86_400_000) {
|
||||
await this.setTimestamp(netflowPruneKey, now, 86_400 + 3_600);
|
||||
this.purgeNetflowData().catch((e) =>
|
||||
console.error('[Poller] NetFlow retention error:', e)
|
||||
);
|
||||
}
|
||||
|
||||
// Stale client pruning — runs once per hour
|
||||
// Deletes inactive client records not seen for longer than retention_clients_days.
|
||||
const pruneKey = 'task:prune_clients';
|
||||
@@ -353,6 +364,48 @@ export class PollerService {
|
||||
}
|
||||
}
|
||||
|
||||
private async purgeNetflowData(): Promise<void> {
|
||||
const rows = await query<{ key: string; value: unknown }>(
|
||||
`SELECT key, value FROM app_settings
|
||||
WHERE key IN ('netflow_retention_days', 'netflow_daily_retention_days')`
|
||||
);
|
||||
const map: Record<string, unknown> = {};
|
||||
for (const row of rows) map[row.key] = row.value;
|
||||
const detailDays = Number(map['netflow_retention_days']) || 30;
|
||||
const dailyDays = Number(map['netflow_daily_retention_days']) || 365;
|
||||
|
||||
// InfluxDB: delete client_traffic points older than the detail retention
|
||||
const { DeleteAPI } = await import('@influxdata/influxdb-client-apis');
|
||||
const { getInfluxClient, org, bucket } = await import('../config/influxdb');
|
||||
const stop = new Date(Date.now() - detailDays * 86_400_000).toISOString();
|
||||
await new DeleteAPI(getInfluxClient())
|
||||
.postDelete({
|
||||
org,
|
||||
bucket,
|
||||
body: {
|
||||
start: '1970-01-01T00:00:00Z',
|
||||
stop,
|
||||
predicate: '_measurement="client_traffic"',
|
||||
},
|
||||
})
|
||||
.catch((e) => console.error('[Poller] NetFlow Influx purge error:', (e as Error).message));
|
||||
|
||||
// Postgres: delete daily rollups older than the rollup retention
|
||||
const result = await query<{ count: string }>(
|
||||
`WITH deleted AS (
|
||||
DELETE FROM client_traffic_daily
|
||||
WHERE day < CURRENT_DATE - ($1 || ' days')::interval
|
||||
RETURNING 1
|
||||
)
|
||||
SELECT COUNT(*) AS count FROM deleted`,
|
||||
[dailyDays]
|
||||
);
|
||||
const count = parseInt(result[0]?.count || '0', 10);
|
||||
if (count > 0) {
|
||||
console.log(`[Poller] NetFlow retention pruned ${count} daily rollup row(s) (> ${dailyDays} days)`);
|
||||
}
|
||||
}
|
||||
|
||||
private async pruneStaleClients(settings: Record<string, unknown>): Promise<void> {
|
||||
// Delete inactive clients not seen for longer than the configured retention period.
|
||||
// Default: 7 days. Preserves any client that was active within the window so
|
||||
|
||||
Reference in New Issue
Block a user