Files
MIKROTIK-RMM/backend/src/services/netflow/__tests__/aggregator.test.ts
T
2GT-RichandClaude Fable 5 b841eb6e7b feat: NetFlow/IPFIX traffic analytics — per-client usage, app breakdown, built-in collector (v0.13.0-beta)
Adds Meraki-style traffic analytics powered by MikroTik Traffic Flow:

Backend
- Built-in NetFlow v9 + IPFIX collector (UDP 2055) with hand-rolled decoder,
  template caching, enterprise-field and IPv6 support
- Flow attribution: client IP → MAC via the clients table, upload/download
  direction, NAT-duplicate records dropped
- App classification by protocol/port (~25 categories: HTTPS, QUIC, DNS,
  SSH, Email, WireGuard, ...)
- Deduplication: per-client best-exporter-per-window so flows crossing two
  managed routers are never double-counted
- NAT-tolerant ingest: flows from sources matching no managed device (NAT /
  Tailscale subnet routers in path) are accepted as per-source unidentified
  exporters instead of silently dropped (netflow_accept_unknown setting)
- Storage: InfluxDB client_traffic time-series (bounded cardinality via
  top-N fold) + Postgres client_traffic_daily rollups; daily retention purge
- One-toggle device config push via RouterOS API (/ip traffic-flow + target)
- New /api/traffic/* analytics endpoints + /api/network-services/netflow
- docker-compose: NETFLOW_PORT (default 2055/udp) published on the backend

Frontend
- New Traffic page: fleet up/down chart, top talkers with client
  click-through, app-category donut, 1h–30d ranges
- New NetFlow config page: collector settings, live ingest stats,
  per-device export toggles with health badges, NAT-source banner
- Client detail: App Traffic card (auto-hides without data)
- Clients page: sortable "Data (today)" column
- Settings: NetFlow retention controls

Includes 17 unit tests (decoder fixtures, dedup, attribution, classifier);
verified end-to-end against live RouterOS devices.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-06-10 10:58:41 -07:00

108 lines
4.8 KiB
TypeScript

import { FlowAggregator, CLIENT_UNKNOWN, CLIENT_OTHER } from '../FlowAggregator';
import { classifyApp } from '../appCategories';
const MAC_A = 'aa:aa:aa:aa:aa:01';
const MAC_B = 'aa:aa:aa:aa:aa:02';
describe('FlowAggregator dedup', () => {
it('keeps only the exporter that saw the most bytes for each client', () => {
const agg = new FlowAggregator();
// Gateway (device 1) sees all of client A's traffic; an intermediate
// switch/router (device 2) sees a subset of the same flows.
agg.add({ exporterId: 1, clientKey: MAC_A, direction: 'download', app: 'HTTPS', bytes: 10_000, packets: 20 });
agg.add({ exporterId: 2, clientKey: MAC_A, direction: 'download', app: 'HTTPS', bytes: 7_000, packets: 14 });
const rows = agg.drain(50);
expect(rows).toHaveLength(1);
expect(rows[0].bytes).toBe(10_000);
});
it('breaks byte ties deterministically by lowest exporter id', () => {
const agg = new FlowAggregator();
agg.add({ exporterId: 5, clientKey: MAC_A, direction: 'upload', app: 'DNS', bytes: 100, packets: 1 });
agg.add({ exporterId: 3, clientKey: MAC_A, direction: 'upload', app: 'QUIC', bytes: 100, packets: 1 });
const rows = agg.drain(50);
expect(rows).toHaveLength(1);
expect(rows[0].app).toBe('QUIC'); // exporter 3 wins the tie
});
it('picks the winning exporter per client independently', () => {
const agg = new FlowAggregator();
agg.add({ exporterId: 1, clientKey: MAC_A, direction: 'download', app: 'HTTPS', bytes: 500, packets: 1 });
agg.add({ exporterId: 2, clientKey: MAC_A, direction: 'download', app: 'HTTPS', bytes: 100, packets: 1 });
agg.add({ exporterId: 1, clientKey: MAC_B, direction: 'download', app: 'HTTPS', bytes: 100, packets: 1 });
agg.add({ exporterId: 2, clientKey: MAC_B, direction: 'download', app: 'HTTPS', bytes: 900, packets: 1 });
const rows = agg.drain(50);
const a = rows.find((r) => r.clientKey === MAC_A);
const b = rows.find((r) => r.clientKey === MAC_B);
expect(a?.bytes).toBe(500); // device 1 wins for A
expect(b?.bytes).toBe(900); // device 2 wins for B
});
it('accumulates repeated samples within the window and clears on drain', () => {
const agg = new FlowAggregator();
agg.add({ exporterId: 1, clientKey: MAC_A, direction: 'upload', app: 'HTTPS', bytes: 100, packets: 1 });
agg.add({ exporterId: 1, clientKey: MAC_A, direction: 'upload', app: 'HTTPS', bytes: 250, packets: 2 });
const rows = agg.drain(50);
expect(rows).toHaveLength(1);
expect(rows[0].bytes).toBe(350);
expect(rows[0].packets).toBe(3);
expect(agg.drain(50)).toHaveLength(0);
});
});
describe('FlowAggregator top-N fold', () => {
it('folds clients beyond the top-N into the "other" bucket', () => {
const agg = new FlowAggregator();
for (let i = 0; i < 5; i++) {
const mac = `aa:aa:aa:aa:bb:0${i}`;
agg.add({ exporterId: 1, clientKey: mac, direction: 'download', app: 'HTTPS', bytes: (i + 1) * 1000, packets: 1 });
}
const rows = agg.drain(2);
const macs = new Set(rows.map((r) => r.clientKey));
expect(macs.has('aa:aa:aa:aa:bb:04')).toBe(true); // biggest
expect(macs.has('aa:aa:aa:aa:bb:03')).toBe(true); // second
expect(macs.has(CLIENT_OTHER)).toBe(true);
const other = rows.find((r) => r.clientKey === CLIENT_OTHER);
expect(other?.bytes).toBe(1000 + 2000 + 3000); // the three folded clients
});
it('never folds the unknown pseudo-client and does not let it consume a top-N slot', () => {
const agg = new FlowAggregator();
agg.add({ exporterId: 1, clientKey: CLIENT_UNKNOWN, direction: 'download', app: 'HTTPS', bytes: 999_999, packets: 1 });
agg.add({ exporterId: 1, clientKey: MAC_A, direction: 'download', app: 'HTTPS', bytes: 10, packets: 1 });
const rows = agg.drain(1);
const macs = new Set(rows.map((r) => r.clientKey));
expect(macs.has(CLIENT_UNKNOWN)).toBe(true);
expect(macs.has(MAC_A)).toBe(true); // still fits — unknown didn't take its slot
});
});
describe('classifyApp', () => {
it('classifies by the remote port first', () => {
expect(classifyApp(6, 54321, 443)).toBe('HTTPS');
expect(classifyApp(17, 54321, 443)).toBe('QUIC');
expect(classifyApp(17, 50000, 53)).toBe('DNS');
expect(classifyApp(6, 50000, 22)).toBe('SSH');
});
it('falls back to the local port for inbound services', () => {
// Remote ephemeral port, local well-known port (client hosts SSH)
expect(classifyApp(6, 22, 51515)).toBe('SSH');
});
it('handles non-port protocols and fallbacks', () => {
expect(classifyApp(1, 0, 0)).toBe('ICMP');
expect(classifyApp(58, 0, 0)).toBe('ICMP');
expect(classifyApp(47, 0, 0)).toBe('GRE');
expect(classifyApp(6, 49152, 49153)).toBe('Other TCP');
expect(classifyApp(17, 49152, 49153)).toBe('Other UDP');
expect(classifyApp(132, 0, 0)).toBe('Other');
});
});