From 273a0da7375897c24b0632e5a384f790cfbb0f9a Mon Sep 17 00:00:00 2001 From: 2GT-Rich <107219997+2GT-Rich@users.noreply.github.com> Date: Sun, 6 Sep 2026 13:08:56 -0700 Subject: [PATCH] fix(deploy): restore NetFlow to the published compose file, and generate it (v0.23.34-beta) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #121. The Quick Deploy file was missing NETFLOW_BIND_PORT and the 2055/udp mapping, so the collector bound inside the container and nothing reached it. NetFlow did nothing, silently, with an empty traffic page and no error — for everyone following the path the README tells new users to take. The report was accurate and understated. docker-compose.ghcr.yml had drifted for seven weeks, and three closed issues had never reached it: * NETFLOW_BIND_PORT and the UDP port — as reported * all five POLLER_* tunables — #114, whose fix was "pass the tunables through" * the Redis memory ceiling and eviction policy — #116 * TZ on the postgres service, which nobody had noticed at all The middle one is the uncomfortable part. Four days ago #114 was closed by forwarding poller settings that were documented but inert, with a comment saying setting them "did precisely nothing". On Quick Deploy that remained true, because one compose file was fixed and the other was not — including the advice given to that reporter about which knob to turn. So patching the four gaps would have been treating the symptom. The two files describe the same system and differ only in where images come from; maintaining both by hand meant remembering every change twice, and the one forgotten was always the file no maintainer runs. docker-compose.ghcr.yml is now generated from docker-compose.yml by scripts/generate-ghcr-compose.mjs, which performs the two build-to-image swaps and nothing else. Substitution is textual rather than parse-and-reserialise so comments survive — the reasoning about Redis eviction is worth more than the two lines it explains. The file stays committed and browsable, because the README tells people to curl it directly. A --check mode runs as the first preflight gate and as its own CI job, both of which fail when the committed file does not match what the generator produces. Verified by deleting the NetFlow line and watching the guard reject it, with the reason and the fix in the failure message. Reported by @mamb1329, on their first issue. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 15 +++++ README.md | 2 +- backend/package.json | 2 +- docker-compose.ghcr.yml | 41 ++++++++++++- docs/architecture.md | 2 +- frontend/package.json | 2 +- frontend/src/version.ts | 2 +- scripts/ci-preflight.sh | 5 ++ scripts/generate-ghcr-compose.mjs | 97 +++++++++++++++++++++++++++++++ 9 files changed, 160 insertions(+), 8 deletions(-) create mode 100755 scripts/generate-ghcr-compose.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9bc8ed..0a5b0f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,21 @@ on: branches: [main] jobs: + # Cheap, and guards a file no maintainer runs. The published deployment file is + # generated from docker-compose.yml; when they were maintained separately the + # published one silently lost the NetFlow port, every poller tunable and the + # Redis memory ceiling, and it took an outside report to notice (#121). + compose: + name: Deployment files in sync + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: '20' + - name: Check docker-compose.ghcr.yml is current + run: node scripts/generate-ghcr-compose.mjs --check + frontend: name: Frontend — Lint, Type-check & Build runs-on: ubuntu-latest diff --git a/README.md b/README.md index dea4e8c..245bdbb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@

- Version + Version License Docker TypeScript diff --git a/backend/package.json b/backend/package.json index b9f8272..c49ebf6 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "mikrotik-manager-backend", - "version": "0.23.33-beta", + "version": "0.23.34-beta", "description": "Mikrotik Network Management Platform - Backend API", "main": "dist/index.js", "scripts": { diff --git a/docker-compose.ghcr.yml b/docker-compose.ghcr.yml index 5c8ef82..528f6cb 100644 --- a/docker-compose.ghcr.yml +++ b/docker-compose.ghcr.yml @@ -1,7 +1,15 @@ version: '3.8' +# ───────────────────────────────────────────────────────────────────────────── +# GENERATED FILE — DO NOT EDIT. +# +# Produced from docker-compose.yml by scripts/generate-ghcr-compose.mjs. +# Edit that file and run `npm run compose:generate`; CI fails if this one is +# out of date. +# # Pre-built image deployment — no source code or build toolchain required. -# For development/build-from-source, use docker-compose.yml instead. +# For development or building from source, use docker-compose.yml instead. +# ───────────────────────────────────────────────────────────────────────────── services: nginx: @@ -21,8 +29,8 @@ services: backend: image: ghcr.io/2gt-media-group-llc/mikrotik-manager-backend:latest environment: - TZ: ${TZ:-UTC} NODE_ENV: production + TZ: ${TZ:-UTC} DATABASE_URL: postgresql://mikrotik:${DB_PASSWORD:-mikrotik_secure_pw}@postgres:5432/mikrotik_manager REDIS_URL: redis://redis:6379 INFLUXDB_URL: http://influxdb:8086 @@ -35,6 +43,18 @@ services: ENCRYPTION_KEY: ${ENCRYPTION_KEY:-} CORS_ORIGIN: ${CORS_ORIGIN:-https://localhost,http://localhost,https://127.0.0.1,http://127.0.0.1} PORT: 3001 + NETFLOW_BIND_PORT: 2055 + # Poller tuning. These were documented in .env.example but never passed + # through, so setting them had no effect on the container at all. + POLLER_CONCURRENCY: ${POLLER_CONCURRENCY:-12} + POLLER_JOB_TIMEOUT_MS: ${POLLER_JOB_TIMEOUT_MS:-45000} + POLLER_INTERVAL_MS: ${POLLER_INTERVAL_MS:-30000} + POLLER_JOB_RETENTION_SEC: ${POLLER_JOB_RETENTION_SEC:-3600} + POLLER_JOB_RETENTION_COUNT: ${POLLER_JOB_RETENTION_COUNT:-5000} + ports: + # NetFlow/IPFIX collector — UDP goes straight to the backend (nginx not involved). + # Host port must match the netflow_collector_port app setting (default 2055). + - "${NETFLOW_PORT:-2055}:2055/udp" depends_on: postgres: condition: service_healthy @@ -61,6 +81,7 @@ services: postgres: image: postgres:15-alpine environment: + TZ: ${TZ:-UTC} POSTGRES_DB: mikrotik_manager POSTGRES_USER: mikrotik POSTGRES_PASSWORD: ${DB_PASSWORD:-mikrotik_secure_pw} @@ -78,7 +99,21 @@ services: redis: image: redis:7-alpine - command: redis-server --appendonly yes + # A memory ceiling so a runaway producer can never take the host down, with + # eviction disabled. + # + # volatile-lru looks tempting — it would evict only keys with an expiry, and + # BullMQ's job hashes have none. But its *control* keys do: job locks and + # per-queue stalled-check markers both carry TTLs, and losing one causes + # duplicate processing or a stalled-job storm. BullMQ warns about this on + # connect, and it is right. noeviction means Redis refuses writes when full + # rather than quietly corrupting queue state: the host stays up, and the + # failure is loud. Retention limits and the stale-pending sweep are what keep + # us away from the ceiling in the first place. + command: > + redis-server --appendonly yes + --maxmemory ${REDIS_MAXMEMORY:-512mb} + --maxmemory-policy noeviction volumes: - redis_data:/data restart: unless-stopped diff --git a/docs/architecture.md b/docs/architecture.md index c1322b3..d3c4467 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -98,7 +98,7 @@ mikrotik-manager/ ├── nginx/ # Reverse proxy config and Dockerfile ├── scripts/ # CI preflight and tooling ├── docker-compose.yml # Build from source -├── docker-compose.ghcr.yml # Pre-built images +├── docker-compose.ghcr.yml # Pre-built images — GENERATED, do not edit by hand └── .env.example ``` diff --git a/frontend/package.json b/frontend/package.json index a6cf3d5..1fd5b30 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mikrotik-manager-frontend", - "version": "0.23.33-beta", + "version": "0.23.34-beta", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/version.ts b/frontend/src/version.ts index a78ff10..79851c8 100644 --- a/frontend/src/version.ts +++ b/frontend/src/version.ts @@ -1 +1 @@ -export const APP_VERSION = 'v0.23.33 Beta'; +export const APP_VERSION = 'v0.23.34 Beta'; diff --git a/scripts/ci-preflight.sh b/scripts/ci-preflight.sh index 9032307..c418c7c 100755 --- a/scripts/ci-preflight.sh +++ b/scripts/ci-preflight.sh @@ -38,6 +38,11 @@ run() { : >/tmp/ci-preflight-failures echo "Running CI gates locally (mirrors .github/workflows/ci.yml)" +# The published deployment file is generated. Checking it first because it costs +# nothing and because drift here is invisible to every maintainer — it only ever +# breaks the Quick Deploy path that no one on the team runs (#121). +run "compose: ghcr in sync" . node scripts/generate-ghcr-compose.mjs --check + run "backend: lint" backend npm run lint run "backend: type-check" backend npx tsc --noEmit run "backend: build" backend npm run build diff --git a/scripts/generate-ghcr-compose.mjs b/scripts/generate-ghcr-compose.mjs new file mode 100755 index 0000000..1670c5c --- /dev/null +++ b/scripts/generate-ghcr-compose.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node +/** + * Generate docker-compose.ghcr.yml from docker-compose.yml. + * + * The two files describe the same system and differ only in where the images + * come from. Maintaining both by hand meant every change had to be remembered + * twice, and the one that got forgotten was the Quick Deploy path — the file the + * README tells every new user to download, and the one no maintainer runs. + * + * That drift was not hypothetical. By the time it was reported (#121) the + * published file had been missing the NetFlow port for weeks, along with all + * five poller tunables and the Redis memory ceiling, each of which had been + * added while closing a different issue. + * + * Substitution is textual rather than parse-and-reserialise so that comments + * survive — the reasoning about Redis eviction is worth more than the two lines + * of config it explains. + * + * Run `npm run compose:generate`. CI regenerates and fails if the result differs + * from what is committed, so the two cannot diverge again. + */ +import { readFileSync, writeFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const root = join(dirname(fileURLToPath(import.meta.url)), '..'); +const SOURCE = join(root, 'docker-compose.yml'); +const TARGET = join(root, 'docker-compose.ghcr.yml'); + +const REGISTRY = 'ghcr.io/2gt-media-group-llc'; + +/** The only intentional differences between the two files. */ +const IMAGE_SWAPS = [ + { + service: 'nginx', + build: ' build:\n context: .\n dockerfile: nginx/Dockerfile\n', + image: ` image: ${REGISTRY}/mikrotik-manager-nginx:latest\n`, + }, + { + service: 'backend', + build: ' build:\n context: ./backend\n dockerfile: Dockerfile\n', + image: ` image: ${REGISTRY}/mikrotik-manager-backend:latest\n`, + }, +]; + +const BANNER = ` +# ───────────────────────────────────────────────────────────────────────────── +# GENERATED FILE — DO NOT EDIT. +# +# Produced from docker-compose.yml by scripts/generate-ghcr-compose.mjs. +# Edit that file and run \`npm run compose:generate\`; CI fails if this one is +# out of date. +# +# Pre-built image deployment — no source code or build toolchain required. +# For development or building from source, use docker-compose.yml instead. +# ───────────────────────────────────────────────────────────────────────────── +`; + +function generate(source) { + let out = source; + + for (const swap of IMAGE_SWAPS) { + if (!out.includes(swap.build)) { + throw new Error( + `Could not find the build block for "${swap.service}" in docker-compose.yml. ` + + `The generator needs updating to match how that service is now defined.` + ); + } + out = out.replace(swap.build, swap.image); + } + + // Banner goes after the version line so the file still parses from line one. + const versionLine = out.match(/^version:.*\n/); + if (!versionLine) throw new Error('docker-compose.yml has no version line to anchor the banner to'); + return out.replace(versionLine[0], versionLine[0] + BANNER); +} + +const generated = generate(readFileSync(SOURCE, 'utf8')); +const check = process.argv.includes('--check'); +const current = (() => { try { return readFileSync(TARGET, 'utf8'); } catch { return null; } })(); + +if (check) { + if (current !== generated) { + console.error( + 'docker-compose.ghcr.yml is out of date.\n' + + 'Run `npm run compose:generate` and commit the result.\n\n' + + 'This guard exists because the published deployment silently lost the NetFlow\n' + + 'port, the poller tunables and the Redis memory ceiling when they were added\n' + + 'to docker-compose.yml alone (#121).' + ); + process.exit(1); + } + console.log('docker-compose.ghcr.yml is up to date'); +} else { + writeFileSync(TARGET, generated); + console.log(current === generated ? 'docker-compose.ghcr.yml unchanged' : 'docker-compose.ghcr.yml regenerated'); +}