diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9bc8ed..0a5b0f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,21 @@ on: branches: [main] jobs: + # Cheap, and guards a file no maintainer runs. The published deployment file is + # generated from docker-compose.yml; when they were maintained separately the + # published one silently lost the NetFlow port, every poller tunable and the + # Redis memory ceiling, and it took an outside report to notice (#121). + compose: + name: Deployment files in sync + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: '20' + - name: Check docker-compose.ghcr.yml is current + run: node scripts/generate-ghcr-compose.mjs --check + frontend: name: Frontend — Lint, Type-check & Build runs-on: ubuntu-latest diff --git a/README.md b/README.md index dea4e8c..245bdbb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@

- Version + Version License Docker TypeScript diff --git a/backend/package.json b/backend/package.json index b9f8272..c49ebf6 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "mikrotik-manager-backend", - "version": "0.23.33-beta", + "version": "0.23.34-beta", "description": "Mikrotik Network Management Platform - Backend API", "main": "dist/index.js", "scripts": { diff --git a/docker-compose.ghcr.yml b/docker-compose.ghcr.yml index 5c8ef82..528f6cb 100644 --- a/docker-compose.ghcr.yml +++ b/docker-compose.ghcr.yml @@ -1,7 +1,15 @@ version: '3.8' +# ───────────────────────────────────────────────────────────────────────────── +# GENERATED FILE — DO NOT EDIT. +# +# Produced from docker-compose.yml by scripts/generate-ghcr-compose.mjs. +# Edit that file and run `npm run compose:generate`; CI fails if this one is +# out of date. +# # Pre-built image deployment — no source code or build toolchain required. -# For development/build-from-source, use docker-compose.yml instead. +# For development or building from source, use docker-compose.yml instead. +# ───────────────────────────────────────────────────────────────────────────── services: nginx: @@ -21,8 +29,8 @@ services: backend: image: ghcr.io/2gt-media-group-llc/mikrotik-manager-backend:latest environment: - TZ: ${TZ:-UTC} NODE_ENV: production + TZ: ${TZ:-UTC} DATABASE_URL: postgresql://mikrotik:${DB_PASSWORD:-mikrotik_secure_pw}@postgres:5432/mikrotik_manager REDIS_URL: redis://redis:6379 INFLUXDB_URL: http://influxdb:8086 @@ -35,6 +43,18 @@ services: ENCRYPTION_KEY: ${ENCRYPTION_KEY:-} CORS_ORIGIN: ${CORS_ORIGIN:-https://localhost,http://localhost,https://127.0.0.1,http://127.0.0.1} PORT: 3001 + NETFLOW_BIND_PORT: 2055 + # Poller tuning. These were documented in .env.example but never passed + # through, so setting them had no effect on the container at all. + POLLER_CONCURRENCY: ${POLLER_CONCURRENCY:-12} + POLLER_JOB_TIMEOUT_MS: ${POLLER_JOB_TIMEOUT_MS:-45000} + POLLER_INTERVAL_MS: ${POLLER_INTERVAL_MS:-30000} + POLLER_JOB_RETENTION_SEC: ${POLLER_JOB_RETENTION_SEC:-3600} + POLLER_JOB_RETENTION_COUNT: ${POLLER_JOB_RETENTION_COUNT:-5000} + ports: + # NetFlow/IPFIX collector — UDP goes straight to the backend (nginx not involved). + # Host port must match the netflow_collector_port app setting (default 2055). + - "${NETFLOW_PORT:-2055}:2055/udp" depends_on: postgres: condition: service_healthy @@ -61,6 +81,7 @@ services: postgres: image: postgres:15-alpine environment: + TZ: ${TZ:-UTC} POSTGRES_DB: mikrotik_manager POSTGRES_USER: mikrotik POSTGRES_PASSWORD: ${DB_PASSWORD:-mikrotik_secure_pw} @@ -78,7 +99,21 @@ services: redis: image: redis:7-alpine - command: redis-server --appendonly yes + # A memory ceiling so a runaway producer can never take the host down, with + # eviction disabled. + # + # volatile-lru looks tempting — it would evict only keys with an expiry, and + # BullMQ's job hashes have none. But its *control* keys do: job locks and + # per-queue stalled-check markers both carry TTLs, and losing one causes + # duplicate processing or a stalled-job storm. BullMQ warns about this on + # connect, and it is right. noeviction means Redis refuses writes when full + # rather than quietly corrupting queue state: the host stays up, and the + # failure is loud. Retention limits and the stale-pending sweep are what keep + # us away from the ceiling in the first place. + command: > + redis-server --appendonly yes + --maxmemory ${REDIS_MAXMEMORY:-512mb} + --maxmemory-policy noeviction volumes: - redis_data:/data restart: unless-stopped diff --git a/docs/architecture.md b/docs/architecture.md index c1322b3..d3c4467 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -98,7 +98,7 @@ mikrotik-manager/ ├── nginx/ # Reverse proxy config and Dockerfile ├── scripts/ # CI preflight and tooling ├── docker-compose.yml # Build from source -├── docker-compose.ghcr.yml # Pre-built images +├── docker-compose.ghcr.yml # Pre-built images — GENERATED, do not edit by hand └── .env.example ``` diff --git a/frontend/package.json b/frontend/package.json index a6cf3d5..1fd5b30 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mikrotik-manager-frontend", - "version": "0.23.33-beta", + "version": "0.23.34-beta", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/version.ts b/frontend/src/version.ts index a78ff10..79851c8 100644 --- a/frontend/src/version.ts +++ b/frontend/src/version.ts @@ -1 +1 @@ -export const APP_VERSION = 'v0.23.33 Beta'; +export const APP_VERSION = 'v0.23.34 Beta'; diff --git a/scripts/ci-preflight.sh b/scripts/ci-preflight.sh index 9032307..c418c7c 100755 --- a/scripts/ci-preflight.sh +++ b/scripts/ci-preflight.sh @@ -38,6 +38,11 @@ run() { : >/tmp/ci-preflight-failures echo "Running CI gates locally (mirrors .github/workflows/ci.yml)" +# The published deployment file is generated. Checking it first because it costs +# nothing and because drift here is invisible to every maintainer — it only ever +# breaks the Quick Deploy path that no one on the team runs (#121). +run "compose: ghcr in sync" . node scripts/generate-ghcr-compose.mjs --check + run "backend: lint" backend npm run lint run "backend: type-check" backend npx tsc --noEmit run "backend: build" backend npm run build diff --git a/scripts/generate-ghcr-compose.mjs b/scripts/generate-ghcr-compose.mjs new file mode 100755 index 0000000..1670c5c --- /dev/null +++ b/scripts/generate-ghcr-compose.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node +/** + * Generate docker-compose.ghcr.yml from docker-compose.yml. + * + * The two files describe the same system and differ only in where the images + * come from. Maintaining both by hand meant every change had to be remembered + * twice, and the one that got forgotten was the Quick Deploy path — the file the + * README tells every new user to download, and the one no maintainer runs. + * + * That drift was not hypothetical. By the time it was reported (#121) the + * published file had been missing the NetFlow port for weeks, along with all + * five poller tunables and the Redis memory ceiling, each of which had been + * added while closing a different issue. + * + * Substitution is textual rather than parse-and-reserialise so that comments + * survive — the reasoning about Redis eviction is worth more than the two lines + * of config it explains. + * + * Run `npm run compose:generate`. CI regenerates and fails if the result differs + * from what is committed, so the two cannot diverge again. + */ +import { readFileSync, writeFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const root = join(dirname(fileURLToPath(import.meta.url)), '..'); +const SOURCE = join(root, 'docker-compose.yml'); +const TARGET = join(root, 'docker-compose.ghcr.yml'); + +const REGISTRY = 'ghcr.io/2gt-media-group-llc'; + +/** The only intentional differences between the two files. */ +const IMAGE_SWAPS = [ + { + service: 'nginx', + build: ' build:\n context: .\n dockerfile: nginx/Dockerfile\n', + image: ` image: ${REGISTRY}/mikrotik-manager-nginx:latest\n`, + }, + { + service: 'backend', + build: ' build:\n context: ./backend\n dockerfile: Dockerfile\n', + image: ` image: ${REGISTRY}/mikrotik-manager-backend:latest\n`, + }, +]; + +const BANNER = ` +# ───────────────────────────────────────────────────────────────────────────── +# GENERATED FILE — DO NOT EDIT. +# +# Produced from docker-compose.yml by scripts/generate-ghcr-compose.mjs. +# Edit that file and run \`npm run compose:generate\`; CI fails if this one is +# out of date. +# +# Pre-built image deployment — no source code or build toolchain required. +# For development or building from source, use docker-compose.yml instead. +# ───────────────────────────────────────────────────────────────────────────── +`; + +function generate(source) { + let out = source; + + for (const swap of IMAGE_SWAPS) { + if (!out.includes(swap.build)) { + throw new Error( + `Could not find the build block for "${swap.service}" in docker-compose.yml. ` + + `The generator needs updating to match how that service is now defined.` + ); + } + out = out.replace(swap.build, swap.image); + } + + // Banner goes after the version line so the file still parses from line one. + const versionLine = out.match(/^version:.*\n/); + if (!versionLine) throw new Error('docker-compose.yml has no version line to anchor the banner to'); + return out.replace(versionLine[0], versionLine[0] + BANNER); +} + +const generated = generate(readFileSync(SOURCE, 'utf8')); +const check = process.argv.includes('--check'); +const current = (() => { try { return readFileSync(TARGET, 'utf8'); } catch { return null; } })(); + +if (check) { + if (current !== generated) { + console.error( + 'docker-compose.ghcr.yml is out of date.\n' + + 'Run `npm run compose:generate` and commit the result.\n\n' + + 'This guard exists because the published deployment silently lost the NetFlow\n' + + 'port, the poller tunables and the Redis memory ceiling when they were added\n' + + 'to docker-compose.yml alone (#121).' + ); + process.exit(1); + } + console.log('docker-compose.ghcr.yml is up to date'); +} else { + writeFileSync(TARGET, generated); + console.log(current === generated ? 'docker-compose.ghcr.yml unchanged' : 'docker-compose.ghcr.yml regenerated'); +}