diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..eb3e371 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,87 @@ +# Documentation site. +# +# Two jobs with different jobs to do. On a pull request the site is *built* and +# nothing is published — a broken link or a page missing from the nav fails the +# check rather than shipping. On main it is published under a version, because +# readers are routinely running something other than the newest build and a +# single "latest" would describe a release they do not have. +name: Documentation + +on: + push: + branches: [main] + paths: + - 'docs/**' + - 'mkdocs.yml' + - 'requirements-docs.txt' + - '.github/workflows/docs.yml' + - 'frontend/package.json' # version stamp comes from here + pull_request: + paths: + - 'docs/**' + - 'mkdocs.yml' + - 'requirements-docs.txt' + - '.github/workflows/docs.yml' + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: docs-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Build (check only) + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + cache: pip + - run: pip install -r requirements-docs.txt + # --strict turns warnings into errors, which is the point of running this + # on a pull request at all: a link to a page that no longer exists should + # fail here rather than 404 for a reader. + - run: mkdocs build --strict + + deploy: + name: Publish versioned site + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 # mike needs history to update gh-pages + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + cache: pip + - run: pip install -r requirements-docs.txt + + - name: Derive version from the shipped package + id: v + run: | + # Documentation is versioned with the product, not separately, so the + # site and the image a reader is running carry the same number. + FULL=$(node -p "require('./frontend/package.json').version") + SERIES=$(echo "$FULL" | sed -E 's/^([0-9]+\.[0-9]+).*/\1/') + echo "full=$FULL" >> "$GITHUB_OUTPUT" + echo "series=$SERIES" >> "$GITHUB_OUTPUT" + + - name: Configure git for mike + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Aliased to "latest" so links stay stable while older series remain + # readable at their own URLs. + - name: Deploy + run: | + mike deploy --push --update-aliases \ + "${{ steps.v.outputs.series }}" latest \ + --title "${{ steps.v.outputs.series }} (${{ steps.v.outputs.full }})" + mike set-default --push latest diff --git a/.gitignore b/.gitignore index 34aede5..50ac78a 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,4 @@ vite.config.ts.timestamp-* # macOS .DS_Store +site/ diff --git a/README.md b/README.md index 6ada7e3..f5ee3b8 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@

- Version + Version License Docker TypeScript @@ -360,7 +360,7 @@ Each network service supports multi-device management with conflict detection: ## Documentation -Deeper reference material lives in [`docs/`](docs/README.md), versioned alongside the code +Deeper reference material lives at **[the documentation site](https://2gt-media-group-llc.github.io/mikrotik-manager/)**, published per release so it matches the version you are running. The same pages are in [`docs/`](docs/README.md) if you would rather read them here. so it moves when behaviour does. | Document | Covers | diff --git a/backend/package.json b/backend/package.json index 71c0c19..1bcd1f9 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "mikrotik-manager-backend", - "version": "0.23.27-beta", + "version": "0.23.28-beta", "description": "Mikrotik Network Management Platform - Backend API", "main": "dist/index.js", "scripts": { diff --git a/docs/README.md b/docs/README.md index fcf68a3..8db6473 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,7 +1,7 @@ # MikroTik Manager documentation Reference material for people already running the platform. If you are still deciding -whether to try it, the [project README](../README.md) is the better starting point. +whether to try it, the [project README](https://github.com/2GT-Media-Group-LLC/mikrotik-manager/blob/main/README.md) is the better starting point. | Document | Covers | |---|---| @@ -10,6 +10,10 @@ whether to try it, the [project README](../README.md) is the better starting poi | [Alerting](alerting.md) | Alert rules, delivery channels, webhooks, scheduled reports, maintenance windows | | [Single sign-on (OIDC)](sso-oidc.md) | Identity provider setup, group-to-role mapping, break-glass behaviour | | [CAPsMAN](capsman.md) | How centrally provisioned access points are modelled, and current limits | +| [Cellular (LTE)](cellular.md) | Signal, carriers, tower movement, and the data-cap reset SMS | +| [SSH keys](ssh-keys.md) | Per-device keypairs, rotation, and what installing one costs | +| [Bulk commands](commands.md) | Running one command across a fleet, in waves | +| [Polling and scaling](scaling.md) | Poller health, headroom, tuning, and clearing a backlog | | [API and automation](api.md) | Scoped tokens, authentication, and driving the platform from scripts | | [Architecture](architecture.md) | How the system is put together, for contributors | @@ -18,3 +22,9 @@ whether to try it, the [project README](../README.md) is the better starting poi - **Settings in the UI** are stored in the database and apply without a restart. - **Environment variables** live in `.env` and require a container restart to change. - Anything described as *read-only* makes no writes to your devices. + +## Which version is this? + +These pages describe a specific release, shown in the header. The platform ships +frequently, so if something here does not match what you see, check that the version you +are running matches the version you are reading. diff --git a/docs/cellular.md b/docs/cellular.md new file mode 100644 index 0000000..624218e --- /dev/null +++ b/docs/cellular.md @@ -0,0 +1,131 @@ +# Cellular (LTE) + +[← Documentation index](README.md) + +Everything here is read from the modem your device already has. Nothing needs enabling +on the device for the monitoring half; the data-cap SMS at the end is the only part that +writes anything. + +## What a modem reports depends on the modem + +This is the single most important thing to know before reading the rest. RouterOS is not +the source of these fields — the cellular module is, and modules disagree. + +The platform was built against a **Quectel EG18-EA** (MikroTik ATL 18), which reports +`data-class` where the documentation says `access-technology`, `iccid` where WinBox shows +`UICC`, `status: running` rather than `connected`, and does not report +`registration-status`, `pin-status` or tracking area at all. + +Every field is therefore treated as optional. A panel showing "—" for something is +usually your modem not reporting it, not a fault. + +## Signal quality + +Four measurements are collected on the fast poll and kept as a time series: + +| Field | Unit | Meaning | +|---|---|---| +| RSRP | dBm | Power of the reference signal — how loud the tower is | +| RSRQ | dB | Quality of that signal | +| SINR | dB | Signal against noise — how *usable* it is | +| RSSI | dBm | Wideband received power | + +The panel grades the link rather than only printing numbers, and **SINR leads that +grade**. RSRP alone misleads: the reference device reports −97 dBm — nominally "fair" — +on a link running 256QAM at maximum reported channel quality with two spatial streams. +That is the modem at its ceiling. RSRP moderates the grade but cannot define it. + +Values that cannot be real are discarded rather than plotted. A positive dBm reading +appears in real logs and means the modem is reporting something other than a measurement. + +## Carriers and aggregation + +`primary-band` and `ca-band` arrive as composite strings — `B1@20Mhz earfcn: 500 +phy-cellid: 190` — and are parsed into band, bandwidth, EARFCN and physical cell id. + +A Cat-18 modem reports **one `ca-band` per aggregated carrier**. Because an API sentence +flattened into an object holds one value per key, repeated attributes are preserved +explicitly; without that, a device aggregating four carriers reports only the last, and +the last is often the narrowest. The panel shows every carrier and the aggregate +bandwidth. + +### Uplink anchor + +Uplink rides the **primary carrier alone**. Left on automatic, a modem anchors to +whichever band it hears loudest — frequently a narrow one — and aggregates wider bands for +downlink only, which caps upload no matter how much downlink is stacked on top. + +When the primary is narrower than a carrier being aggregated, the panel says so. This is +the actual reason operators lock bands, and it is invisible in RouterOS. + +## Observed bands + +Bands seen serving the device accumulate from polling, and are shown beside the bands +your configuration permits. + +This exists because scanning is not a dependable answer to "is this band available here?" +— it costs service to run and returns nothing at a site with a single base station. A band +the modem has actually used demonstrably works there; one never observed is a candidate +for stranding a device on a mast. + +## Tower and band movement + +Handovers, re-registrations and band changes are reconstructed by comparing consecutive +polls. A session uptime that runs *backwards* means the modem re-registered; a changed +cell id means it handed over. + +This is deliberately not log-based. The modem this was built for may log nothing at all, +so polling is the vantage point that always exists. + +## Data-cap reset SMS + +Several European carriers sell "unlimited" data that is throttled past a daily allowance, +and lift the throttle when the subscriber texts a short code. + +Configure per interface: **number**, **allowance**, **reset time and timezone**, +**cooldown**, and how far **early** to send. + +### Why it sends early + +Interface byte counters do not survive a reboot, so there is no running total to read — +usage is accumulated poll to poll. That makes the figure a permanent **undercount**: +traffic during a reboot, a missed poll, or before the device was adopted is invisible. + +Undercounting fires *late*, and late means already throttled. So the trigger fires below +the configured allowance, 5% by default. Firing early costs one message; firing late costs +the evening. + +The saving grace is that the carrier resets daily too, so the error cannot accumulate past +a single period. + +### Calibrating the margin + +If you are throttled while the bar still reads under the trigger point, the gap **is** the +correction. Throttled at 8.7 GB against a 10 GB allowance means roughly a 13% undercount; +set "send early by" to 15–20%. + +Device reboots and manager downtime both widen the gap and are visible to you but not to +us. + +### Sending manually + +The **Send now** button is independent of everything else — not gated on the rule being +enabled, on a threshold, or on the usage estimate. It is the trustworthy half: one known +action for a reason you already have. The automatic trigger is a convenience layer on top. + +The automatic side carries a cooldown, because usage sits *at* the cap once reached rather +than above it. Without one, a link parked on its limit would text the carrier on every +poll. + +### Requirements and a trap + +The modem needs SMS enabled — `/tool/sms/print` should report `receive-enabled: yes` and +the port your interface uses. + +If that output also shows `sms-storage: sim` with `remove-sent-sms-after-send: no`, sent +messages accumulate on the SIM. A SIM holds twenty or thirty, and **a full store makes +sending fail silently** — the worst possible failure here. Setting +`remove-sent-sms-after-send=yes` removes the risk. + +Every send is recorded with the counter reading at the time, so a misfire is explainable +rather than mysterious. diff --git a/docs/change-guard.md b/docs/change-guard.md index e07c3e6..e60149a 100644 --- a/docs/change-guard.md +++ b/docs/change-guard.md @@ -8,7 +8,7 @@ very path you are managing the device over. MikroTik's own documented advice for VLAN filtering is to have a serial console ready. Three layers address that. This page covers how they behave and how to configure them; -the [README](../README.md#what-makes-it-different) has the short version. +the [README](https://github.com/2GT-Media-Group-LLC/mikrotik-manager/blob/main/README.md#what-makes-it-different) has the short version. ## Change Guard — device-side auto-revert diff --git a/docs/commands.md b/docs/commands.md new file mode 100644 index 0000000..84a4a48 --- /dev/null +++ b/docs/commands.md @@ -0,0 +1,68 @@ +# Bulk commands + +[← Documentation index](README.md) + +Run one RouterOS console command across many devices, in waves, stopping at the first +failure. + +Commands run over **SSH**, because console syntax — `:put`, `:foreach`, `/interface print` +— is not reachable through the binary API, which has its own command tree. + +## Why waves exist + +A shell loop already runs a command on many hosts, costs nothing, and prints errors as +they happen. What it cannot do is **stop**. + +Devices run together within a wave; waves run strictly in sequence. With halt-on-failure +enabled, a mistake reaches one wave rather than the whole fleet. That is the entire reason +to do this here rather than in a terminal — without it, this would be a worse shell loop. + +Start with a wave of one. Widen it once the command has proved itself. + +## Detecting failure + +RouterOS reports most errors in the **output text** rather than through an exit code, so a +rejected command and a successful one are indistinguishable unless the output is read. + +Output is scanned for unambiguous markers — `syntax error`, `bad command name`, `no such +item`, `failure:` and similar. The matching is deliberately conservative: mistaking +ordinary output for failure would halt a run that was working, which is worse than the +error it guards against. + +## Change Guard + +Each device is wrapped by [Change Guard](change-guard.md) by default, so one that stops +answering after the command restores itself. + +It is **one click to turn off**. You may know exactly why you are running something that +will drop a device, and a tool that refuses to cut is not a sharp tool. + +The one combination that asks for explicit acknowledgement is a command that can sever +management **with the guard disabled** — the only case where nothing catches a mistake. + +## Preview + +Before running, the preview shows: + +- which devices are selected and which wave each falls into +- whether the command matches patterns that can cut management access, and why +- devices with **no SSH credential**, which would otherwise produce a wave of identical + authentication failures + +It warns; it never blocks. + +## Output + +Captured per device and kept with the run. Some commands exist purely for their output — +`:put [:resolve google.com]` across a fleet is a legitimate use — and "it failed" without +the device's own words is not actionable. + +## Stopping a run + +Cancelling prevents the **next wave**. Devices already executing finish, because that is +the only point at which stopping is safe. + +## Requirements + +Every targeted device needs an SSH credential: either a password, or a verified +[SSH key](ssh-keys.md). On a fleet of any size, keys are the practical answer. diff --git a/docs/scaling.md b/docs/scaling.md new file mode 100644 index 0000000..8646f4f --- /dev/null +++ b/docs/scaling.md @@ -0,0 +1,100 @@ +# Polling and scaling + +[← Documentation index](README.md) + +How the poller works, how to tell whether it is keeping up, and what to change when it is +not. + +## The model + +Every device is polled on three cadences, dispatched through Redis-backed queues: + +| Poll | Default | Collects | +|---|---|---| +| Fast | 30 s | Interface traffic, resources, clients, status | +| Logs | 60 s | Device log events | +| Slow | 5 min | Interfaces, VLANs, system info, wireless, STP | + +Enqueue rate is not connect rate. Workers process jobs with limited concurrency, so what +matters is whether they drain faster than the schedule fills. + +## Is it keeping up? + +**Settings → Polling**, or `GET /api/system/poller`. + +The number to read is **headroom** — service capacity divided by demand: + +``` +arrival devices / poll interval +service worker concurrency / mean poll duration +headroom service / arrival +``` + +Below **1.0** the backlog grows every cycle and devices go stale. Comfortably above it, +the fleet is being polled on schedule. + +**Headroom alone is not enough**, which is why backlog and drain estimate sit beside it. It +describes flow — whether the workers can keep up *from here* — and is blind to work already +queued. A healthy headroom next to a large backlog means capacity is fine but you are still +working through a queue. + +A single `status` summarises all of it: `ok`, `draining`, or `saturated`. + +## Which devices are being missed + +The same page lists devices not polled recently, and distinguishes two states that look +identical from the outside: + +- **not attempted recently** — the poller never reached it +- **responded with…** — it was reached and did not answer + +`device_poll_stats` records attempt and success separately per device and per kind, with +duration and last error. Their difference is that distinction. + +## Tuning + +All of these are environment variables and need a container restart. + +| Variable | Default | Effect | +|---|---|---| +| `POLLER_CONCURRENCY` | 12 | Workers per queue. The first lever when headroom is low | +| `POLLER_INTERVAL_MS` | 30000 | Scheduler tick. Lengthening reduces demand | +| `POLLER_JOB_TIMEOUT_MS` | 45000 | Ceiling on one poll, so a dead device cannot hold a worker | +| `POLLER_JOB_RETENTION_SEC` | 3600 | How long finished jobs are kept | +| `POLLER_JOB_RETENTION_COUNT` | 5000 | How many finished jobs are kept | +| `REDIS_MAXMEMORY` | 512mb | Ceiling on Redis | + +Raise concurrency first. Lengthen the interval second — it reduces freshness, which is the +thing you are paying for. + +## Behaviour worth knowing + +**A device already queued is not enqueued again.** Deduplication is by job id, so a +backlog cannot compound: one outstanding poll per device and kind, however long it waits. + +**Periodic polls are not retried in-job.** The schedule *is* the retry — a device that did +not answer is unlikely to answer a second later, and a retry occupies a worker the rest of +the fleet needs. + +**Stale queued polls are discarded**, at startup and every five minutes. A periodic poll +that has waited longer than its own cadence is worthless: running it produces a reading the +next cycle would take anyway. Discarding is safe precisely because the work is periodic — +the schedule re-enqueues within one interval. + +**Job history is bounded.** Redis runs with `noeviction` rather than an LRU policy on +purpose: BullMQ's job locks and stalled-check keys carry TTLs, and evicting one causes +duplicate processing or a stalled-job storm. A memory ceiling protects the host; retention +keeps usage far below it. + +## Clearing a backlog + +**Settings → Polling → Clear queued**, or `POST /api/system/poller/drain`. + +Nothing is lost. The scheduler re-enqueues whatever is still due on its next tick, so the +cost is at most one interval of freshness. + +## Scheduled work and timezones + +Scheduled tasks are evaluated in the timezone set under **Settings → General**, not the +container's. The container runs UTC, so without this a backup set for 02:00 would fire at +02:00 UTC wherever you are — and on the wrong day far enough east. diff --git a/docs/ssh-keys.md b/docs/ssh-keys.md new file mode 100644 index 0000000..9dc6ef7 --- /dev/null +++ b/docs/ssh-keys.md @@ -0,0 +1,74 @@ +# SSH keys + +[← Documentation index](README.md) + +Each device can be given its own SSH keypair, used for backups, config export and bulk +commands instead of a stored password. + +## Read this before installing one + +**Installing a key disables password SSH for that user.** RouterOS binds the key to the +account and then refuses password authentication entirely — this is RouterOS behaviour, +not a policy choice here, and it is confirmed on hardware: removing the key restores +password login immediately. + +Consequences worth stating plainly: + +- Colleagues, scripts and other tools that SSH in with a password **will stop working**. +- The **binary API and WinBox are unaffected** and remain a way in. +- **If this server loses its database or `ENCRYPTION_KEY`, the private key goes with it.** + SSH would then need recovering through the API or WinBox, because the password will + already have been refused. +- Revoking the key restores password SSH. + +On a shared fleet this is a change to announce, not to discover. + +## How deployment works + +1. A unique **ed25519** keypair is generated on the manager — one per device, never a + fleet key. A shared key would make any single compromise fleet-wide and would make + rotating one device impossible without touching all of them. +2. The public half is uploaded over the existing password session and imported. +3. A **completely fresh connection** authenticates with the new key. +4. Only then is anything stored. + +Step 4 is the important one. Nothing trusts a key until it has been proved, and a +deployment that fails verification **removes the key it just installed** rather than +leaving a device demanding a credential nobody holds. + +## Rotation + +Rotation authenticates with the *existing* key, because the password is no longer an +option once a key is installed. The replacement is deployed and proved before the old key +is removed — removing first would leave a window in which neither works. + +Each key carries a unique comment, so retiring one cannot remove another. + +## Recovery + +**The binary API is the recovery path**, not the password. It is unaffected by SSH key +state and can remove a key at any time. + +Revoking clears *every* key this manager has installed for a device, not only the one on +record. Orphans left by a half-failed rotation would otherwise keep password SSH disabled +with no way back through the product. + +If deployment fails with an authentication error and orphaned keys are present, the error +says so. + +## Fleet-wide deployment + +`POST /api/devices/ssh-keys/deploy-all` keys every device that has a password but no +verified key. It runs sequentially rather than in parallel — each deployment opens two SSH +sessions and ends by proving the key. + +It **refuses without explicit confirmation**, because it disables password SSH on every +device it touches. + +## What uses a key + +Backups and config export prefer a verified key where one exists and fall back to a +password otherwise. [Bulk commands](commands.md) use the same resolution. + +Only a *verified* key is used. One that was generated or pushed but never authenticated is +not evidence the device will accept it. diff --git a/frontend/package.json b/frontend/package.json index 77cc880..072fac1 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "mikrotik-manager-frontend", - "version": "0.23.27-beta", + "version": "0.23.28-beta", "private": true, "type": "module", "scripts": { diff --git a/frontend/src/version.ts b/frontend/src/version.ts index 7a847c5..691e27f 100644 --- a/frontend/src/version.ts +++ b/frontend/src/version.ts @@ -1 +1 @@ -export const APP_VERSION = 'v0.23.27 Beta'; +export const APP_VERSION = 'v0.23.28 Beta'; diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..28e7a9a --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,74 @@ +# Documentation site. Source of truth is docs/*.md, which stays readable when +# browsing the repository on GitHub — the site renders those files rather than +# replacing them. Keeping docs beside the code is the only thing that reliably +# prevents drift: the same pull request changes behaviour and description. +site_name: MikroTik Manager +site_description: Self-hosted management for MikroTik fleets +repo_url: https://github.com/2GT-Media-Group-LLC/mikrotik-manager +repo_name: 2GT-Media-Group-LLC/mikrotik-manager +edit_uri: edit/main/docs/ + +docs_dir: docs + +theme: + name: material + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + primary: indigo + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: indigo + toggle: + icon: material/brightness-4 + name: Switch to light mode + features: + - navigation.instant + - navigation.top + - navigation.sections + - content.code.copy + - search.highlight + +plugins: + - search + # Versioned docs. The platform ships betas frequently and images are published + # to GHCR, so readers are routinely on something other than the newest build — + # a single "latest" would describe a release they are not running. + - mike: + canonical_version: latest + +extra: + version: + provider: mike + default: latest + +markdown_extensions: + - admonition + - attr_list + - tables + - toc: + permalink: true + - pymdownx.highlight + - pymdownx.superfences + +nav: + - Home: README.md + - Getting started: + - Configuration: configuration.md + - Operating: + - Change Guard and Config Health: change-guard.md + - Polling and scaling: scaling.md + - Alerting: alerting.md + - SSH keys: ssh-keys.md + - Bulk commands: commands.md + - Devices: + - Cellular (LTE): cellular.md + - CAPsMAN: capsman.md + - Integration: + - API and automation: api.md + - Single sign-on (OIDC): sso-oidc.md + - Contributing: + - Architecture: architecture.md diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 0000000..5c82536 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,4 @@ +# Pinned so a docs build cannot break on an upstream release the way an +# unpinned toolchain would. +mkdocs-material==9.5.44 +mike==2.1.3