diff --git a/.dockerignore b/.dockerignore index c1dc287..28ed355 100644 --- a/.dockerignore +++ b/.dockerignore @@ -26,8 +26,9 @@ config.yaml # Runtime data (mounted at runtime via volume) playbook_cache/ -# Plugins (mounted at runtime via volume) -plugins/ +# NOTE: first-party plugins under plugins/ now ship IN the image +# (Dockerfile `COPY plugins/`), so plugins/ must NOT be ignored here. +# Third-party plugins are mounted at runtime into /data/plugins instead. # Deployment files not needed in image docker-compose.yml diff --git a/.env.example b/.env.example index 8d36b78..f5201bf 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,11 @@ # Copy to .env for Docker / local development secrets # These override values in config.yaml -STEWARD_SECRET_KEY=change-me -STEWARD_DATABASE__URL=postgresql+asyncpg://steward:password@localhost/steward +STEWARD_DATABASE_URL=postgresql+asyncpg://steward:password@localhost/steward STEWARD_SMTP__PASSWORD= + +# --- compose.deploy.yml (remote instance) only --- +# Password for the bundled Postgres; the steward service builds its +# STEWARD_DATABASE_URL from it. Leave STEWARD_SECRET_KEY above unset to let the +# app generate + persist one on the /data volume. +POSTGRES_PASSWORD=change-me diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 0000000..800746e --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,129 @@ +name: CI + +# CI lanes per FabledRulebook forgejo.md "CI philosophy": +# - lint: ruff only, no dep install — fast-fail for the common lint bounce. +# - unit: pytest -m "not integration"; no service containers, no DB. +# - integration: postgres service container; boots the real app + runs all +# (core + bundled-plugin) migrations. +# Single repo: first-party plugins are bundled in-tree under plugins/, so the +# unit lane imports them directly — no cross-repo checkout. + +on: + push: + branches: [dev, main] + # pull_request intentionally absent — push on [dev, main] already fires CI for + # every dev commit and dev→main PRs. Single-operator repo, no fork PRs. + +jobs: + # Fast-fail lint lane. ruff is pre-installed in the ci-python image, so this + # runs with NO dependency install and surfaces lint bounces in seconds. + lint: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + steps: + - uses: actions/checkout@v4 + - name: Ruff lint + run: ruff check steward/ plugins/ tests/ + + unit: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + steps: + - uses: actions/checkout@v4 + - name: Install Steward + test deps + # uv: 5-10x faster wheel resolve than pip on cold caches. Falls back to + # pip on uv-missing runners. Steward declares its deps in pyproject; the + # [dev] extra adds pytest + pytest-asyncio. + run: | + if command -v uv >/dev/null 2>&1; then + uv pip install --system -e '.[dev]' + else + pip install -e '.[dev]' + fi + - name: Pytest (unit only — integration runs in its own lane) + run: pytest tests/ -v -m "not integration" + + integration: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + env: + STEWARD_SECRET_KEY: ci_integration_placeholder + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: steward + POSTGRES_PASSWORD: steward + POSTGRES_DB: steward + options: >- + --health-cmd "pg_isready -U steward" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + - name: Boot + migrate against Postgres + # act_runner (swarm-runner) puts service containers on the default bridge + # with no embedded DNS, so the hostname `postgres` won't resolve — we + # discover the service container's bridge IP via the docker socket. The + # job name `integration` (no underscore) is the docker-ps name filter. + run: | + set -euxo pipefail + echo "=== container landscape (diagnostic for filter scoping) ===" + docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}' + echo "=== end landscape ===" + PG=$(docker ps --filter "name=integration" --filter "ancestor=postgres:16-alpine" -q | head -n1) + test -n "$PG" + PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") + test -n "$PG_IP" + export STEWARD_DATABASE_URL="postgresql+asyncpg://steward:steward@$PG_IP:5432/steward" + for i in $(seq 1 60); do + (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break + sleep 2 + done + if command -v uv >/dev/null 2>&1; then + uv pip install --system -e '.[dev,ansible]' + else + pip install -e '.[dev,ansible]' + fi + pytest tests/integration -v -m integration --durations=10 + + # Image-publish lane — FabledRulebook rule 46: every push to dev/main publishes + # an immutable commit-SHA image (the rollback unit); dev moves :dev, main moves + # :latest. Gated on the three test lanes via `needs:` so a red commit never + # produces a :dev image. Mechanics mirror CI-runner's build-ci-python.yml. + publish: + needs: [lint, unit, integration] + runs-on: python-ci + container: + # ci-builder carries docker + buildx + git. The docker socket is + # auto-mounted by act_runner (valid_volumes) — do NOT add a + # container.options "-v /var/run/docker.sock:..." mount; that duplicate + # mount fails the job at "Set up job". + image: git.fabledsword.com/bvandeusen/ci-builder:latest + steps: + - uses: actions/checkout@v4 + - name: Registry login + # The injected GITHUB_TOKEN lacks write:package scope, so docker push + # 401s with it. REGISTRY_TOKEN is a repo Actions secret holding a + # Forgejo token scoped read:package + write:package. + run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.fabledsword.com -u bvandeusen --password-stdin + - name: Build + push (commit-SHA always; :dev on dev, :latest on main) + run: | + set -euxo pipefail + IMAGE=git.fabledsword.com/bvandeusen/steward + docker build -t "$IMAGE:${{ github.sha }}" . + docker push "$IMAGE:${{ github.sha }}" + if [ "${{ github.ref }}" = "refs/heads/dev" ]; then + docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:dev" + docker push "$IMAGE:dev" + elif [ "${{ github.ref }}" = "refs/heads/main" ]; then + docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:latest" + docker push "$IMAGE:latest" + fi + - name: Prune dangling layers + if: always() + run: docker image prune -f diff --git a/.gitignore b/.gitignore index fb2b9af..2678717 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ # Planning files docs/superpowers/ -# Plugin directory — managed as a separate repo (bvandeusen/steward-plugins) -# PLUGIN_DIR in config.yaml points here at runtime -/plugins/ +# First-party plugins are tracked in-tree under plugins/ and ship in the image. +# Third-party plugins are mounted at runtime into the external dir +# (STEWARD_PLUGIN_DIR, default /data/plugins) and are not tracked here. +/plugins/**/*.zip # Python __pycache__/ @@ -50,3 +51,9 @@ Thumbs.db # Playbook cache (runtime data) playbook_cache/ + +# Superpowers brainstorm scratch (visual companion mockups, state) +.superpowers/ + +# Local dev Ansible playbooks (operator content, mounted into the container) +/ansible-dev/ diff --git a/Dockerfile b/Dockerfile index 938501c..fc8a570 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,11 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \ iputils-ping \ # gosu — minimal privilege-drop tool used by entrypoint.sh gosu \ + # openssh-client — ansible-playbook reaches remote hosts over ssh + openssh-client \ + # sshpass — Ansible shells out to it for first-contact password SSH + # (provisioning a fresh host before the managed key is installed) + sshpass \ && rm -rf /var/lib/apt/lists/* # Upgrade pip to suppress the version notice @@ -14,9 +19,16 @@ WORKDIR /app COPY pyproject.toml . COPY steward/ steward/ -RUN pip install --no-cache-dir . +# Bundle the extras whose first-party plugins ship in the image: [ansible] for +# the playbook runner, [snmp] (pysnmp) for the SNMP poller. Without them those +# bundled plugins load but silently no-op for want of their runtime dependency. +RUN pip install --no-cache-dir '.[ansible,snmp]' COPY alembic.ini . +# First-party plugins ship inside the image (bundled root at /app/plugins). +# Third-party plugins are mounted at runtime into the external root +# (STEWARD_PLUGIN_DIR, default /data/plugins). +COPY plugins/ plugins/ COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh diff --git a/README.md b/README.md index b7a6239..d7e2999 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,42 @@ docker compose up -d Open `http://localhost:5000`. On first run you'll be prompted to create the admin account. +This compose file builds the image locally and bind-mounts the source for live editing. To run a persistent instance off the published CI image instead, see **Remote dev instance** below. + +--- + +## Remote dev instance + +For a long-lived instance on a remote server that tracks the `dev` CI line, use `compose.deploy.yml`. It pulls the published image (`git.fabledsword.com/bvandeusen/steward:dev`) rather than building, runs no source bind-mounts, and persists data in named volumes. + +**One-time prerequisite — registry push token (on the Git host).** CI publishes the image, so the FabledSteward repo needs a push credential. The injected `GITHUB_TOKEN` lacks `write:package`, so create a Forgejo token scoped **`read:package` + `write:package`** and add it as the repo Actions secret **`REGISTRY_TOKEN`**. Until this exists, the `publish` CI job 401s and no `:dev` image is produced. + +**Standup (on the server):** + +```bash +# 1. Authenticate to the registry (read:package token is enough to pull) +docker login git.fabledsword.com -u + +# 2. Configure secrets +cp .env.example .env +# Set POSTGRES_PASSWORD. Leave STEWARD_SECRET_KEY unset to let the app +# generate + persist one on the /data volume. + +# 3. Pull + boot +docker compose -f compose.deploy.yml up -d +``` + +Open `http://:5000` and create the admin account on first run. + +**Update to the latest dev build:** + +```bash +docker compose -f compose.deploy.yml pull +docker compose -f compose.deploy.yml up -d +``` + +Every push to `dev` that goes CI-green publishes a fresh `:dev` (and an immutable `:` for rollback). To pin a specific commit, change the `image:` tag in `compose.deploy.yml` to `:`. + --- ## Quick Start — Bare Metal diff --git a/ci-requirements.md b/ci-requirements.md new file mode 100644 index 0000000..86c04bd --- /dev/null +++ b/ci-requirements.md @@ -0,0 +1,58 @@ +# CI Requirements — Steward + +> Spec: https://git.fabledsword.com/bvandeusen/CI-runner/src/branch/main/docs/process.md + +## Runtime image + +git.fabledsword.com/bvandeusen/ci-python:3.14 + +## Image deps used + +- python 3.14 +- ruff (analyzer for `steward/`, `plugins/`, `tests/`) +- docker CLI (integration lane: bridge-IP discovery of the Postgres service container) + +## Per-job tool installs + +- `uv pip install --system -e '.[dev]'` (pip fallback) — in the `unit` job. +- `uv pip install --system -e '.[dev,ansible]'` (pip fallback) — in the + `integration` job, which boots the real app and exercises the Ansible runner, + so it needs the `ansible` extra (`ansible>=10,<13`) at import time. + + Steward declares its deps in `pyproject.toml`; the `[dev]` extra adds + `pytest` + `pytest-asyncio`. No `requirements.txt` is tracked. + +## Lanes + +- **lint** — `ruff check steward/ plugins/ tests/`, no dep install. +- **unit** — `pytest -m "not integration"`. The whole current suite runs here: + it builds the app with `testing=True`, which mocks `db_sessionmaker`, so no DB + is touched. First-party plugins are bundled in-tree under `plugins/`, so plugin + unit tests import them directly — no cross-repo checkout. +- **integration** — `pytest tests/integration -m integration` against a + `postgres:16-alpine` service. The single test boots the real app + (`create_app(testing=False)`), which runs core + every bundled plugin's + migrations, then round-trips the DB. This is the guard that the folded-in + plugin migration graph resolves. +- **publish** — builds the runtime `Dockerfile` and pushes to the package + registry. Runs in `ci-builder:latest` (docker + buildx, socket auto-mounted), + gated on `needs: [lint, unit, integration]` so a red commit never ships an + image. Publishes `git.fabledsword.com/bvandeusen/steward:` always, plus + `:dev` on `dev` and `:latest` on `main` (FabledRulebook rule 46). Needs the + `REGISTRY_TOKEN` repo Actions secret (`read:package` + `write:package`) — + the injected `GITHUB_TOKEN` can't push packages. + +## Notes + +- Steward has **no frontend and no Redis/Celery** (no external workers), so there + is no frontend-build lane and the only service container is Postgres. +- Integration uses Forgejo Actions `services:` + a socket-discovered bridge IP + because `act_runner` (swarm-runner v0.6+) puts services on the default bridge + with no embedded DNS. FabledCurator's `ci.yml` is the canonical example of the + pattern; Steward's is the same shape minus Redis and minus sharding. +- The job name `integration` has no underscore — `act_runner` strips underscores + from job names when building service-container labels, so the `docker ps` + name filter uses the bare job name. +- Migrations run via the app itself (`create_app` → `run_core_migrations`, async + through `asyncpg`), so the integration lane needs no separate `alembic upgrade` + step and no psycopg/sync driver. diff --git a/compose.deploy.yml b/compose.deploy.yml new file mode 100644 index 0000000..c5bd8fb --- /dev/null +++ b/compose.deploy.yml @@ -0,0 +1,69 @@ +# Remote deployment — pulls the published :dev image instead of building locally. +# +# This is NOT the local-dev compose. `docker-compose.yml` bind-mounts ./steward +# + ./plugins and runs --debug for live editing; THIS file runs the image as the +# source of truth (no bind-mounts, no reloader) for a persistent remote instance +# that tracks the dev CI line. +# +# Standup + update runbook: see README → "Remote dev instance". + +services: + steward: + image: git.fabledsword.com/bvandeusen/steward:dev + container_name: steward + # Re-pull :dev on every `up` so restarting the stack picks up the latest + # green dev build without a manual `docker pull`. + pull_policy: always + ports: + - "5000:5000" + networks: + - backend + volumes: + # /data holds the auto-generated secret key, third-party plugins, and the + # Ansible playbook cache — all of which must survive image updates. No + # source bind-mounts: core + first-party plugins live inside the image. + # + # The container runs as the unprivileged 'app' user (uid 1000), so /data + # MUST be writable by uid 1000. A named docker volume (below) gets that + # automatically. If you bind-mount a host/NFS path instead, chown it to + # 1000:1000 (and mind NFS root_squash) — otherwise the app can't persist + # its secret key, mints a fresh one each boot, and every encrypted secret + # (managed SSH key, SMTP/OIDC/LDAP creds) becomes unrecoverable. The app + # now refuses to start in that state rather than silently degrade. + - app_data:/data + environment: + - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:${POSTGRES_PASSWORD:-steward}@db/steward + # STEWARD_SECRET_KEY is intentionally absent — the app generates a random + # key on first boot and persists it to /data/secret.key, which the app_data + # volume keeps across image updates. On multi-node Swarm with a shared bind + # mount, pin STEWARD_SECRET_KEY (env or a docker secret) instead, so the key + # never depends on filesystem ownership being correct on every node. + depends_on: + db: + condition: service_healthy + restart: unless-stopped + + db: + image: postgres:16-alpine + networks: + - backend + environment: + POSTGRES_USER: steward + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-steward} + POSTGRES_DB: steward + volumes: + - pgdata:/var/lib/postgresql/data + # No host port published — db is reachable only on the backend network. + healthcheck: + test: ["CMD-SHELL", "pg_isready -U steward"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + +networks: + backend: + +volumes: + pgdata: + app_data: diff --git a/docker-compose.yml b/docker-compose.yml index faedd9f..9f7cf81 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,15 +3,29 @@ services: build: . container_name: steward image: steward:latest + # Dev: --debug turns on Quart's auto-reloader so edits to the mounted source + # below take effect without a rebuild. + command: ["steward", "--host", "0.0.0.0", "--port", "5000", "--debug"] ports: - "5000:5000" volumes: - app_data:/data - - ./plugins:/app/plugins:ro - - /mnt/Data/traefik/log:/var/log/traefik:ro - - /var/run/docker.sock:/var/run/docker.sock:ro + # Live-edit core + first-party plugins without rebuilding. PYTHONPATH=/app + # (below) makes this mounted source win over the image-installed package. + - ./steward:/app/steward + - ./plugins:/app/plugins + # Dev playbooks: edit on the host under ./ansible-dev/, register a *local* + # Ansible source pointing at /srv/ansible in Settings → Ansible. + - ./ansible-dev:/srv/ansible + # Optional host mounts — enable the matching plugin in Settings first, then + # uncomment (the traefik path must exist on this host): + # - /mnt/Data/traefik/log:/var/log/traefik:ro # traefik plugin + # - /var/run/docker.sock:/var/run/docker.sock:ro # docker plugin environment: - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@db/steward + # Dev-only fixed key so sessions survive restarts. Replace in production. + - STEWARD_SECRET_KEY=dev-secret-key-change-me + - PYTHONPATH=/app depends_on: db: condition: service_healthy @@ -23,6 +37,9 @@ services: POSTGRES_USER: steward POSTGRES_PASSWORD: steward POSTGRES_DB: steward + ports: + # Exposed for dev convenience (psql from the host). Drop for prod. + - "5432:5432" volumes: - pgdata:/var/lib/postgresql/data healthcheck: diff --git a/docs/reference/playbook-authoring.md b/docs/reference/playbook-authoring.md new file mode 100644 index 0000000..8b6c19e --- /dev/null +++ b/docs/reference/playbook-authoring.md @@ -0,0 +1,191 @@ +# Authoring Steward-friendly Ansible playbooks + +This is the contract between a playbook and Steward's run UI. Follow it and a +playbook drops into Steward with a description, fill-in variable fields, correct +targeting, and credentials supplied automatically — no per-playbook wiring. + +It applies to **any** playbook in a configured source (bundled, the writable +local source, or a git source), including third-party ones. + +--- + +## 1. Describe what it does — `# description:` + +Steward shows a one-line description when a playbook is selected in the run form. + +```yaml +--- +# description: Reclaim disk on Docker/Swarm nodes by pruning unused images and build cache. +- name: Docker system prune + hosts: all + ... +``` + +- Format: a comment line `# description: ` anywhere in the file. **First + match wins.** Case-insensitive on the `description:` key. +- Why a comment and not a key: Ansible rejects unknown *play* keys (you can't add + `description:` to a play), so a comment is the portable place. It survives + `ansible-playbook` untouched. +- Fallback: if there's no `# description:` comment, Steward uses the **first + play's `name:`**. So always give your play a meaningful `name:` even without + the comment. +- Keep it to one readable line. Longer "how to use" notes can go in additional + normal comments — Steward only reads the `description:` line. + +## 2. Declare tunables in `vars:` — they become fill-in fields + +Every **scalar** entry in a play's `vars:` block becomes an editable field in the +run form, with the default shown as the input's placeholder. + +```yaml + vars: + prune_all_images: false # → checkbox-ish text field, placeholder "default: false" + keep_last_days: 7 # → field, placeholder "default: 7" + registry_url: "" # → field, placeholder "no default" +``` + +- **Blank field = use the default.** Steward only sends fields the operator + actually fills, so an untouched field falls through to the playbook/inventory + default rather than overriding it. +- Only scalars (string/int/float/bool) surface as fields. Lists/dicts are + skipped — set those in the playbook or via inventory group/host vars. +- Values are delivered as **extra-vars** (`-e`), which are the **highest** + precedence in Ansible — they override the `vars:` defaults. (This is why the + default can be empty and still be safely overridden at run time.) + +### `vars_prompt:` also works + +Steward reads `vars_prompt` too. Use it when you want an explicit prompt or a +required value: + +```yaml + vars_prompt: + - name: release_tag + prompt: "Which release to deploy?" # shown as the field label + # no default → Steward marks the field REQUIRED + - name: admin_password + prompt: "Admin password" + private: true # → masked field, never stored +``` + +## 3. Secrets — name them so they're masked and not persisted + +A field is treated as **secret** (rendered masked, and its value is **never +written to the DB / run history**) when either: + +- the variable name contains `password`, `passwd`, `secret`, `token`, + `api_key` / `apikey`, `private_key`, or `credential` (case-insensitive), **or** +- it's a `vars_prompt` entry with `private: true` (Ansible's default for + vars_prompt is private). + +So name sensitive variables accordingly (`db_password`, `api_token`, +`vault_secret`) and they're handled safely with no extra config. Non-secret +run-time vars are persisted (so scheduled runs can reuse them); secret ones are +passed to the run only. + +## 4. Target with `hosts: all` + +Steward builds the inventory itself from the **target / group** the operator +picks in the run form (or the single host on a host page). Your play should: + +```yaml + hosts: all # run against whatever Steward scoped to +``` + +- Don't hardcode hostnames or rely on a checked-in inventory for Steward runs + (Steward generates a fresh inventory per run). +- Per-host connection vars (`ansible_host`, plus anything you set on the target + in **Ansible → Inventory**) arrive as inventory host vars. +- The run form's **Limit** / **Tags** map to `--limit` / `--tags`. + +## 5. Privileges & connection — don't put credentials in the playbook + +Steward supplies SSH and become for you: + +- Steady-state runs connect as the managed **`steward`** account using Steward's + managed key; that account has **passwordless sudo**. So just use + `become: true` where you need root. +- First-contact provisioning uses a one-time bootstrap user/password the + operator enters (never stored). +- Never embed SSH keys, passwords, or `ansible_user`/`ansible_ssh_pass` in the + playbook. Connection identity is global (Settings → Ansible) or per-target. + +## 6. Be idempotent + +Steward re-runs playbooks (updates, schedules, retries). Use modules that +converge (state-based) rather than ad-hoc `command:`/`shell:` where possible, so +re-runs are safe. + +--- + +## What Steward reads from a playbook (summary) + +| Source in the playbook | What Steward does with it | +|---|---| +| `# description: ` comment | Description shown on selection (first match) | +| first play `name:` | Description fallback | +| `# steward:category: ` | Grouping badge in the browse list + run form | +| `# steward:confirm: true` | Requires a confirmation tick before the run launches | +| `vars:` scalar entries | Run-time fill-in fields (placeholder = default) | +| `vars_prompt:` entries | Run-time fields (required if no default) | +| secret-looking var name / `private: true` | Field masked + value not persisted | +| `hosts:` | Expected to be `all`; Steward provides the inventory | + +Everything else (SSH user/key, become password, the inventory, `steward_token` +etc. for the agent playbooks) is injected by Steward at run time. + +## Metadata comments + +Steward reads metadata from magic comments (Ansible rejects unknown play keys, +so comments are the portable place). Two forms: + +- `# description: ` — the description (see §1). +- `# steward:: ` — the namespaced metadata block. First match per + key wins; keys are case-insensitive. + +**Implemented `# steward:` keys:** + +| Key | Example | Effect | +|---|---|---| +| `category` | `# steward:category: maintenance` | Free-text grouping label. Shown as a badge in the browse list and on the run form. Purely organizational. | +| `confirm` | `# steward:confirm: true` | Marks the playbook as significant/destructive. The run form then requires an explicit confirmation tick before it can be launched. Use for data-loss-capable plays (prune with volumes, resets, etc.). Truthy values: `true`/`yes`/`1`/`on`. | +| `description` | `# steward:description: ...` | Alias for `# description:` (the unprefixed form takes precedence if both exist). | + +```yaml +--- +# description: Reclaim disk on Docker/Swarm nodes by pruning unused data. +# steward:category: maintenance +# steward:confirm: true +- name: Docker system prune + hosts: all + ... +``` + +Other `# steward::` keys are simply ignored today — the namespace is +reserved, so it's safe to add ones Steward doesn't yet understand without +breaking anything, but only `category`, `confirm`, and `description` do something. + +## Minimal annotated example + +```yaml +--- +# description: Restart a systemd service and confirm it came back up. +- name: Restart a service + hosts: all + become: true + gather_facts: false + vars: + service_name: "" # required-ish: operator fills it in the run form + tasks: + - name: Validate input + ansible.builtin.assert: + that: service_name | default('') | length > 0 + fail_msg: "Set service_name." + - name: Restart + ansible.builtin.systemd: + name: "{{ service_name }}" + state: restarted + - name: Confirm active + ansible.builtin.command: "systemctl is-active {{ service_name }}" + changed_when: false +``` diff --git a/docs/superpowers/plans/2026-06-05-ansible-inventory.md b/docs/superpowers/plans/2026-06-05-ansible-inventory.md new file mode 100644 index 0000000..e0aed90 --- /dev/null +++ b/docs/superpowers/plans/2026-06-05-ansible-inventory.md @@ -0,0 +1,2071 @@ +# Ansible Inventory Infrastructure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add AnsibleTarget + AnsibleGroup models with M2M membership, DB-generated `--list` inventory, per-source PAT auth via GIT_ASKPASS, and a full CRUD UI so Ansible runs are driven by Steward's own inventory records rather than git-repo inventory files. + +**Architecture:** Three new DB tables (`ansible_targets`, `ansible_groups`, `ansible_target_groups`) plus one new column (`ansible_runs.inventory_scope`) added via Alembic migrations. A pure `generate_inventory()` function in `inventory_gen.py` converts ORM objects to the Ansible `--list` JSON format. The existing `browse.html` run form gains a scope picker that replaces the repo-inventory select; `create_run()` fetches scope targets and passes the generated JSON as `inventory_content` to the existing executor. A new `inventory_routes.py` blueprint serves `/ansible/inventory/` CRUD pages for groups and targets. + +**Tech Stack:** Python 3.11+, SQLAlchemy (async/asyncpg), Alembic migrations, Quart, Jinja2 templates (no JS framework beyond existing HTMX), PyYAML (already in project via Ansible dependency). + +--- + +## File Map + +| Path | Action | Responsibility | +|------|--------|----------------| +| `steward/migrations/versions/0015_ansible_inventory_targets.py` | Create | Alembic migration: `ansible_targets` table | +| `steward/migrations/versions/0016_ansible_inventory_groups.py` | Create | Alembic migration: `ansible_groups` + `ansible_target_groups` tables | +| `steward/migrations/versions/0017_ansible_run_scope.py` | Create | Alembic migration: `inventory_scope` column + make `inventory_path` nullable | +| `steward/models/ansible_inventory.py` | Create | `AnsibleTarget`, `AnsibleGroup`, `ansible_target_groups` association table | +| `steward/models/__init__.py` | Modify | Import/export new models so Alembic discovers them | +| `steward/models/ansible.py` | Modify | Add `inventory_scope` field + make `inventory_path` optional on `AnsibleRun` | +| `steward/ansible/inventory_gen.py` | Create | Pure `generate_inventory(targets)` + async `fetch_scope_targets(db, scope)` | +| `steward/ansible/sources.py` | Modify | Add `http_token` to returned source dict + GIT_ASKPASS in `git_pull()` | +| `steward/ansible/routes.py` | Modify | Update `create_run()` scope handling; update `index()` context | +| `steward/ansible/inventory_routes.py` | Create | Blueprint: CRUD for `/ansible/inventory/groups` + `/ansible/inventory/targets` | +| `steward/app.py` | Modify | Register `inventory_bp` | +| `steward/settings/routes.py` | Modify | Include `http_token` in `ansible_save_source()` + `ansible_add_source()` | +| `steward/templates/ansible/browse.html` | Modify | Replace inventory select with scope picker ` + +``` + +Inside the `ansible-add-git-fields` div, after the `pull_interval_seconds` form-group, add: + +```html +
+ + +
+``` + +- [ ] **Step 5.8: Commit** + +```bash +git add steward/ansible/sources.py \ + steward/settings/routes.py \ + steward/templates/settings/_ansible_sources.html \ + tests/core/test_git_askpass.py +git commit -m "feat(ansible): GIT_ASKPASS per-source HTTP token auth + settings UI" +``` + +--- + +## Task 6: Wire Inventory Generation into create_run() + +**Files:** +- Modify: `steward/ansible/routes.py` +- Modify: `steward/templates/ansible/browse.html` +- Modify: `steward/templates/ansible/run_detail.html` + +- [ ] **Step 6.1: Update `index()` in `steward/ansible/routes.py`** + +The `index()` route needs to pass `inventory_scope` label context to the run history table (for display). No test needed — it's a display change. + +Change the `index()` route to also fetch target/group counts for displaying in the nav: + +```python +@ansible_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + from sqlalchemy import func + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleRun).order_by(AnsibleRun.started_at.desc()).limit(50) + ) + runs = result.scalars().all() + target_count = (await db.execute(select(func.count()).select_from(AnsibleTarget))).scalar() + return await render_template("ansible/index.html", runs=runs, target_count=target_count) +``` + +- [ ] **Step 6.2: Update `create_run()` to use inventory_scope** + +Replace the existing `create_run()` implementation. The key changes are: +- Accept `inventory_scope` from the form instead of `inventory_path` +- Generate `inventory_content` from DB for `steward:*` scopes +- Store `inventory_scope` on the `AnsibleRun` row + +Full updated function (replaces the existing one at line ~108): + +```python +@ansible_bp.post("/runs") +@require_role(UserRole.operator) +async def create_run(): + import json as _json + from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory + + form = await request.form + playbook_path = form.get("playbook_path", "").strip() + source_name = form.get("source_name", "").strip() + inventory_scope = form.get("inventory_scope", "steward:all").strip() + + if not playbook_path: + return "playbook_path is required", 400 + + all_sources = _get_sources() + source = next((s for s in all_sources if s["name"] == source_name), None) + if source is None: + return "Source not found", 404 + + # Resolve inventory content for DB-backed scopes. + inventory_content: str | None = None + inventory_path: str | None = None + if inventory_scope.startswith("steward:"): + async with current_app.db_sessionmaker() as db: + targets = await fetch_scope_targets(db, inventory_scope) + inventory_content = _json.dumps(generate_inventory(targets)) + elif inventory_scope.startswith("repo:"): + # repo:: + parts = inventory_scope.split(":", 2) + inventory_path = parts[2] if len(parts) == 3 else "" + else: + return "Invalid inventory_scope", 400 + + # Optional run parameters. + extra_vars: list[str] = [] + for line in (form.get("extra_vars", "") or "").splitlines(): + line = line.strip() + if not line: + continue + if "=" not in line: + return f"Invalid extra var (expected key=value): {line!r}", 400 + extra_vars.append(line) + limit = (form.get("limit", "") or "").strip() + tags = (form.get("tags", "") or "").strip() + check = "check" in form + + params: dict = {} + if extra_vars: + params["extra_vars"] = extra_vars + if limit: + params["limit"] = limit + if tags: + params["tags"] = tags + if check: + params["check"] = True + params_or_none = params or None + + run_id = str(uuid.uuid4()) + now = datetime.now(timezone.utc) + + run = AnsibleRun( + id=run_id, + playbook_path=playbook_path, + inventory_path=inventory_path, + inventory_scope=inventory_scope, + source_name=source_name, + triggered_by=session["user_id"], + status=AnsibleRunStatus.running, + started_at=now, + params=params_or_none, + ) + + async with current_app.db_sessionmaker() as db: + async with db.begin(): + db.add(run) + + asyncio.ensure_future( + executor.start_run( + current_app._get_current_object(), # type: ignore[attr-defined] + run_id, + playbook_path=playbook_path, + inventory_path=inventory_path or "", + source_path=source["path"], + params=params_or_none, + inventory_content=inventory_content, + ) + ) + return await render_template("ansible/run_started.html", run_id=run_id) +``` + +- [ ] **Step 6.3: Update `browse()` route to pass scope context** + +The `browse()` route needs to pass targets and groups for the scope picker. Update it: + +```python +@ansible_bp.get("/browse") +@require_role(UserRole.viewer) +async def browse(): + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + + all_sources = _get_sources() + source_data = [] + for source in all_sources: + playbooks = src_module.discover_playbooks(source["path"]) + inventories = src_module.discover_inventories(source["path"]) + source_data.append({ + "source": source, + "playbooks": playbooks, + "inventories": inventories, + }) + + async with current_app.db_sessionmaker() as db: + targets = (await db.execute( + select(AnsibleTarget).order_by(AnsibleTarget.name) + )).scalars().all() + groups = (await db.execute( + select(AnsibleGroup).order_by(AnsibleGroup.name) + )).scalars().all() + + return await render_template( + "ansible/browse.html", + source_data=source_data, + targets=targets, + groups=groups, + ) +``` + +- [ ] **Step 6.4: Update `steward/templates/ansible/browse.html` run form** + +Replace the existing "Inventory" form-group block (lines 73–85) with the scope picker. The existing block is: + +```html +
+ + +
+
+ + +
+``` + +Replace with: + +```html +
+ + +
+``` + +Also update the "Execute" button's onclick — remove the `manual-inv` logic since there's no manual field anymore: + +```html + +``` + +(The form can submit directly now.) + +- [ ] **Step 6.5: Update `run_detail.html` to show scope label** + +In `steward/templates/ansible/run_detail.html`, find where `run.inventory_path` is displayed and add `inventory_scope` display. After the playbook path line, add: + +```html + + Scope + {{ run.inventory_scope }} + +``` + +(Check the existing table structure in `run_detail.html` first — look for `run.playbook_path` and follow the same `` pattern.) + +- [ ] **Step 6.6: Commit** + +```bash +git add steward/ansible/routes.py \ + steward/templates/ansible/browse.html \ + steward/templates/ansible/run_detail.html +git commit -m "feat(ansible): scope picker in run form + create_run() DB inventory generation" +``` + +--- + +## Task 7: Inventory Blueprint — Groups CRUD + +**Files:** +- Create: `steward/ansible/inventory_routes.py` +- Create: `steward/templates/ansible/inventory/groups.html` +- Create: `steward/templates/ansible/inventory/group_detail.html` +- Modify: `steward/app.py` + +- [ ] **Step 7.1: Create `steward/ansible/inventory_routes.py`** + +```python +"""CRUD routes for Ansible inventory: groups and targets.""" +from __future__ import annotations +import uuid +import yaml + +from quart import Blueprint, current_app, redirect, render_template, request, url_for +from sqlalchemy import select +from sqlalchemy.orm import selectinload + +from steward.auth.middleware import require_role +from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget +from steward.models.users import UserRole + +inventory_bp = Blueprint("ansible_inventory", __name__, url_prefix="/ansible/inventory") + + +def _parse_ansible_vars(raw: str) -> dict: + """Parse a YAML/JSON string into a dict. Raises ValueError on invalid input.""" + raw = raw.strip() + if not raw: + return {} + try: + parsed = yaml.safe_load(raw) + except yaml.YAMLError as exc: + raise ValueError(f"Invalid YAML: {exc}") from exc + if parsed is None: + return {} + if not isinstance(parsed, dict): + raise ValueError("ansible_vars must be a YAML mapping (key: value pairs)") + return parsed + + +@inventory_bp.get("/groups") +@require_role(UserRole.viewer) +async def groups_list(): + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleGroup) + .options(selectinload(AnsibleGroup.targets)) + .order_by(AnsibleGroup.name) + ) + groups = result.scalars().all() + return await render_template("ansible/inventory/groups.html", groups=groups) + + +@inventory_bp.post("/groups") +@require_role(UserRole.operator) +async def groups_create(): + form = await request.form + name = form.get("name", "").strip() + if not name: + return "name is required", 400 + try: + ansible_vars = _parse_ansible_vars(form.get("ansible_vars", "")) + except ValueError as exc: + return str(exc), 400 + + group = AnsibleGroup(id=str(uuid.uuid4()), name=name, ansible_vars=ansible_vars) + async with current_app.db_sessionmaker() as db: + async with db.begin(): + db.add(group) + return redirect(url_for("ansible_inventory.groups_list")) + + +@inventory_bp.get("/groups/") +@require_role(UserRole.viewer) +async def group_detail(group_id: str): + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleGroup) + .where(AnsibleGroup.id == group_id) + .options(selectinload(AnsibleGroup.targets)) + ) + group = result.scalar_one_or_none() + if group is None: + return "Group not found", 404 + all_targets = ( + await db.execute(select(AnsibleTarget).order_by(AnsibleTarget.name)) + ).scalars().all() + member_ids = {t.id for t in group.targets} + ansible_vars_yaml = yaml.dump(group.ansible_vars) if group.ansible_vars else "" + return await render_template( + "ansible/inventory/group_detail.html", + group=group, + all_targets=all_targets, + member_ids=member_ids, + ansible_vars_yaml=ansible_vars_yaml, + ) + + +@inventory_bp.post("/groups/") +@require_role(UserRole.operator) +async def group_update(group_id: str): + form = await request.form + try: + ansible_vars = _parse_ansible_vars(form.get("ansible_vars", "")) + except ValueError as exc: + return str(exc), 400 + + async with current_app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute( + select(AnsibleGroup) + .where(AnsibleGroup.id == group_id) + .options(selectinload(AnsibleGroup.targets)) + ) + group = result.scalar_one_or_none() + if group is None: + return "Group not found", 404 + + name = form.get("name", group.name).strip() + if name: + group.name = name + group.ansible_vars = ansible_vars + + # Update member list — form sends target_ids[] checkboxes. + new_ids = set(form.getlist("target_ids")) + all_targets = ( + await db.execute(select(AnsibleTarget)) + ).scalars().all() + target_map = {t.id: t for t in all_targets} + group.targets = [target_map[tid] for tid in new_ids if tid in target_map] + + return redirect(url_for("ansible_inventory.group_detail", group_id=group_id)) + + +@inventory_bp.post("/groups//delete") +@require_role(UserRole.operator) +async def group_delete(group_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute( + select(AnsibleGroup).where(AnsibleGroup.id == group_id) + ) + group = result.scalar_one_or_none() + if group is not None: + await db.delete(group) + return redirect(url_for("ansible_inventory.groups_list")) +``` + +- [ ] **Step 7.2: Create `steward/templates/ansible/inventory/groups.html`** + +```html +{% extends "base.html" %} +{% block title %}Inventory Groups — Steward{% endblock %} +{% block content %} +
+

Inventory Groups

+ Targets +
+ +
+

New Group

+
+
+ + +
+ +
+
+ +{% if not groups %} +
+

No groups yet. Create one above.

+
+{% else %} +
+ + + + + + {% for grp in groups %} + + + + + + {% endfor %} + +
NameMembers
{{ grp.name }}{{ grp.targets | length }} + Edit +
+ +
+
+
+{% endif %} +{% endblock %} +``` + +- [ ] **Step 7.3: Create `steward/templates/ansible/inventory/group_detail.html`** + +```html +{% extends "base.html" %} +{% block title %}Group: {{ group.name }} — Steward{% endblock %} +{% block content %} +
+

Group: {{ group.name }}

+ ← Groups +
+ +
+
+

Settings

+
+ + +
+
+ + +
+
+ +
+

Members

+ {% if not all_targets %} +

No targets exist yet. + Add targets first.

+ {% else %} +
+ {% for tgt in all_targets %} + + {% endfor %} +
+ {% endif %} +
+ + +
+{% endblock %} +``` + +- [ ] **Step 7.4: Register blueprint in `steward/app.py`** + +After the existing `from .ansible.routes import ansible_bp` and `app.register_blueprint(ansible_bp)` lines, add: + +```python +from .ansible.inventory_routes import inventory_bp +app.register_blueprint(inventory_bp) +``` + +(These are inside the `create_app()` function, alongside the other blueprint registrations.) + +- [ ] **Step 7.5: Run the app and verify `/ansible/inventory/groups` loads** + +```bash +cd /home/bvandeusen/Nextcloud/Projects/FabledSteward +docker compose up -d # or start local dev server +# Navigate to http://localhost:5000/ansible/inventory/groups in browser +``` + +Expected: Groups list page renders with "No groups yet" message. No 500 errors. + +- [ ] **Step 7.6: Commit** + +```bash +git add steward/ansible/inventory_routes.py \ + steward/app.py \ + steward/templates/ansible/inventory/groups.html \ + steward/templates/ansible/inventory/group_detail.html +git commit -m "feat(ansible): inventory groups CRUD routes + templates" +``` + +--- + +## Task 8: Inventory Blueprint — Targets CRUD + +**Files:** +- Modify: `steward/ansible/inventory_routes.py` +- Create: `steward/templates/ansible/inventory/targets.html` +- Create: `steward/templates/ansible/inventory/target_detail.html` + +- [ ] **Step 8.1: Add targets routes to `steward/ansible/inventory_routes.py`** + +Append to the existing `inventory_routes.py` file (after the group routes): + +```python +@inventory_bp.get("/targets") +@require_role(UserRole.viewer) +async def targets_list(): + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleTarget) + .options(selectinload(AnsibleTarget.groups)) + .order_by(AnsibleTarget.name) + ) + targets = result.scalars().all() + return await render_template("ansible/inventory/targets.html", targets=targets) + + +@inventory_bp.post("/targets") +@require_role(UserRole.operator) +async def targets_create(): + form = await request.form + name = form.get("name", "").strip() + address = form.get("address", "").strip() + if not name or not address: + return "name and address are required", 400 + try: + ansible_vars = _parse_ansible_vars(form.get("ansible_vars", "")) + except ValueError as exc: + return str(exc), 400 + + target = AnsibleTarget( + id=str(uuid.uuid4()), name=name, address=address, ansible_vars=ansible_vars + ) + async with current_app.db_sessionmaker() as db: + async with db.begin(): + db.add(target) + return redirect(url_for("ansible_inventory.targets_list")) + + +@inventory_bp.get("/targets/") +@require_role(UserRole.viewer) +async def target_detail(target_id: str): + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleTarget) + .where(AnsibleTarget.id == target_id) + .options(selectinload(AnsibleTarget.groups)) + ) + target = result.scalar_one_or_none() + if target is None: + return "Target not found", 404 + all_groups = ( + await db.execute(select(AnsibleGroup).order_by(AnsibleGroup.name)) + ).scalars().all() + member_group_ids = {g.id for g in target.groups} + ansible_vars_yaml = yaml.dump(target.ansible_vars) if target.ansible_vars else "" + return await render_template( + "ansible/inventory/target_detail.html", + target=target, + all_groups=all_groups, + member_group_ids=member_group_ids, + ansible_vars_yaml=ansible_vars_yaml, + ) + + +@inventory_bp.post("/targets/") +@require_role(UserRole.operator) +async def target_update(target_id: str): + form = await request.form + try: + ansible_vars = _parse_ansible_vars(form.get("ansible_vars", "")) + except ValueError as exc: + return str(exc), 400 + + async with current_app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute( + select(AnsibleTarget) + .where(AnsibleTarget.id == target_id) + .options(selectinload(AnsibleTarget.groups)) + ) + target = result.scalar_one_or_none() + if target is None: + return "Target not found", 404 + + name = form.get("name", target.name).strip() + address = form.get("address", target.address).strip() + if name: + target.name = name + if address: + target.address = address + target.ansible_vars = ansible_vars + + new_group_ids = set(form.getlist("group_ids")) + all_groups_result = await db.execute(select(AnsibleGroup)) + group_map = {g.id: g for g in all_groups_result.scalars().all()} + target.groups = [group_map[gid] for gid in new_group_ids if gid in group_map] + + return redirect(url_for("ansible_inventory.target_detail", target_id=target_id)) + + +@inventory_bp.post("/targets//delete") +@require_role(UserRole.operator) +async def target_delete(target_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute( + select(AnsibleTarget).where(AnsibleTarget.id == target_id) + ) + target = result.scalar_one_or_none() + if target is not None: + await db.delete(target) + return redirect(url_for("ansible_inventory.targets_list")) +``` + +- [ ] **Step 8.2: Create `steward/templates/ansible/inventory/targets.html`** + +```html +{% extends "base.html" %} +{% block title %}Inventory Targets — Steward{% endblock %} +{% block content %} +
+

Inventory Targets

+ Groups +
+ +
+

New Target

+
+
+ + +
+
+ + +
+ +
+
+ +{% if not targets %} +
+

No targets yet. Add one above.

+
+{% else %} +
+ + + + + + {% for tgt in targets %} + + + + + + + {% endfor %} + +
NameAddressGroups
{{ tgt.name }}{{ tgt.address }} + {% for grp in tgt.groups %} + {{ grp.name }} + {% endfor %} + + Edit +
+ +
+
+
+{% endif %} +{% endblock %} +``` + +- [ ] **Step 8.3: Create `steward/templates/ansible/inventory/target_detail.html`** + +```html +{% extends "base.html" %} +{% block title %}Target: {{ target.name }} — Steward{% endblock %} +{% block content %} +
+

Target: {{ target.name }}

+ ← Targets +
+ +
+
+

Connection

+
+ + +
+
+ + +
+
+ + +
+
+ +
+

Groups

+ {% if not all_groups %} +

No groups exist yet. + Create a group first.

+ {% else %} +
+ {% for grp in all_groups %} + + {% endfor %} +
+ {% endif %} +
+ + +
+{% endblock %} +``` + +- [ ] **Step 8.4: Verify targets CRUD works** + +Navigate to `http://localhost:5000/ansible/inventory/targets` in browser. Create a target, edit it, assign it to a group, verify it appears in the groups list as a member. + +- [ ] **Step 8.5: Commit** + +```bash +git add steward/ansible/inventory_routes.py \ + steward/templates/ansible/inventory/targets.html \ + steward/templates/ansible/inventory/target_detail.html +git commit -m "feat(ansible): inventory targets CRUD routes + templates" +``` + +--- + +## Task 9: Host Edit Form — Ansible Target Link Section + +**Files:** +- Modify: `steward/templates/hosts/form.html` +- Modify: `steward/hosts/routes.py` + +- [ ] **Step 9.1: Update `edit_host()` route in `steward/hosts/routes.py`** + +The `edit_host()` function currently (around line ~149) renders `hosts/form.html` with just the host. Extend it to also pass the linked Ansible target and all available targets: + +```python +@hosts_bp.get("//edit") +@require_role(UserRole.operator) +async def edit_host(host_id: str): + from steward.models.ansible_inventory import AnsibleTarget + from sqlalchemy.orm import selectinload + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(Host).where(Host.id == host_id) + ) + host = result.scalar_one_or_none() + if host is None: + return "Host not found", 404 + + linked_target = None + linked_result = await db.execute( + select(AnsibleTarget) + .where(AnsibleTarget.host_id == host_id) + .options(selectinload(AnsibleTarget.groups)) + ) + linked_target = linked_result.scalar_one_or_none() + + all_targets = ( + await db.execute( + select(AnsibleTarget) + .where(AnsibleTarget.host_id.is_(None)) + .order_by(AnsibleTarget.name) + ) + ).scalars().all() + + return await render_template( + "hosts/form.html", + host=host, + probe_types=list(ProbeType), + linked_target=linked_target, + linkable_targets=all_targets, + ) +``` + +Note: `all_targets` shows only unlinked targets (those without a `host_id`) plus the currently linked target if any — so the user can re-select it. The form template handles this. + +Also add a new route `POST //ansible-link` to handle the link action: + +```python +@hosts_bp.post("//ansible-link") +@require_role(UserRole.operator) +async def ansible_link(host_id: str): + from steward.models.ansible_inventory import AnsibleTarget + + form = await request.form + action = form.get("action", "") + + async with current_app.db_sessionmaker() as db: + async with db.begin(): + if action == "unlink": + # Clear host_id from the currently linked target. + linked_result = await db.execute( + select(AnsibleTarget).where(AnsibleTarget.host_id == host_id) + ) + linked = linked_result.scalar_one_or_none() + if linked: + linked.host_id = None + elif action == "link": + target_id = form.get("target_id", "").strip() + if target_id: + result = await db.execute( + select(AnsibleTarget).where(AnsibleTarget.id == target_id) + ) + target = result.scalar_one_or_none() + if target: + # Clear any previous link for this host. + old_result = await db.execute( + select(AnsibleTarget).where(AnsibleTarget.host_id == host_id) + ) + old = old_result.scalar_one_or_none() + if old and old.id != target_id: + old.host_id = None + target.host_id = host_id + elif action == "create": + # Create a new target pre-filled from host data. + host_result = await db.execute( + select(Host).where(Host.id == host_id) + ) + host = host_result.scalar_one_or_none() + if host: + new_target = AnsibleTarget( + id=str(uuid.uuid4()), + name=host.name, + address=host.address, + host_id=host_id, + ) + db.add(new_target) + + return redirect(f"/hosts/{host_id}/edit") +``` + +Add `import uuid` to the top of `hosts/routes.py` if not already present. Also add `from sqlalchemy import select` and `from steward.models.hosts import Host, ProbeType` imports (check what's already imported). + +- [ ] **Step 9.2: Add Ansible section to `steward/templates/hosts/form.html`** + +At the end of the file, before `{% endblock %}`, add: + +```html +{% if host %} +
+

Ansible Target

+ {% if linked_target %} +
+
+ {{ linked_target.name }} + {{ linked_target.address }} + {% if linked_target.groups %} +
+ {% for grp in linked_target.groups %} + {{ grp.name }} + {% endfor %} +
+ {% endif %} +
+ Edit Target +
+ + +
+
+ {% else %} +

+ No Ansible target linked. Link an existing target or create a new one from this host's data. +

+
+ {% if linkable_targets %} +
+ + + +
+ {% endif %} +
+ + +
+
+ {% endif %} +
+{% endif %} +``` + +- [ ] **Step 9.3: Verify the Ansible section appears on host edit** + +Navigate to `http://localhost:5000/hosts//edit`. Verify the Ansible Target section appears at the bottom. Create a target from a host, verify it links and the target appears. Unlink and verify it clears. + +- [ ] **Step 9.4: Commit** + +```bash +git add steward/hosts/routes.py steward/templates/hosts/form.html +git commit -m "feat(ansible): host edit page Ansible target link/create/unlink section" +``` + +--- + +## Task 10: Integration Test — Migrations + fetch_scope_targets + +**Files:** +- Create: `tests/integration/test_ansible_inventory.py` + +- [ ] **Step 10.1: Write integration tests** + +Create `tests/integration/test_ansible_inventory.py`: + +```python +"""Integration tests: ansible_targets / ansible_groups tables + fetch_scope_targets. + +Requires a live Postgres DB. Run with: + pytest tests/integration/test_ansible_inventory.py -m integration -v +""" +import pytest +import pytest_asyncio +from sqlalchemy import select, text + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_ansible_targets_table_exists(app): + """Migration 0015 created the ansible_targets table.""" + async with app.db_sessionmaker() as db: + result = await db.execute( + text("SELECT column_name FROM information_schema.columns WHERE table_name = 'ansible_targets'") + ) + columns = {row[0] for row in result.fetchall()} + assert "id" in columns + assert "name" in columns + assert "address" in columns + assert "ansible_vars" in columns + assert "host_id" in columns + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_ansible_groups_table_exists(app): + """Migration 0016 created ansible_groups and ansible_target_groups.""" + async with app.db_sessionmaker() as db: + result = await db.execute( + text("SELECT column_name FROM information_schema.columns WHERE table_name = 'ansible_groups'") + ) + columns = {row[0] for row in result.fetchall()} + assert "id" in columns + assert "name" in columns + assert "ansible_vars" in columns + + async with app.db_sessionmaker() as db: + result = await db.execute( + text("SELECT column_name FROM information_schema.columns WHERE table_name = 'ansible_target_groups'") + ) + join_columns = {row[0] for row in result.fetchall()} + assert "target_id" in join_columns + assert "group_id" in join_columns + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_ansible_run_scope_column_exists(app): + """Migration 0017 added inventory_scope to ansible_runs.""" + async with app.db_sessionmaker() as db: + result = await db.execute( + text("SELECT column_name FROM information_schema.columns WHERE table_name = 'ansible_runs'") + ) + columns = {row[0] for row in result.fetchall()} + assert "inventory_scope" in columns + assert "inventory_path" in columns # still present, now nullable + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_create_target_and_group_and_fetch(app): + """Can create a target+group, assign membership, and fetch via fetch_scope_targets.""" + import uuid + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory + + target_id = str(uuid.uuid4()) + group_id = str(uuid.uuid4()) + + async with app.db_sessionmaker() as db: + async with db.begin(): + grp = AnsibleGroup( + id=group_id, + name=f"test-group-{group_id[:8]}", + ansible_vars={"env": "test"}, + ) + tgt = AnsibleTarget( + id=target_id, + name=f"test-target-{target_id[:8]}", + address="10.0.99.1", + ansible_vars={"ansible_user": "ubuntu"}, + ) + tgt.groups.append(grp) + db.add(grp) + db.add(tgt) + + # Fetch all + async with app.db_sessionmaker() as db: + all_targets = await fetch_scope_targets(db, "steward:all") + names = [t.name for t in all_targets] + assert f"test-target-{target_id[:8]}" in names + + # Fetch by group + async with app.db_sessionmaker() as db: + group_targets = await fetch_scope_targets(db, f"steward:group:{group_id}") + assert any(t.id == target_id for t in group_targets) + + # Fetch single target + async with app.db_sessionmaker() as db: + single = await fetch_scope_targets(db, f"steward:target:{target_id}") + assert len(single) == 1 + assert single[0].id == target_id + + # Generate inventory + async with app.db_sessionmaker() as db: + targets = await fetch_scope_targets(db, f"steward:group:{group_id}") + inv = generate_inventory(targets) + target_name = f"test-target-{target_id[:8]}" + group_name = f"test-group-{group_id[:8]}" + assert target_name in inv["all"]["hosts"] + assert target_name in inv[group_name]["hosts"] + assert inv["_meta"]["hostvars"][target_name]["ansible_host"] == "10.0.99.1" + assert inv["_meta"]["hostvars"][target_name]["ansible_user"] == "ubuntu" + assert inv["_meta"]["hostvars"][target_name]["env"] == "test" + + # Cleanup + async with app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute(select(AnsibleTarget).where(AnsibleTarget.id == target_id)) + t = result.scalar_one_or_none() + if t: + await db.delete(t) + result = await db.execute(select(AnsibleGroup).where(AnsibleGroup.id == group_id)) + g = result.scalar_one_or_none() + if g: + await db.delete(g) +``` + +- [ ] **Step 10.2: Run integration tests** + +```bash +pytest tests/integration/test_ansible_inventory.py -m integration -v +``` + +Expected: 4 PASSED (requires running Postgres with migrations applied). + +- [ ] **Step 10.3: Commit** + +```bash +git add tests/integration/test_ansible_inventory.py +git commit -m "test(ansible): integration tests for inventory tables + fetch_scope_targets" +``` + +--- + +## Self-Review Checklist + +- [x] **Spec coverage**: All 8 spec sections are covered: + - §1 Data Model → Tasks 1–3 + - §2 PAT/HTTP Token → Task 5 + - §3 Inventory Generation → Task 4 + - §4 Run Form → Task 6 + - §5 UI Surfaces → Tasks 7–9 + - §6 Repo integration → Task 6 (scope fallback preserved) + - §7 Migrations → Tasks 1–3 + - §8 Out of scope → not implemented ✓ +- [x] **No placeholders**: All code blocks are complete. +- [x] **Type consistency**: `AnsibleTarget`, `AnsibleGroup`, `ansible_target_groups` defined in Task 1 and referenced consistently throughout. `fetch_scope_targets(db, scope)` / `generate_inventory(targets)` defined in Task 4 and called in Task 6. +- [x] **Migration chain**: `0015 → 0016 → 0017`, each `down_revision` points to the previous. +- [x] **Session pattern**: All routes use `async with current_app.db_sessionmaker() as db:` matching existing code. +- [x] **Import pattern**: New models added to `steward/models/__init__.py` for Alembic discovery. +- [x] **`get_sources()` http_token**: Added for both git and local sources (empty string for local). diff --git a/plugins/__init__.py b/plugins/__init__.py new file mode 100644 index 0000000..1648f8e --- /dev/null +++ b/plugins/__init__.py @@ -0,0 +1 @@ +# plugins/__init__.py diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py new file mode 100644 index 0000000..5e09d1d --- /dev/null +++ b/plugins/docker/__init__.py @@ -0,0 +1,54 @@ +# plugins/docker/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + from .models import DockerContainer, DockerMetric # noqa: F401 — register with Base + + # Publish the per-host persist hook so the host agent can hand off the + # `docker` array from each sample without importing our models (opportunistic + # synergy — host_agent gates on has_capability and no-ops if we're disabled). + # required_role=viewer: this is a trusted server-side data-plane write driven + # by the authenticated agent ingest, not a user-facing privileged action. + from steward.core.capabilities import register_capability + from steward.models.users import UserRole + from .ingest import persist_host_docker + from .retention import run_docker_retention + register_capability( + "docker.persist_host_samples", persist_host_docker, + label="Persist host Docker samples", + description="Store per-host container state + metrics pushed by the host agent.", + required_role=UserRole.viewer, + ) + # Roll up + prune Docker time-series, driven by the core cleanup task without + # it importing our models. Same trusted server-side data-plane role as above. + register_capability( + "docker.run_retention", run_docker_retention, + label="Run Docker retention", + description="Roll up old docker_metrics to hourly + prune stale metrics/events.", + required_role=UserRole.viewer, + ) + + +def get_scheduled_tasks() -> list: + # Collection is now agent-driven (pushed to the host_agent ingest); the + # central socket scrape was removed. No periodic task to register. + return [] + + +def get_blueprint(): + from .routes import docker_bp + return docker_bp + + +def get_nav() -> list[dict]: + # Surfaces in the sidebar's Infrastructure group (fleet-wide container view). + return [{"label": "Docker", "href": "/plugins/docker/"}] diff --git a/plugins/docker/dedup.py b/plugins/docker/dedup.py new file mode 100644 index 0000000..5a1e5b7 --- /dev/null +++ b/plugins/docker/dedup.py @@ -0,0 +1,27 @@ +"""Model-free Docker view helpers. + +Kept separate from routes.py/models.py so tests can import it directly: importing +a module that re-imports plugins.docker.models AFTER the app has loaded the docker +plugin raises "Table 'docker_containers' is already defined". This module touches +no ORM models, so it's safe to import in either environment. +""" +from __future__ import annotations + + +def dedup_by_container_id(containers: list) -> list: + """Collapse containers double-reported across hosts. Swarm-aware agents on + every manager list cluster-wide tasks, so the same container (identical + container_id) arrives once per manager and would otherwise be counted N times. + Keep the first occurrence per non-empty container_id — callers order + running-first, so a running report wins over a stale stopped one. Rows without + a container_id (older agents) can't be matched, so they're kept as-is.""" + seen: set[str] = set() + out = [] + for c in containers: + cid = (c.container_id or "").strip() + if cid: + if cid in seen: + continue + seen.add(cid) + out.append(c) + return out diff --git a/plugins/docker/ingest.py b/plugins/docker/ingest.py new file mode 100644 index 0000000..b67a404 --- /dev/null +++ b/plugins/docker/ingest.py @@ -0,0 +1,336 @@ +# plugins/docker/ingest.py +"""Persist host-scoped Docker samples pushed by the host agent. + +Published as the "docker.persist_host_samples" capability (see __init__.setup), +so the host_agent plugin can hand off a sample's `docker` array (and, on a swarm +manager, its `swarm` object) WITHOUT importing the docker models — the coupling +is opportunistic and degrades to a no-op when the docker plugin is disabled. +Runs inside the caller's open transaction (the ingest handler's SAVEPOINT); +never opens or commits its own. +""" +from __future__ import annotations + +import json +from datetime import datetime + +from sqlalchemy import delete, select + + +# Stopped/terminal container states (anything not "running"/"paused"/"restarting"). +_STOPPED_STATES = {"exited", "dead", "stopped"} + + +def _parse_started_at(value) -> datetime | None: + """Parse the agent's ISO-8601 started_at string back to a datetime.""" + if not isinstance(value, str) or not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def _derive_events(old_state: dict, new_containers: list) -> list: + """Diff a fresh snapshot against stored per-container state → lifecycle events. + + Pure function (no DB) so it's unit-testable. `old_state` maps name → the + previously stored {status, health, oom_killed, exit_code}; `new_containers` + is the newest snapshot's list of container dicts. Returns a list of + (container_name, event, detail) tuples: + + start — a container transitions into running (or a genuinely new + container appears already running) + stop — running → not-running, or a running container is removed + die — same as stop but with a non-zero exit code (abnormal) + oom — OOMKilled flips False→True + health_change — HEALTHCHECK status string changes + + The CALLER skips this entirely on a host's first-ever snapshot (empty + old_state) so the baseline doesn't emit a spurious "start" per existing + container — a later-appearing container still gets one because by then + old_state is populated and that container's prior entry is simply absent. + """ + events: list = [] + new_by_name = {c["name"]: c for c in new_containers if c.get("name")} + + for name, c in new_by_name.items(): + new_status = (c.get("status") or "").lower() + new_running = new_status == "running" + new_oom = bool(c.get("oom_killed")) + new_health = c.get("health") + new_exit = c.get("exit_code") + old = old_state.get(name) + + if old is None: + # Newly observed container — only a start is meaningful (we have no + # prior state to diff a death against). + if new_running: + events.append((name, "start", c.get("image") or None)) + continue + + old_running = (old.get("status") or "").lower() == "running" + if new_running and not old_running: + events.append((name, "start", c.get("image") or None)) + elif old_running and not new_running: + if new_exit not in (None, 0): + events.append((name, "die", f"exit {new_exit}")) + else: + events.append((name, "stop", None)) + + if new_oom and not bool(old.get("oom_killed")): + events.append((name, "oom", None)) + + if new_health != old.get("health") and (new_health or old.get("health")): + events.append((name, "health_change", + f"{old.get('health') or '?'}→{new_health or '?'}")) + + # A running container that vanished from the listing entirely (removed). + for name, old in old_state.items(): + if name not in new_by_name and (old.get("status") or "").lower() == "running": + events.append((name, "stop", "removed")) + + return events + + +async def _persist_swarm(session, host, swarm: dict) -> None: + """Upsert this manager's swarm topology; drop rows no longer reported. + + Current-state tables (not time-series): a manager re-reports its full + services/nodes set every sample, so we upsert what's present and delete what + isn't (scoped to this host so two managers don't clobber each other). + """ + from datetime import timezone + from .models import DockerSwarmNode, DockerSwarmService + + now = datetime.now(timezone.utc) + + services = swarm.get("services") or [] + seen_services: set[str] = set() + for s in services: + name = s.get("service_name") + if not name: + continue + seen_services.add(name) + row = await session.get(DockerSwarmService, (host.id, name)) + if row is None: + row = DockerSwarmService(host_id=host.id, service_name=name) + session.add(row) + row.mode = s.get("mode") or "replicated" + row.desired = int(s.get("desired") or 0) + row.running = int(s.get("running") or 0) + row.image = s.get("image") + row.placement_json = json.dumps(s.get("placement") or []) + row.updated_at = now + stale_services = delete(DockerSwarmService).where( + DockerSwarmService.host_id == host.id) + if seen_services: + stale_services = stale_services.where( + DockerSwarmService.service_name.notin_(seen_services)) + await session.execute(stale_services) + + nodes = swarm.get("nodes") or [] + seen_nodes: set[str] = set() + for n in nodes: + nid = n.get("node_id") + if not nid: + continue + seen_nodes.add(nid) + row = await session.get(DockerSwarmNode, (host.id, nid)) + if row is None: + row = DockerSwarmNode(host_id=host.id, node_id=nid) + session.add(row) + row.hostname = n.get("hostname") or "" + row.role = n.get("role") or "worker" + row.availability = n.get("availability") or "active" + row.status = n.get("status") or "unknown" + row.leader = bool(n.get("leader", False)) + row.updated_at = now + stale_nodes = delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id) + if seen_nodes: + stale_nodes = stale_nodes.where(DockerSwarmNode.node_id.notin_(seen_nodes)) + await session.execute(stale_nodes) + + +async def _persist_disk(session, host, disk: dict) -> None: + """Upsert this host's /system/df summary + replace its image storage rows. + + Current-state (not time-series): the agent re-reports the full summary each + interval, so we overwrite the single summary row and replace the image set + (delete rows no longer present, upsert the rest), scoped to this host. + """ + from datetime import timezone + from .models import DockerDiskUsage, DockerImage + + now = datetime.now(timezone.utc) + + summary = await session.get(DockerDiskUsage, host.id) + if summary is None: + summary = DockerDiskUsage(host_id=host.id) + session.add(summary) + summary.layers_size = int(disk.get("layers_size") or 0) + summary.images_size = int(disk.get("images_size") or 0) + summary.images_reclaimable = int(disk.get("images_reclaimable") or 0) + summary.containers_size = int(disk.get("containers_size") or 0) + summary.volumes_size = int(disk.get("volumes_size") or 0) + summary.build_cache_size = int(disk.get("build_cache_size") or 0) + summary.images_total = int(disk.get("images_total") or 0) + summary.images_active = int(disk.get("images_active") or 0) + summary.containers_count = int(disk.get("containers_count") or 0) + summary.volumes_count = int(disk.get("volumes_count") or 0) + summary.updated_at = now + + images = disk.get("images") or [] + seen: set[str] = set() + for im in images: + iid = im.get("image_id") + if not iid or iid in seen: + continue + seen.add(iid) + row = await session.get(DockerImage, (host.id, iid)) + if row is None: + row = DockerImage(host_id=host.id, image_id=iid) + session.add(row) + row.repo_tag = (im.get("repo_tag") or "")[:512] + row.size = int(im.get("size") or 0) + row.shared_size = int(im.get("shared_size") or 0) + row.containers = int(im.get("containers") or 0) + row.updated_at = now + stale = delete(DockerImage).where(DockerImage.host_id == host.id) + if seen: + stale = stale.where(DockerImage.image_id.notin_(seen)) + await session.execute(stale) + + +async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -> None: + """Upsert containers + time-series + lifecycle events + swarm for one host. + + `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) — + one entry per ingested sample carrying docker data (usually one; more when + the agent flushes a backlog). Every snapshot contributes time-series + DockerMetric rows; the newest snapshot drives current container state, the + alert pipeline, and lifecycle-event derivation. `swarm` is the newest + sample's swarm object (or None off managers) — persisted when present. + `disk` is the newest sample's /system/df summary (or None on Docker-less + hosts) — persisted when present. + """ + from steward.core.alerts import record_metric + from .models import DockerContainer, DockerEvent, DockerMetric + + if swarm is not None: + await _persist_swarm(session, host, swarm) + if disk is not None: + await _persist_disk(session, host, disk) + + if not snapshots: + return + ordered = sorted(snapshots, key=lambda s: s[0]) + latest_at, latest_containers = ordered[-1] + + # Time-series points for every snapshot (running containers with a CPU read). + for recorded_at, containers in ordered: + for c in containers: + if c.get("status") == "running" and c.get("cpu_pct") is not None: + session.add(DockerMetric( + host_id=host.id, + container_name=c["name"], + scraped_at=recorded_at, + cpu_pct=c["cpu_pct"], + mem_pct=c.get("mem_pct") or 0.0, + mem_usage_bytes=c.get("mem_usage_bytes") or 0, + )) + + # Snapshot of stored per-container state BEFORE the upsert overwrites it — + # used to diff lifecycle events. Skip event derivation on the host's first + # snapshot (empty state) so we don't emit a start per pre-existing container. + old_rows = (await session.execute( + select(DockerContainer).where(DockerContainer.host_id == host.id) + )).scalars().all() + old_state = { + r.name: {"status": r.status, "health": r.health, + "oom_killed": r.oom_killed, "exit_code": r.exit_code} + for r in old_rows + } + if old_state: + for name, event, detail in _derive_events(old_state, latest_containers): + session.add(DockerEvent( + host_id=host.id, container_name=name, + event=event, detail=detail, at=latest_at, + )) + + # Current state + alerts from the newest snapshot only. + for c in latest_containers: + name = c.get("name") + if not name: + continue + existing = await session.get(DockerContainer, (host.id, name)) + if existing is None: + existing = DockerContainer(host_id=host.id, name=name) + session.add(existing) + existing.container_id = c.get("container_id", "") or "" + existing.image = c.get("image", "") or "" + existing.status = c.get("status", "unknown") or "unknown" + existing.cpu_pct = c.get("cpu_pct") + existing.mem_usage_bytes = c.get("mem_usage_bytes") + existing.mem_limit_bytes = c.get("mem_limit_bytes") + existing.mem_pct = c.get("mem_pct") + existing.ports_json = json.dumps(c.get("ports") or []) + existing.started_at = _parse_started_at(c.get("started_at")) + existing.scraped_at = latest_at + # Enrichment (agent ≥ 1.4.0; .get keeps older agents working — fields + # stay None/0 when absent from the payload). + existing.restart_count = c.get("restart_count", 0) or 0 + existing.health = c.get("health") + existing.exit_code = c.get("exit_code") + existing.oom_killed = bool(c.get("oom_killed", False)) + existing.compose_project = c.get("compose_project") + existing.service_name = c.get("service_name") + existing.task_id = c.get("task_id") + existing.node_id = c.get("node_id") + existing.net_rx_bytes = c.get("net_rx_bytes") + existing.net_tx_bytes = c.get("net_tx_bytes") + existing.blk_read_bytes = c.get("blk_read_bytes") + existing.blk_write_bytes = c.get("blk_write_bytes") + + # Alert pipeline — resource is host-scoped so containers of the same name + # on different hosts don't collide in the metric/alert namespace. + resource = f"{host.name}/{name}" + if c.get("status") == "running" and c.get("cpu_pct") is not None: + await record_metric( + session=session, source_module="docker", + resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"], + ) + if c.get("mem_pct") is not None: + await record_metric( + session=session, source_module="docker", + resource_name=resource, metric_name="mem_pct", value=c["mem_pct"], + ) + # Restart count is alertable regardless of state (crash-looping matters + # most while the container is down/restarting). + await record_metric( + session=session, source_module="docker", + resource_name=resource, metric_name="restart_count", + value=float(c.get("restart_count", 0) or 0), + ) + # Health → 1.0/0.0 only for containers that actually define a HEALTHCHECK + # (health is None otherwise — recording 0 would false-alarm every plain + # container). + health = c.get("health") + if health in ("healthy", "unhealthy", "starting"): + await record_metric( + session=session, source_module="docker", + resource_name=resource, metric_name="is_healthy", + value=1.0 if health == "healthy" else 0.0, + ) + + # Reap containers that vanished from the host's latest listing. Without this, + # removed one-shot containers (CI job runners, buildkit builders, codex jobs) + # accumulate forever as permanent "stopped" rows and inflate every count. + # The listing is authoritative (event derivation already treats absence as + # removal), so anything not in the newest snapshot no longer exists on the + # host. Mirrors the image/swarm/disk persisters. An empty snapshot legitimately + # means the host has no containers, so the unfiltered delete is correct. + seen_names = {c["name"] for c in latest_containers if c.get("name")} + reap = delete(DockerContainer).where(DockerContainer.host_id == host.id) + if seen_names: + reap = reap.where(DockerContainer.name.notin_(seen_names)) + await session.execute(reap) diff --git a/steward/dns/__init__.py b/plugins/docker/migrations/__init__.py similarity index 100% rename from steward/dns/__init__.py rename to plugins/docker/migrations/__init__.py diff --git a/plugins/docker/migrations/env.py b/plugins/docker/migrations/env.py new file mode 100644 index 0000000..3878d67 --- /dev/null +++ b/plugins/docker/migrations/env.py @@ -0,0 +1,70 @@ +# plugins/docker/migrations/env.py +"""Alembic env.py for the Docker plugin.""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from steward.models.base import Base +import steward.models # noqa: F401 +from plugins.docker.models import DockerContainer, DockerMetric # noqa: F401 + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("STEWARD_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/steward/ping/__init__.py b/plugins/docker/migrations/versions/__init__.py similarity index 100% rename from steward/ping/__init__.py rename to plugins/docker/migrations/versions/__init__.py diff --git a/plugins/docker/migrations/versions/docker_001_initial.py b/plugins/docker/migrations/versions/docker_001_initial.py new file mode 100644 index 0000000..90e5298 --- /dev/null +++ b/plugins/docker/migrations/versions/docker_001_initial.py @@ -0,0 +1,47 @@ +"""Docker plugin initial tables + +Revision ID: docker_001_initial +Revises: (none — branch off core via depends_on) +Create Date: 2026-03-22 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "docker" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "docker_containers", + sa.Column("name", sa.String(255), primary_key=True), + sa.Column("container_id", sa.String(64), nullable=False, server_default=""), + sa.Column("image", sa.String(512), nullable=False, server_default=""), + sa.Column("status", sa.String(32), nullable=False, server_default="unknown"), + sa.Column("cpu_pct", sa.Float, nullable=True), + sa.Column("mem_usage_bytes", sa.Integer, nullable=True), + sa.Column("mem_limit_bytes", sa.Integer, nullable=True), + sa.Column("mem_pct", sa.Float, nullable=True), + sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"), + sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "docker_metrics", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("container_name", sa.String(255), nullable=False, index=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True), + sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"), + ) + + +def downgrade() -> None: + op.drop_table("docker_metrics") + op.drop_table("docker_containers") diff --git a/plugins/docker/migrations/versions/docker_002_host_scoped.py b/plugins/docker/migrations/versions/docker_002_host_scoped.py new file mode 100644 index 0000000..aaefa83 --- /dev/null +++ b/plugins/docker/migrations/versions/docker_002_host_scoped.py @@ -0,0 +1,94 @@ +"""Docker collection goes per-host: add host_id, re-key by (host_id, name) + +Collection moved from the central single-socket scrape to the host agent, so +containers are now scoped to the host that reported them. docker_containers is +re-keyed (host_id, name) and docker_metrics gains host_id. + +Dev-only posture (rule 122): the old tables only ever held the Steward box's +own containers (a single global namespace), which are disposable — so this +DROP+recreates rather than backfilling a host_id onto orphan rows. + +Revision ID: docker_002_host_scoped +Revises: docker_001_initial +Create Date: 2026-06-18 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_002_host_scoped" +down_revision: Union[str, None] = "docker_001_initial" +branch_labels: Union[str, Sequence[str], None] = None +# FK targets hosts.id (created in 0002_core_monitors) — make the edge explicit. +depends_on: Union[str, Sequence[str], None] = ("0002_core_monitors",) + + +def upgrade() -> None: + op.drop_table("docker_metrics") + op.drop_table("docker_containers") + + op.create_table( + "docker_containers", + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True), + sa.Column("name", sa.String(255), primary_key=True), + sa.Column("container_id", sa.String(64), nullable=False, server_default=""), + sa.Column("image", sa.String(512), nullable=False, server_default=""), + sa.Column("status", sa.String(32), nullable=False, server_default="unknown"), + sa.Column("cpu_pct", sa.Float, nullable=True), + sa.Column("mem_usage_bytes", sa.Integer, nullable=True), + sa.Column("mem_limit_bytes", sa.Integer, nullable=True), + sa.Column("mem_pct", sa.Float, nullable=True), + sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"), + sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "docker_metrics", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False), + sa.Column("container_name", sa.String(255), nullable=False), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"), + ) + op.create_index("ix_docker_metrics_host_id", "docker_metrics", ["host_id"]) + op.create_index("ix_docker_metrics_container_name", "docker_metrics", + ["container_name"]) + op.create_index("ix_docker_metrics_scraped_at", "docker_metrics", ["scraped_at"]) + op.create_index("ix_docker_metrics_host_container_time", "docker_metrics", + ["host_id", "container_name", "scraped_at"]) + + +def downgrade() -> None: + op.drop_table("docker_metrics") + op.drop_table("docker_containers") + + op.create_table( + "docker_containers", + sa.Column("name", sa.String(255), primary_key=True), + sa.Column("container_id", sa.String(64), nullable=False, server_default=""), + sa.Column("image", sa.String(512), nullable=False, server_default=""), + sa.Column("status", sa.String(32), nullable=False, server_default="unknown"), + sa.Column("cpu_pct", sa.Float, nullable=True), + sa.Column("mem_usage_bytes", sa.Integer, nullable=True), + sa.Column("mem_limit_bytes", sa.Integer, nullable=True), + sa.Column("mem_pct", sa.Float, nullable=True), + sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"), + sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_table( + "docker_metrics", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("container_name", sa.String(255), nullable=False, index=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True), + sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"), + ) diff --git a/plugins/docker/migrations/versions/docker_003_container_enrichment.py b/plugins/docker/migrations/versions/docker_003_container_enrichment.py new file mode 100644 index 0000000..4a7f794 --- /dev/null +++ b/plugins/docker/migrations/versions/docker_003_container_enrichment.py @@ -0,0 +1,41 @@ +"""Docker container enrichment: health, exit/restart, grouping, I/O counters + +Adds the fields the agent (≥1.4.0) now reports per container beyond the basic +state: health status, exit code, OOM flag, compose/swarm grouping labels, and +cumulative network/block I/O counters. Additive columns — no data loss, so no +DROP+recreate needed here. + +Revision ID: docker_003_container_enrichment +Revises: docker_002_host_scoped +Create Date: 2026-06-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_003_container_enrichment" +down_revision: Union[str, None] = "docker_002_host_scoped" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("docker_containers", sa.Column("health", sa.String(16), nullable=True)) + op.add_column("docker_containers", sa.Column("exit_code", sa.Integer, nullable=True)) + op.add_column("docker_containers", + sa.Column("oom_killed", sa.Boolean, nullable=False, server_default=sa.false())) + op.add_column("docker_containers", sa.Column("compose_project", sa.String(255), nullable=True)) + op.add_column("docker_containers", sa.Column("service_name", sa.String(255), nullable=True)) + op.add_column("docker_containers", sa.Column("task_id", sa.String(64), nullable=True)) + op.add_column("docker_containers", sa.Column("node_id", sa.String(64), nullable=True)) + op.add_column("docker_containers", sa.Column("net_rx_bytes", sa.BigInteger, nullable=True)) + op.add_column("docker_containers", sa.Column("net_tx_bytes", sa.BigInteger, nullable=True)) + op.add_column("docker_containers", sa.Column("blk_read_bytes", sa.BigInteger, nullable=True)) + op.add_column("docker_containers", sa.Column("blk_write_bytes", sa.BigInteger, nullable=True)) + + +def downgrade() -> None: + for col in ("blk_write_bytes", "blk_read_bytes", "net_tx_bytes", "net_rx_bytes", + "node_id", "task_id", "service_name", "compose_project", + "oom_killed", "exit_code", "health"): + op.drop_column("docker_containers", col) diff --git a/plugins/docker/migrations/versions/docker_004_events_swarm.py b/plugins/docker/migrations/versions/docker_004_events_swarm.py new file mode 100644 index 0000000..2f236ca --- /dev/null +++ b/plugins/docker/migrations/versions/docker_004_events_swarm.py @@ -0,0 +1,77 @@ +"""Docker lifecycle events + swarm topology tables + +Adds the storage for milestone-77 monitoring depth that doesn't fit on the +per-container row: + + * docker_events — lifecycle (start/stop/die/oom/health_change) derived by + diffing consecutive host snapshots; host-scoped, indexed for both timeline + lookups (host_id, container_name, at) and retention pruning (at). + * docker_swarm_services / docker_swarm_nodes — manager-reported Swarm + topology (desired-vs-running replicas, node role/availability/status). + +All new tables — purely additive, so no DROP+recreate. `event`/`mode`/`role` +etc. are plain strings (no CHECK whitelist), matching how docker_containers +models `status`; keeps the value sets open without a migration per addition +(so rule 36 doesn't apply here). + +Revision ID: docker_004_events_swarm +Revises: docker_003_container_enrichment +Create Date: 2026-06-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_004_events_swarm" +down_revision: Union[str, None] = "docker_003_container_enrichment" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "docker_events", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False), + sa.Column("container_name", sa.String(255), nullable=False), + sa.Column("event", sa.String(16), nullable=False), + sa.Column("detail", sa.String(255), nullable=True), + sa.Column("at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index("ix_docker_events_host_container_time", "docker_events", + ["host_id", "container_name", "at"]) + op.create_index("ix_docker_events_at", "docker_events", ["at"]) + + op.create_table( + "docker_swarm_services", + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True), + sa.Column("service_name", sa.String(255), primary_key=True), + sa.Column("mode", sa.String(16), nullable=False, server_default="replicated"), + sa.Column("desired", sa.Integer, nullable=False, server_default="0"), + sa.Column("running", sa.Integer, nullable=False, server_default="0"), + sa.Column("image", sa.String(512), nullable=True), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "docker_swarm_nodes", + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True), + sa.Column("node_id", sa.String(64), primary_key=True), + sa.Column("hostname", sa.String(255), nullable=False, server_default=""), + sa.Column("role", sa.String(16), nullable=False, server_default="worker"), + sa.Column("availability", sa.String(16), nullable=False, server_default="active"), + sa.Column("status", sa.String(16), nullable=False, server_default="unknown"), + sa.Column("leader", sa.Boolean, nullable=False, server_default=sa.false()), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("docker_swarm_nodes") + op.drop_table("docker_swarm_services") + op.drop_index("ix_docker_events_at", table_name="docker_events") + op.drop_index("ix_docker_events_host_container_time", table_name="docker_events") + op.drop_table("docker_events") diff --git a/plugins/docker/migrations/versions/docker_005_swarm_placement.py b/plugins/docker/migrations/versions/docker_005_swarm_placement.py new file mode 100644 index 0000000..0e90bd5 --- /dev/null +++ b/plugins/docker/migrations/versions/docker_005_swarm_placement.py @@ -0,0 +1,29 @@ +"""Docker swarm service placement column + +Adds docker_swarm_services.placement_json — the task→node placement of a +service's running replicas, captured from the agent's swarm payload (a manager +sees every task, so this records cross-node placement the local container rows +can't). Additive column; no DROP+recreate. + +Revision ID: docker_005_swarm_placement +Revises: docker_004_events_swarm +Create Date: 2026-06-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_005_swarm_placement" +down_revision: Union[str, None] = "docker_004_events_swarm" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("docker_swarm_services", + sa.Column("placement_json", sa.Text, nullable=False, + server_default="[]")) + + +def downgrade() -> None: + op.drop_column("docker_swarm_services", "placement_json") diff --git a/plugins/docker/migrations/versions/docker_006_metric_rollup.py b/plugins/docker/migrations/versions/docker_006_metric_rollup.py new file mode 100644 index 0000000..782abfd --- /dev/null +++ b/plugins/docker/migrations/versions/docker_006_metric_rollup.py @@ -0,0 +1,45 @@ +"""Docker hourly metric rollup table + +Adds docker_metrics_hourly — the coarse series that retention rolls raw +docker_metrics into before pruning them, so multi-day history stays cheap. +One row per (host, container, hour bucket); the unique constraint is the +conflict target for the idempotent rollup upsert. Additive create_table. + +Revision ID: docker_006_metric_rollup +Revises: docker_005_swarm_placement +Create Date: 2026-06-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_006_metric_rollup" +down_revision: Union[str, None] = "docker_005_swarm_placement" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "docker_metrics_hourly", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("host_id", sa.String(length=36), nullable=False), + sa.Column("container_name", sa.String(length=255), nullable=False), + sa.Column("bucket", sa.DateTime(timezone=True), nullable=False), + sa.Column("cpu_pct", sa.Float(), nullable=False, server_default="0"), + sa.Column("mem_pct", sa.Float(), nullable=False, server_default="0"), + sa.Column("mem_usage_bytes", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("sample_count", sa.Integer(), nullable=False, server_default="0"), + sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("host_id", "container_name", "bucket", + name="uq_docker_metrics_hourly_bucket"), + ) + op.create_index("ix_docker_metrics_hourly_bucket", + "docker_metrics_hourly", ["bucket"]) + + +def downgrade() -> None: + op.drop_index("ix_docker_metrics_hourly_bucket", + table_name="docker_metrics_hourly") + op.drop_table("docker_metrics_hourly") diff --git a/plugins/docker/migrations/versions/docker_007_disk_usage.py b/plugins/docker/migrations/versions/docker_007_disk_usage.py new file mode 100644 index 0000000..f6f0cb5 --- /dev/null +++ b/plugins/docker/migrations/versions/docker_007_disk_usage.py @@ -0,0 +1,55 @@ +"""Docker disk usage + image storage tables + +Adds docker_disk_usage (one per-host /system/df summary row) and docker_images +(the heavy-hitter image storage records). Both current-state, host-scoped, +re-reported each interval. Additive create_table. + +Revision ID: docker_007_disk_usage +Revises: docker_006_metric_rollup +Create Date: 2026-06-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_007_disk_usage" +down_revision: Union[str, None] = "docker_006_metric_rollup" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "docker_disk_usage", + sa.Column("host_id", sa.String(length=36), nullable=False), + sa.Column("layers_size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("images_size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("images_reclaimable", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("containers_size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("volumes_size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("build_cache_size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("images_total", sa.Integer(), nullable=False, server_default="0"), + sa.Column("images_active", sa.Integer(), nullable=False, server_default="0"), + sa.Column("containers_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("volumes_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("host_id"), + ) + op.create_table( + "docker_images", + sa.Column("host_id", sa.String(length=36), nullable=False), + sa.Column("image_id", sa.String(length=64), nullable=False), + sa.Column("repo_tag", sa.String(length=512), nullable=False, server_default=""), + sa.Column("size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("shared_size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("containers", sa.Integer(), nullable=False, server_default="0"), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("host_id", "image_id"), + ) + + +def downgrade() -> None: + op.drop_table("docker_images") + op.drop_table("docker_disk_usage") diff --git a/plugins/docker/migrations/versions/docker_008_bigint_mem.py b/plugins/docker/migrations/versions/docker_008_bigint_mem.py new file mode 100644 index 0000000..b11b9ad --- /dev/null +++ b/plugins/docker/migrations/versions/docker_008_bigint_mem.py @@ -0,0 +1,33 @@ +"""Widen memory byte columns to BIGINT + +docker_metrics.mem_usage_bytes and docker_containers.mem_usage_bytes / +mem_limit_bytes were int4, which overflows for any container using >2.1 GB +(asyncpg: "value out of int32 range"), failing the whole ingest batch. Widen to +BIGINT — a safe in-place int4→int8 promotion, no data rewrite. + +Revision ID: docker_008_bigint_mem +Revises: docker_007_disk_usage +Create Date: 2026-06-20 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_008_bigint_mem" +down_revision: Union[str, None] = "docker_007_disk_usage" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column("docker_metrics", "mem_usage_bytes", type_=sa.BigInteger()) + op.alter_column("docker_containers", "mem_usage_bytes", type_=sa.BigInteger()) + op.alter_column("docker_containers", "mem_limit_bytes", type_=sa.BigInteger()) + + +def downgrade() -> None: + # Narrowing back to int4 will error if any stored value exceeds 2^31 — that's + # the very condition this migration fixed, so downgrade is best-effort. + op.alter_column("docker_containers", "mem_limit_bytes", type_=sa.Integer()) + op.alter_column("docker_containers", "mem_usage_bytes", type_=sa.Integer()) + op.alter_column("docker_metrics", "mem_usage_bytes", type_=sa.Integer()) diff --git a/plugins/docker/models.py b/plugins/docker/models.py new file mode 100644 index 0000000..60f625d --- /dev/null +++ b/plugins/docker/models.py @@ -0,0 +1,264 @@ +# plugins/docker/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import ( + BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text, + UniqueConstraint, +) +from sqlalchemy.orm import Mapped, mapped_column +from steward.models.base import Base + + +class DockerContainer(Base): + """Latest known state per container, scoped to the host that reported it. + + Collection is per-host via the host agent, so container names are only + unique within a host — the natural key is (host_id, name). host_id is NOT + NULL: every container arrives through a host_agent ingest that resolves a + Host first. Deleting a host cascades its containers away. + """ + __tablename__ = "docker_containers" + + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True + ) + name: Mapped[str] = mapped_column(String(255), primary_key=True) + container_id: Mapped[str] = mapped_column(String(64), nullable=False, default="") + image: Mapped[str] = mapped_column(String(512), nullable=False, default="") + status: Mapped[str] = mapped_column(String(32), nullable=False, default="unknown") + # running | stopped | paused | exited | dead + cpu_pct: Mapped[float | None] = mapped_column(Float, nullable=True) + # BigInteger: a container's memory usage/limit routinely exceeds 2^31 bytes + # (~2.1 GB), which overflows int4 and fails the insert. + mem_usage_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + mem_limit_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + mem_pct: Mapped[float | None] = mapped_column(Float, nullable=True) + restart_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + ports_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + # JSON: [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}] + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + # ── Enrichment (agent ≥ 1.4.0) ──────────────────────────────────────────── + # Health/exit/restart come from `docker inspect` (not the list endpoint); + # exit_code is only meaningful for stopped containers. + health: Mapped[str | None] = mapped_column(String(16), nullable=True) # healthy|unhealthy|starting + exit_code: Mapped[int | None] = mapped_column(Integer, nullable=True) + oom_killed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + # Grouping: compose project + swarm placement, read off container labels. + compose_project: Mapped[str | None] = mapped_column(String(255), nullable=True) + service_name: Mapped[str | None] = mapped_column(String(255), nullable=True) + task_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + node_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + # Cumulative-since-start I/O counters (BigInteger — they exceed 2^31 quickly). + net_rx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + net_tx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + blk_read_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + blk_write_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + + +class DockerMetric(Base): + """Time-series CPU/memory per container — one row per sample per running + container, scoped to the reporting host.""" + __tablename__ = "docker_metrics" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True + ) + container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + index=True, + ) + cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + # BigInteger: per-container memory usage exceeds 2^31 bytes (~2.1 GB) routinely. + mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + + # Per-container history lookups filter on (host_id, container_name) then sort + # by time — a composite index keeps the rows() sparkline queries cheap. + __table_args__ = ( + Index("ix_docker_metrics_host_container_time", + "host_id", "container_name", "scraped_at"), + ) + + +class DockerMetricHourly(Base): + """Hourly rollup of docker_metrics — avg cpu/mem per container per hour. + + Raw per-sample rows (~2880/container/day at 30s) are pruned beyond a short + window; before deletion they're aggregated here so multi-day history stays + cheap to store and query. One row per (host, container, hour bucket); the + unique constraint lets retention upsert idempotently if it re-runs before the + raw rows are deleted. `bucket` is the hour-truncated sample time. + """ + __tablename__ = "docker_metrics_hourly" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False + ) + container_name: Mapped[str] = mapped_column(String(255), nullable=False) + bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + __table_args__ = ( + # One bucket per container per host — the conflict target for the + # idempotent rollup upsert; doubles as the history-query index. + UniqueConstraint("host_id", "container_name", "bucket", + name="uq_docker_metrics_hourly_bucket"), + Index("ix_docker_metrics_hourly_bucket", "bucket"), + ) + + +class DockerEvent(Base): + """Lifecycle events derived by diffing consecutive host snapshots. + + The agent only reports current state, so Steward synthesises lifecycle by + comparing each new snapshot against the stored DockerContainer state for the + host: a container that appears → `start`; one that vanishes → `stop`; a + transition to a stopped status with a non-zero exit → `die`; an OOM-kill + flip → `oom`; a health status change → `health_change`. Scoped to the + reporting host (names are only unique within a host). `event` is a plain + string (no CHECK whitelist), matching how `status` is modelled on + DockerContainer — keeps the value set open without a migration per addition. + """ + __tablename__ = "docker_events" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False + ) + container_name: Mapped[str] = mapped_column(String(255), nullable=False) + event: Mapped[str] = mapped_column(String(16), nullable=False) + # start | stop | die | oom | health_change + detail: Mapped[str | None] = mapped_column(String(255), nullable=True) + # e.g. "exit 137", "healthy→unhealthy", image on start + at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + # Timeline lookups filter on (host_id, container_name) and sort by time; + # retention prunes by `at` alone — index both access paths. + __table_args__ = ( + Index("ix_docker_events_host_container_time", + "host_id", "container_name", "at"), + Index("ix_docker_events_at", "at"), + ) + + +class DockerSwarmService(Base): + """A Swarm service as seen by a manager host: desired vs running replicas. + + Manager-scoped — only a host that is a Swarm manager reports these, so a + single-manager cluster yields one row set keyed by that host. Service names + are unique within a swarm; we still key by (host_id, service_name) so two + managers reporting independently don't collide. + """ + __tablename__ = "docker_swarm_services" + + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True + ) + service_name: Mapped[str] = mapped_column(String(255), primary_key=True) + mode: Mapped[str] = mapped_column(String(16), nullable=False, default="replicated") + # replicated | global + desired: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + running: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + image: Mapped[str | None] = mapped_column(String(512), nullable=True) + # task→node placement of running replicas: [{"node_id", "running"}], JSON. + # A manager sees every task, so this captures cross-node placement that the + # local docker_containers rows (this host only) can't. + placement_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + +class DockerDiskUsage(Base): + """Per-host Docker disk usage summary from `/system/df` (current state). + + One row per host (the agent re-reports the full summary each interval). + Reclaimable = bytes held by images no container references. Sizes are bytes + (BigInteger — image caches exceed 2^31 easily). + """ + __tablename__ = "docker_disk_usage" + + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True + ) + layers_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + images_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + images_reclaimable: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + containers_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + volumes_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + build_cache_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + images_total: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + images_active: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + containers_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + volumes_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + +class DockerImage(Base): + """Per-host image storage record (the heavy hitters from `/system/df`). + + Keyed (host_id, image_id); `containers` is the reference count (0 ⇒ the + image's size is reclaimable). Replaced wholesale per host on each report. + """ + __tablename__ = "docker_images" + + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True + ) + image_id: Mapped[str] = mapped_column(String(64), primary_key=True) + repo_tag: Mapped[str] = mapped_column(String(512), nullable=False, default="") + size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + shared_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + containers: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + +class DockerSwarmNode(Base): + """A Swarm node as seen by a manager host: role / availability / status.""" + __tablename__ = "docker_swarm_nodes" + + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True + ) + node_id: Mapped[str] = mapped_column(String(64), primary_key=True) + hostname: Mapped[str] = mapped_column(String(255), nullable=False, default="") + role: Mapped[str] = mapped_column(String(16), nullable=False, default="worker") + # manager | worker + availability: Mapped[str] = mapped_column(String(16), nullable=False, default="active") + # active | pause | drain + status: Mapped[str] = mapped_column(String(16), nullable=False, default="unknown") + # ready | down | unknown + leader: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) diff --git a/plugins/docker/plugin.yaml b/plugins/docker/plugin.yaml new file mode 100644 index 0000000..1237a62 --- /dev/null +++ b/plugins/docker/plugin.yaml @@ -0,0 +1,16 @@ +name: docker +version: "2.0.0" +description: "Per-host Docker container status + resource usage, collected by the host agent" +author: "Steward" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker" +tags: + - containers + - docker + - infrastructure + +# No config: collection is agent-driven (the host agent reads each host's local +# socket and pushes containers to the host_agent ingest). This plugin is pure +# presentation + storage, so there's no socket path or scrape interval to tune. diff --git a/plugins/docker/retention.py b/plugins/docker/retention.py new file mode 100644 index 0000000..9919f92 --- /dev/null +++ b/plugins/docker/retention.py @@ -0,0 +1,128 @@ +# plugins/docker/retention.py +"""Bound Docker time-series growth: roll up old metrics, prune old rows. + +Published as the "docker.run_retention" capability (see __init__.setup) so the +core cleanup task can drive it WITHOUT importing the docker models (same +opportunistic-coupling pattern as docker.persist_host_samples). Runs inside the +caller's open transaction; never opens or commits its own. + +The scaling concern is docker_metrics: ~2880 rows/container/day at a 30s sample. +We keep raw samples for a short window, then aggregate everything older into +hourly averages (docker_metrics_hourly) and delete the raw rows — so multi-day +history stays cheap to store and query. docker_events is light but unbounded +without a cutoff, so it gets a (longer) window too. +""" +from __future__ import annotations + +from datetime import datetime, timedelta + + +def _hour_floor(dt: datetime) -> datetime: + """Truncate a datetime down to the start of its hour (drops min/sec/µs).""" + return dt.replace(minute=0, second=0, microsecond=0) + + +def _rollup_cutoff(now: datetime, raw_days: int) -> datetime: + """Hour-aligned boundary below which raw metrics get rolled up + deleted. + + Aligning to the hour means we only ever roll up *whole* elapsed hours — a + bucket is never split across the keep/roll boundary, so re-running can't + produce a partial-then-complete duplicate for the same hour. + """ + return _hour_floor(now - timedelta(days=raw_days)) + + +async def run_docker_retention( + session, + *, + events_days: int, + metrics_raw_days: int, + metrics_rollup_days: int, + now: datetime | None = None, +) -> dict: + """Roll up + prune Docker time-series. Returns a counts dict for logging. + + 1. Aggregate docker_metrics older than the (hour-aligned) raw window into + docker_metrics_hourly (avg cpu/mem per container per hour), upserting so a + re-run is idempotent, then delete those raw rows. + 2. Prune rolled-up rows older than the rollup window. + 3. Prune docker_events older than the events window. + """ + from datetime import timezone + + from sqlalchemy import delete, func, select + from sqlalchemy.dialects.postgresql import insert as pg_insert + + from .models import DockerEvent, DockerMetric, DockerMetricHourly + + if now is None: + now = datetime.now(timezone.utc) + + rolled = rolled_rows = events_pruned = rollup_pruned = 0 + + # ── 1. Roll up raw metrics older than the raw window into hourly buckets ── + raw_cutoff = _rollup_cutoff(now, metrics_raw_days) + hour = func.date_trunc("hour", DockerMetric.scraped_at) + agg = ( + select( + DockerMetric.host_id, + DockerMetric.container_name, + hour.label("bucket"), + func.avg(DockerMetric.cpu_pct).label("cpu_pct"), + func.avg(DockerMetric.mem_pct).label("mem_pct"), + func.avg(DockerMetric.mem_usage_bytes).label("mem_usage_bytes"), + func.count().label("sample_count"), + ) + .where(DockerMetric.scraped_at < raw_cutoff) + .group_by(DockerMetric.host_id, DockerMetric.container_name, hour) + ) + for r in (await session.execute(agg)).all(): + stmt = ( + pg_insert(DockerMetricHourly) + .values( + host_id=r.host_id, + container_name=r.container_name, + bucket=r.bucket, + cpu_pct=float(r.cpu_pct or 0.0), + mem_pct=float(r.mem_pct or 0.0), + mem_usage_bytes=int(r.mem_usage_bytes or 0), + sample_count=int(r.sample_count or 0), + ) + .on_conflict_do_update( + constraint="uq_docker_metrics_hourly_bucket", + set_={ + "cpu_pct": float(r.cpu_pct or 0.0), + "mem_pct": float(r.mem_pct or 0.0), + "mem_usage_bytes": int(r.mem_usage_bytes or 0), + "sample_count": int(r.sample_count or 0), + }, + ) + ) + await session.execute(stmt) + rolled += 1 + rolled_rows += int(r.sample_count or 0) + if rolled: + await session.execute( + delete(DockerMetric).where(DockerMetric.scraped_at < raw_cutoff) + ) + + # ── 2. Prune rolled-up rows beyond the rollup window ── + rollup_cutoff = now - timedelta(days=metrics_rollup_days) + res = await session.execute( + delete(DockerMetricHourly).where(DockerMetricHourly.bucket < rollup_cutoff) + ) + rollup_pruned = res.rowcount or 0 + + # ── 3. Prune lifecycle events beyond the events window ── + events_cutoff = now - timedelta(days=events_days) + res = await session.execute( + delete(DockerEvent).where(DockerEvent.at < events_cutoff) + ) + events_pruned = res.rowcount or 0 + + return { + "buckets_rolled": rolled, + "raw_rows_rolled": rolled_rows, + "rollup_pruned": rollup_pruned, + "events_pruned": events_pruned, + } diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py new file mode 100644 index 0000000..0736a05 --- /dev/null +++ b/plugins/docker/routes.py @@ -0,0 +1,565 @@ +# plugins/docker/routes.py +from __future__ import annotations +import json +from datetime import datetime, timedelta, timezone + +from quart import Blueprint, current_app, render_template, request +from sqlalchemy import func, select + +from steward.auth.middleware import require_role +from steward.models.users import UserRole +from steward.models.hosts import Host +from steward.core.time_range import parse_range, DEFAULT_RANGE +from .dedup import dedup_by_container_id +from .swarm_view import build_swarm_services +from .models import ( + DockerContainer, DockerDiskUsage, DockerEvent, DockerImage, DockerMetric, + DockerSwarmNode, DockerSwarmService, +) + +docker_bp = Blueprint("docker", __name__, template_folder="templates") + + +def _human_bytes(n: int | None) -> str: + """Compact binary size string (e.g. '1.4 GiB', '512 MiB', '0 B').""" + if n is None: + return "—" + size = float(n) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if abs(size) < 1024.0 or unit == "TiB": + if unit == "B": + return f"{int(size)} {unit}" + return f"{size:.1f} {unit}" + size /= 1024.0 + return f"{size:.1f} TiB" + + +def _human_uptime(started_at: datetime | None) -> str | None: + """Compact 'how long running' string (e.g. '3d 4h', '5h 12m', '8m').""" + if started_at is None: + return None + if started_at.tzinfo is None: + started_at = started_at.replace(tzinfo=timezone.utc) + secs = int((datetime.now(timezone.utc) - started_at).total_seconds()) + if secs < 0: + return None + d, rem = divmod(secs, 86400) + h, rem = divmod(rem, 3600) + m, _ = divmod(rem, 60) + if d: + return f"{d}d {h}h" + if h: + return f"{h}h {m}m" + return f"{m}m" + + +def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: + if len(values) < 2: + return f'' + mn, mx = min(values), max(values) + if mx == mn: + mx = mn + 1.0 + step = width / (len(values) - 1) + pts = [] + for i, v in enumerate(values): + x = i * step + y = height - (v - mn) / (mx - mn) * (height - 2) - 1 + pts.append(f"{x:.1f},{y:.1f}") + poly = " ".join(pts) + return ( + f'' + f'' + f'' + ) + + +def _bucket(values: list[float], target: int = 40) -> list[float]: + """Bucket-average a series down to ~target points (DB-agnostic, in Python). + + Replaces the old SQL strftime() bucketing, which was SQLite-only and broke + on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of + rows — averaging into a readable point count keeps the sparkline's shape. + """ + n = len(values) + if n <= target: + return values + size = (n + target - 1) // target + out: list[float] = [] + for i in range(0, n, size): + chunk = values[i:i + size] + out.append(sum(chunk) / len(chunk)) + return out + + +async def _container_history(db, host_id: str, name: str, since) -> tuple[list, list]: + """Return (cpu_series, mem_series) sparkline-ready for one host's container.""" + rows = (await db.execute( + select(DockerMetric.cpu_pct, DockerMetric.mem_pct) + .where(DockerMetric.host_id == host_id) + .where(DockerMetric.container_name == name) + .where(DockerMetric.scraped_at >= since) + .order_by(DockerMetric.scraped_at) + )).all() + cpu = _bucket([r.cpu_pct or 0.0 for r in rows]) + mem = _bucket([r.mem_pct or 0.0 for r in rows]) + return cpu, mem + + +@docker_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + current_range = request.args.get("range", DEFAULT_RANGE) + # Show the Swarm link only when a manager has actually reported topology, so + # non-swarm installs aren't offered an always-empty page. + async with current_app.db_sessionmaker() as db: + has_swarm = (await db.execute( + select(DockerSwarmService.host_id).limit(1))).first() is not None + has_disk = (await db.execute( + select(DockerDiskUsage.host_id).limit(1))).first() is not None + return await render_template( + "docker/index.html", + poll_interval=poll_interval, + current_range=current_range, + has_swarm=has_swarm, + has_disk=has_disk, + ) + + +async def _host_map(db, host_ids: set[str]) -> dict[str, Host]: + if not host_ids: + return {} + return { + h.id: h for h in (await db.execute( + select(Host).where(Host.id.in_(host_ids)))).scalars().all() + } + + +@docker_bp.get("/rows") +@require_role(UserRole.viewer) +async def rows(): + """HTMX fragment: swarm services (collapsed, cluster-complete) on top, then + non-swarm containers grouped by host with resource sparklines.""" + since, range_key = parse_range(request.args.get("range")) + + async with current_app.db_sessionmaker() as db: + containers = list((await db.execute( + select(DockerContainer).order_by( + (DockerContainer.status == "running").desc(), + DockerContainer.name, + ) + )).scalars()) + services = list((await db.execute(select(DockerSwarmService))).scalars()) + nodes = list((await db.execute(select(DockerSwarmNode))).scalars()) + hosts = await _host_map( + db, + {c.host_id for c in containers} + | {n.host_id for n in nodes} | {s.host_id for s in services}, + ) + + # Swarm tasks collapse into per-service rows (each replica labelled with + # its host, plus "ghost" replicas for tasks on agent-less nodes, drawn + # from the managers' placement). Non-swarm containers keep the host view. + node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname} + host_name = {h.id: h.name for h in hosts.values()} + swarm_view = build_swarm_services(containers, services, node_hostname, host_name) + + standalone = [c for c in containers if not c.service_name] + groups: dict[str, dict] = {} + for c in standalone: + g = groups.get(c.host_id) + if g is None: + host = hosts.get(c.host_id) + g = groups[c.host_id] = { + "host": host, + "host_id": c.host_id, + "host_name": host.name if host else c.host_id, + "containers": [], "running": 0, "stopped": 0, + } + cpu_hist, mem_hist = await _container_history(db, c.host_id, c.name, since) + g["containers"].append({ + "container": c, + "ports": json.loads(c.ports_json) if c.ports_json else [], + "uptime": _human_uptime(c.started_at) if c.status == "running" else None, + "sparkline_cpu": _sparkline(cpu_hist), + "sparkline_mem": _sparkline(mem_hist), + }) + if c.status == "running": + g["running"] += 1 + else: + g["stopped"] += 1 + + # Sub-group each host's containers by compose project, preserving the + # running-first ordering. Containers with no project fall into an unlabelled + # bucket rendered flat, so plain hosts look unchanged. + for g in groups.values(): + subs: dict[str, dict] = {} + order: list[str] = [] + for item in g["containers"]: + c = item["container"] + label = c.compose_project or None + key = label or "" + if key not in subs: + subs[key] = {"label": label, "containers": []} + order.append(key) + subs[key]["containers"].append(item) + g["subgroups"] = [subs[k] for k in order] + g["grouped"] = any(s["label"] for s in g["subgroups"]) + + host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower()) + swarm_services = swarm_view["services"] + standalone_running = sum(g["running"] for g in host_groups) + ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"]) + return await render_template( + "docker/rows.html", + host_groups=host_groups, + swarm_services=swarm_services, + running=standalone_running + sum(s["running"] for s in swarm_services), + stopped=len(standalone) - standalone_running, + total=len(containers) + ghost_total, + range_key=range_key, + ) + + +@docker_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """HTMX dashboard widget: container status overview, grouped by host.""" + show_stopped = request.args.get("show_stopped", "no") == "yes" + widget_id = request.args.get("wid", "0") + since_24h = datetime.now(timezone.utc) - timedelta(hours=24) + + async with current_app.db_sessionmaker() as db: + all_containers = dedup_by_container_id(list((await db.execute( + select(DockerContainer).order_by( + (DockerContainer.status == "running").desc(), + DockerContainer.name, + ) + )).scalars())) + services = list((await db.execute(select(DockerSwarmService))).scalars()) + nodes = list((await db.execute(select(DockerSwarmNode))).scalars()) + # Recent failures are more actionable than a static "stopped" tally. Count + # DISTINCT container names so the two managers' duplicate die/oom events + # don't double it. + failed_24h = (await db.execute( + select(func.count(func.distinct(DockerEvent.container_name))) + .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since_24h) + )).scalar() or 0 + hosts = await _host_map( + db, + {c.host_id for c in all_containers} + | {n.host_id for n in nodes} | {s.host_id for s in services}, + ) + + # Swarm tasks collapse into per-service rows (each replica tagged with its + # host); non-swarm containers stay grouped by host. + node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname} + host_name = {h.id: h.name for h in hosts.values()} + swarm_services = build_swarm_services(all_containers, services, node_hostname, host_name)["services"] + + standalone = [c for c in all_containers if not c.service_name] + running = [c for c in standalone if c.status == "running"] + display = standalone if show_stopped else running + + # Group for display so multi-host fleets read clearly; single-host stays flat. + groups: dict[str, dict] = {} + for c in display: + g = groups.setdefault(c.host_id, { + "host_id": c.host_id, + "host_name": host_name.get(c.host_id, c.host_id), + "containers": [], + }) + g["containers"].append(c) + host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower()) + + return await render_template( + "docker/widget.html", + host_groups=host_groups, + multi_host=len(host_groups) > 1, + swarm_services=swarm_services, + running_count=len(running) + sum(s["running"] for s in swarm_services), + failed_24h=failed_24h, + total_count=len(all_containers) + len(swarm_services), + show_stopped=show_stopped, + widget_id=widget_id, + ) + + +@docker_bp.get("/widget/resources") +@require_role(UserRole.viewer) +async def widget_resources(): + """HTMX dashboard widget: CPU + memory usage for the busiest containers.""" + limit = max(1, min(20, int(request.args.get("limit", 10) or 10))) + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + # Dedup before the limit, else swarm tasks reported by both managers can + # fill the list with duplicates of the same container. + containers = dedup_by_container_id(list((await db.execute( + select(DockerContainer) + .where(DockerContainer.status == "running") + .order_by(DockerContainer.cpu_pct.desc().nullslast()) + )).scalars()))[:limit] + hosts = await _host_map(db, {c.host_id for c in containers}) + + rows_data = [ + {"c": c, "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id} + for c in containers + ] + return await render_template( + "docker/widget_resources.html", + rows=rows_data, + multi_host=len({c.host_id for c in containers}) > 1, + widget_id=widget_id, + ) + + +@docker_bp.get("/host/") +@require_role(UserRole.viewer) +async def host_panel(host_id: str): + """Per-host Docker fragment embedded on the core Hosts hub via HTMX. + + Renders nothing when the host reports no containers, so hosts without Docker + don't carry an empty card. + """ + async with current_app.db_sessionmaker() as db: + containers = list((await db.execute( + select(DockerContainer) + .where(DockerContainer.host_id == host_id) + .order_by( + (DockerContainer.status == "running").desc(), + DockerContainer.name, + ) + )).scalars()) + + if not containers: + return "" + running = sum(1 for c in containers if c.status == "running") + return await render_template( + "docker/host_panel.html", + containers=containers, + running=running, + stopped=len(containers) - running, + ) + + +# Glyph + colour per lifecycle event, so the timeline reads at a glance. +_EVENT_STYLE = { + "start": ("▲", "var(--green)"), + "stop": ("■", "var(--text-muted)"), + "die": ("✕", "var(--red)"), + "oom": ("☠", "var(--red)"), + "health_change": ("◐", "var(--orange)"), +} + + +@docker_bp.get("/container//") +@require_role(UserRole.viewer) +async def container_detail(host_id: str, name: str): + """Full detail page for one container: facts, resource history, lifecycle.""" + async with current_app.db_sessionmaker() as db: + c = await db.get(DockerContainer, (host_id, name)) + host = await db.get(Host, host_id) + events = [] + if c is not None: + events = list((await db.execute( + select(DockerEvent) + .where(DockerEvent.host_id == host_id) + .where(DockerEvent.container_name == name) + .order_by(DockerEvent.at.desc()) + .limit(50) + )).scalars()) + + if c is None: + return await render_template( + "docker/container_detail.html", + container=None, host_id=host_id, name=name, + ), 404 + + return await render_template( + "docker/container_detail.html", + container=c, + host=host, + host_id=host_id, + name=name, + ports=json.loads(c.ports_json) if c.ports_json else [], + uptime=_human_uptime(c.started_at) if c.status == "running" else None, + net_rx=_human_bytes(c.net_rx_bytes), net_tx=_human_bytes(c.net_tx_bytes), + blk_read=_human_bytes(c.blk_read_bytes), blk_write=_human_bytes(c.blk_write_bytes), + events=[ + {"event": e.event, "detail": e.detail, "at": e.at, + "glyph": _EVENT_STYLE.get(e.event, ("•", "var(--text-muted)"))[0], + "colour": _EVENT_STYLE.get(e.event, ("•", "var(--text-muted)"))[1]} + for e in events + ], + current_range=request.args.get("range", DEFAULT_RANGE), + ) + + +@docker_bp.get("/swarm") +@require_role(UserRole.viewer) +async def swarm(): + """Swarm topology page: services with replica health, nodes, placement. + + Swarm services/nodes are cluster-global — every manager's API returns the + same list — but each manager reports them independently (rows keyed by + host_id). So we group the reporting managers into swarms (managers that share + any node_id are the same cluster) and dedup within each: one row per service + (by name) and per node (by node_id), keeping the freshest. Without this, two + managers in one cluster list every service/node twice. + """ + async with current_app.db_sessionmaker() as db: + services = list((await db.execute( + select(DockerSwarmService).order_by(DockerSwarmService.service_name) + )).scalars()) + nodes = list((await db.execute( + select(DockerSwarmNode).order_by( + DockerSwarmNode.role, DockerSwarmNode.hostname) + )).scalars()) + hosts = await _host_map(db, {s.host_id for s in services} | {n.host_id for n in nodes}) + + # ── Group reporting managers into swarms by shared node membership ────────── + manager_ids = {s.host_id for s in services} | {n.host_id for n in nodes} + nodes_by_mgr: dict[str, set[str]] = {} + for n in nodes: + nodes_by_mgr.setdefault(n.host_id, set()).add(n.node_id) + + parent = {m: m for m in manager_ids} + + def _find(x: str) -> str: + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + mgrs = list(manager_ids) + for i in range(len(mgrs)): + for j in range(i + 1, len(mgrs)): + a, b = mgrs[i], mgrs[j] + if nodes_by_mgr.get(a) and nodes_by_mgr.get(b) and (nodes_by_mgr[a] & nodes_by_mgr[b]): + parent[_find(a)] = _find(b) + + swarm_members: dict[str, set[str]] = {} + for m in manager_ids: + swarm_members.setdefault(_find(m), set()).add(m) + + # ── Dedup services (by name) + nodes (by node_id) within each swarm ───────── + swarms = [] + for members in swarm_members.values(): + svc_by_name: dict[str, DockerSwarmService] = {} + for s in services: + if s.host_id in members and ( + s.service_name not in svc_by_name + or s.updated_at > svc_by_name[s.service_name].updated_at + ): + svc_by_name[s.service_name] = s + node_by_id: dict[str, DockerSwarmNode] = {} + for n in nodes: + if n.host_id in members and ( + n.node_id not in node_by_id + or n.updated_at > node_by_id[n.node_id].updated_at + ): + node_by_id[n.node_id] = n + + node_names = {nid: (nrow.hostname or nid) for nid, nrow in node_by_id.items()} + svc_dicts = [] + for s in sorted(svc_by_name.values(), key=lambda s: s.service_name): + placement = [] + for p in (json.loads(s.placement_json) if s.placement_json else []): + nid = p.get("node_id", "") + placement.append({ + "node": node_names.get(nid, nid[:12] or "?"), + "running": p.get("running", 0), + }) + svc_dicts.append({ + "name": s.service_name, "mode": s.mode, + "running": s.running, "desired": s.desired, "image": s.image, + "healthy": s.running >= s.desired and s.desired > 0, + "degraded": 0 < s.running < s.desired, + "down": s.running == 0 and s.desired > 0, + "placement": placement, + }) + node_rows = sorted( + node_by_id.values(), key=lambda n: (n.role, (n.hostname or n.node_id).lower()) + ) + manager_names = sorted(hosts[m].name if m in hosts else m for m in members) + swarms.append({ + "managers": manager_names, + "services": svc_dicts, + "nodes": node_rows, + }) + + swarms.sort(key=lambda g: g["managers"][0].lower() if g["managers"] else "") + return await render_template("docker/swarm.html", swarms=swarms) + + +@docker_bp.get("/disk") +@require_role(UserRole.viewer) +async def disk(): + """Image/disk usage page: reclaimable space, per-image sizes, stopped count. + + Read-only — prune actions are deferred to the cleanup-actions milestone, so + this surfaces the numbers and notes where reclaim lives. + """ + async with current_app.db_sessionmaker() as db: + summaries = list((await db.execute(select(DockerDiskUsage))).scalars()) + images = list((await db.execute( + select(DockerImage).order_by(DockerImage.size.desc()) + )).scalars()) + # Stopped-container count per host, surfaced alongside reclaimable space. + stopped_rows = (await db.execute( + select(DockerContainer.host_id) + .where(DockerContainer.status != "running") + )).scalars() + stopped_by_host: dict[str, int] = {} + for hid in stopped_rows: + stopped_by_host[hid] = stopped_by_host.get(hid, 0) + 1 + hosts = await _host_map(db, {s.host_id for s in summaries}) + + images_by_host: dict[str, list] = {} + for im in images: + images_by_host.setdefault(im.host_id, []).append({ + "repo_tag": im.repo_tag, "size": _human_bytes(im.size), + "shared": _human_bytes(im.shared_size), "containers": im.containers, + "reclaimable": im.containers == 0, + }) + + host_groups = [ + { + "host_id": s.host_id, + "host_name": hosts[s.host_id].name if s.host_id in hosts else s.host_id, + "summary": { + "images_total": s.images_total, "images_active": s.images_active, + "images_size": _human_bytes(s.images_size), + "images_reclaimable": _human_bytes(s.images_reclaimable), + "layers_size": _human_bytes(s.layers_size), + "containers_count": s.containers_count, + "containers_size": _human_bytes(s.containers_size), + "volumes_count": s.volumes_count, + "volumes_size": _human_bytes(s.volumes_size), + "build_cache_size": _human_bytes(s.build_cache_size), + }, + "stopped": stopped_by_host.get(s.host_id, 0), + "images": images_by_host.get(s.host_id, []), + } + for s in summaries + ] + host_groups.sort(key=lambda g: g["host_name"].lower()) + return await render_template("docker/disk.html", host_groups=host_groups) + + +@docker_bp.get("/container///history") +@require_role(UserRole.viewer) +async def container_history(host_id: str, name: str): + """HTMX fragment: CPU + memory sparklines for the selected time range.""" + since, range_key = parse_range(request.args.get("range")) + async with current_app.db_sessionmaker() as db: + cpu_hist, mem_hist = await _container_history(db, host_id, name, since) + return await render_template( + "docker/_container_history.html", + sparkline_cpu=_sparkline(cpu_hist, width=320, height=48), + sparkline_mem=_sparkline(mem_hist, width=320, height=48), + have_data=len(cpu_hist) >= 2, + range_key=range_key, + ) diff --git a/plugins/docker/swarm_view.py b/plugins/docker/swarm_view.py new file mode 100644 index 0000000..cfffa80 --- /dev/null +++ b/plugins/docker/swarm_view.py @@ -0,0 +1,140 @@ +"""Model-free builder for the swarm-aware container view. + +Merges two sources into one service-grouped, cluster-complete picture: + • docker_containers — real container rows, collected LOCAL-per-node, so each + carries the host (= node) it runs on plus its swarm service/node labels. + • DockerSwarmService.placement_json — the managers' cluster-wide task→node + counts, which include tasks on nodes that run no Steward agent (and so have + no container row of their own). + +For every swarm service we list the real replicas we have detail for, then add +"ghost" replicas for the remaining placement count on each node (agent-less +nodes). Non-swarm containers pass through grouped by host, unchanged. + +Kept model-free (no ORM imports) so it's unit-testable and safe to import in +tests without the plugin-loader "table already defined" gotcha. +""" +from __future__ import annotations + +import json + + +def _service_state(running: int, desired: int) -> str: + if desired > 0 and running >= desired: + return "healthy" + if running > 0: + return "degraded" + return "down" + + +def _placement(service) -> list[dict]: + raw = getattr(service, "placement_json", None) or "[]" + try: + data = json.loads(raw) + except (ValueError, TypeError): + return [] + return [p for p in data if isinstance(p, dict)] + + +def build_swarm_services(containers, services, node_hostname: dict, host_name: dict) -> dict: + """Return {"services": [...], "standalone": [...]}. + + services[i] = {name, mode, desired, running, state, replicas:[...]} + real replica = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct, + health, restart_count} + ghost replica = {ghost:True, host, count} # placement with no local row + standalone[i] = {host_id, host, containers:[...]} # non-swarm, by host + """ + containers = list(containers) + swarm_cs = [c for c in containers if getattr(c, "service_name", None)] + standalone_cs = [c for c in containers if not getattr(c, "service_name", None)] + + # Dedup service rows by name (every manager reports the same cluster-global + # set); keep the first — placement is identical across managers. + svc_by_name: dict[str, object] = {} + for s in services: + name = getattr(s, "service_name", None) + if name and name not in svc_by_name: + svc_by_name[name] = s + + # Real containers grouped by service name. + cs_by_service: dict[str, list] = {} + for c in swarm_cs: + cs_by_service.setdefault(c.service_name, []).append(c) + + def _host_of(c): + return (host_name.get(getattr(c, "host_id", None)) + or node_hostname.get(getattr(c, "node_id", None)) + or getattr(c, "host_id", None) or "?") + + out_services = [] + for name in sorted(set(svc_by_name) | set(cs_by_service)): + svc = svc_by_name.get(name) + reals = cs_by_service.get(name, []) + + replicas = [] + for c in sorted(reals, key=lambda c: (_host_of(c).lower(), c.name)): + replicas.append({ + "ghost": False, + "host": _host_of(c), + "host_id": getattr(c, "host_id", None), + "name": c.name, + "status": getattr(c, "status", "unknown"), + "cpu_pct": getattr(c, "cpu_pct", None), + "mem_pct": getattr(c, "mem_pct", None), + "health": getattr(c, "health", None), + "restart_count": getattr(c, "restart_count", 0) or 0, + }) + + # Real running replicas per node, to subtract from placement. + real_running_by_node: dict[str, int] = {} + for c in reals: + if (getattr(c, "status", "") or "").lower() == "running": + nid = getattr(c, "node_id", None) + if nid: + real_running_by_node[nid] = real_running_by_node.get(nid, 0) + 1 + + ghosts = [] + for p in _placement(svc) if svc is not None else []: + nid = p.get("node_id", "") + placed = int(p.get("running", 0) or 0) + ghost = placed - real_running_by_node.get(nid, 0) + if ghost > 0: + ghosts.append({ + "ghost": True, + "host": node_hostname.get(nid) or (nid[:12] if nid else "?"), + "count": ghost, + }) + ghosts.sort(key=lambda g: g["host"].lower()) + replicas.extend(ghosts) + + running = getattr(svc, "running", None) if svc is not None else None + desired = getattr(svc, "desired", None) if svc is not None else None + # No service row (manager didn't report it): infer from what we can see. + if running is None: + running = sum(1 for r in replicas + if (not r["ghost"] and (r["status"] or "").lower() == "running")) + if desired is None: + desired = len(replicas) + + out_services.append({ + "name": name, + "mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated", + "running": running, + "desired": desired, + "state": _service_state(int(running or 0), int(desired or 0)), + "replicas": replicas, + }) + + # Non-swarm containers grouped by host, running-first order preserved. + groups: dict[str, dict] = {} + for c in standalone_cs: + hid = getattr(c, "host_id", None) + g = groups.get(hid) + if g is None: + g = groups[hid] = {"host_id": hid, "host": host_name.get(hid) or hid or "?", + "containers": []} + g["containers"].append(c) + standalone = sorted(groups.values(), key=lambda g: g["host"].lower()) + + return {"services": out_services, "standalone": standalone} diff --git a/plugins/docker/templates/docker/_container_history.html b/plugins/docker/templates/docker/_container_history.html new file mode 100644 index 0000000..0a9b780 --- /dev/null +++ b/plugins/docker/templates/docker/_container_history.html @@ -0,0 +1,17 @@ +{# docker/_container_history.html — CPU/mem sparklines for the selected range #} +{% if have_data %} +
+
+
CPU %
+ {{ sparkline_cpu | safe }} +
+
+
Memory %
+ {{ sparkline_mem | safe }} +
+
+{% else %} +
+ Not enough samples in this range yet — history fills in as the agent reports. +
+{% endif %} diff --git a/plugins/docker/templates/docker/container_detail.html b/plugins/docker/templates/docker/container_detail.html new file mode 100644 index 0000000..bf4d76e --- /dev/null +++ b/plugins/docker/templates/docker/container_detail.html @@ -0,0 +1,110 @@ +{# docker/container_detail.html — full detail page for one container #} +{% extends "base.html" %} +{% from "_macros.html" import crumbs %} +{% block title %}{{ name }} — Docker — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %} +{% block content %} + +{% if container is none %} +
+
Container not found
+
+ No container named {{ name }} is currently reported for this host. + It may have been removed, or the host agent hasn't reported recently. +
+
+{% else %} + +{# ── Header ──────────────────────────────────────────────────────────────── #} +
+ +

{{ container.name }}

+ {% if container.health == 'healthy' %}healthy + {% elif container.health == 'unhealthy' %}unhealthy + {% elif container.health == 'starting' %}starting{% endif %} +
+
+ {{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %} + {% if host %} · on {{ host.name }}{% endif %} +
+ +{# ── Facts grid ──────────────────────────────────────────────────────────── #} +{% macro fact(label, value, colour="") %} +
+
{{ label }}
+
{{ value }}
+
+{% endmacro %} + +
+ {{ fact("CPU", "%.1f%%" | format(container.cpu_pct) if container.cpu_pct is not none else "—") }} + {{ fact("Memory", "%.1f%%" | format(container.mem_pct) if container.mem_pct is not none else "—") }} + {{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }} + {{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—", + "var(--red)" if (container.exit_code is not none and container.exit_code != 0) else "") }} + {{ fact("Net in", net_rx) }} + {{ fact("Net out", net_tx) }} + {{ fact("Block read", blk_read) }} + {{ fact("Block write", blk_write) }} +
+ +{# ── Image / placement ───────────────────────────────────────────────────── #} +
+
+ Image + {{ container.image or "—" }} + {% if container.compose_project %} + Compose project{{ container.compose_project }} + {% endif %} + {% if container.service_name %} + Swarm service{{ container.service_name }} + {% endif %} + {% if container.node_id %} + Node{{ container.node_id }} + {% endif %} + {% if ports %} + Ports + {% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %} + {% endif %} +
+
+ +{# ── Resource history (range-toggled) ────────────────────────────────────── #} +
+
+

Resource history

+ {% include "_time_range.html" %} +
+
+
Loading…
+
+
+ +{# ── Lifecycle timeline ──────────────────────────────────────────────────── #} +
+

Lifecycle

+ {% if events %} +
+ {% for e in events %} +
+ {{ e.glyph }} + {{ e.event }} + {{ e.detail or "" }} + {{ e.at.strftime("%Y-%m-%d %H:%M") }} +
+ {% endfor %} +
+ {% else %} +
+ No lifecycle events recorded yet. Start/stop/health changes appear here as the + agent reports them over time. +
+ {% endif %} +
+ +{% endif %} +{% endblock %} diff --git a/plugins/docker/templates/docker/disk.html b/plugins/docker/templates/docker/disk.html new file mode 100644 index 0000000..22a0e16 --- /dev/null +++ b/plugins/docker/templates/docker/disk.html @@ -0,0 +1,83 @@ +{# docker/disk.html — image/disk usage: reclaimable space, per-image sizes #} +{% extends "base.html" %} +{% from "_macros.html" import crumbs %} +{% block title %}Disk — Docker — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Image & disk usage", "")]) }}{% endblock %} +{% block content %} + +

Image & disk usage

+

+ Reclaimable = space held by images no container references. Cleanup actions + (prune) arrive in a later release — these are read-only figures for now. +

+ +{% if host_groups %} +{% for g in host_groups %} +
+
{{ g.host_name }}
+ + {# ── Summary stats ────────────────────────────────────────────────────── #} +
+
+
Reclaimable
+ {{ g.summary.images_reclaimable }} +
+
+
Images
+ {{ g.summary.images_size }} +
{{ g.summary.images_active }}/{{ g.summary.images_total }} in use
+
+
+
Stopped
+ {{ g.stopped }} +
+
+
Writable layers
+ {{ g.summary.containers_size }} +
{{ g.summary.containers_count }} containers
+
+
+
Volumes
+ {{ g.summary.volumes_size }} +
{{ g.summary.volumes_count }} volumes
+
+
+
Build cache
+ {{ g.summary.build_cache_size }} +
+
+ + {# ── Per-image table ──────────────────────────────────────────────────── #} +
+ + + + + + {% for im in g.images %} + + + + + + + {% else %} + + {% endfor %} + +
ImageSizeSharedContainers
+ {{ im.repo_tag }} + {% if im.reclaimable %}reclaimable{% endif %} + {{ im.size }}{{ im.shared }}{{ im.containers }}
No images reported.
+
+
+{% endfor %} +{% else %} +
+
+ No disk usage reported yet. The host agent reports image/disk usage from + docker system df on hosts running Docker. +
+
+{% endif %} +{% endblock %} diff --git a/plugins/docker/templates/docker/host_panel.html b/plugins/docker/templates/docker/host_panel.html new file mode 100644 index 0000000..942aef4 --- /dev/null +++ b/plugins/docker/templates/docker/host_panel.html @@ -0,0 +1,22 @@ +{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub #} +
+
+

Docker

+ + {{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %} + All → + +
+
+ {% for c in containers %} +
+ + {{ c.name }} + + {% if c.cpu_pct is not none %}{{ "%.1f" | format(c.cpu_pct) }}%{% endif %} + {% if c.mem_pct is not none %} · {{ "%.1f" | format(c.mem_pct) }}%{% endif %} + +
+ {% endfor %} +
+
diff --git a/plugins/docker/templates/docker/index.html b/plugins/docker/templates/docker/index.html new file mode 100644 index 0000000..950dd06 --- /dev/null +++ b/plugins/docker/templates/docker/index.html @@ -0,0 +1,24 @@ +{% extends "base.html" %} +{% block title %}Docker — Steward{% endblock %} +{% block content %} +
+
+

Docker

+ {% if has_swarm %} + Swarm → + {% endif %} + {% if has_disk %} + Disk → + {% endif %} +
+ {% include "_time_range.html" %} +
+ +
+
Loading...
+
+{% endblock %} diff --git a/plugins/docker/templates/docker/rows.html b/plugins/docker/templates/docker/rows.html new file mode 100644 index 0000000..1d04717 --- /dev/null +++ b/plugins/docker/templates/docker/rows.html @@ -0,0 +1,174 @@ +{# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #} + +{# ── Summary strip ─────────────────────────────────────────────────────────── #} +
+
+
Running
+ {{ running }} +
+
+
Stopped
+ {{ stopped }} +
+
+
Total
+ {{ total }} +
+ {% if swarm_services %} +
+
Services
+ {{ swarm_services | length }} +
+ {% endif %} +
+
Hosts
+ {{ host_groups | length }} +
+
+ +{# ── Swarm services: collapsed per-service, each replica labelled with its host. + Ghost replicas (tasks on nodes with no Steward agent) come from the managers' + placement data so the picture is cluster-complete. ──────────────────────── #} +{% if swarm_services %} +
+
+ Swarm services + Topology → +
+
+ {% for s in swarm_services %} +
+
+ + {{ s.name }} + {{ s.mode }} + + {{ s.running }}/{{ s.desired }} running + +
+
+ {% for r in s.replicas %} + {% if r.ghost %} +
+ + {{ r.host }} + {{ r.count }} · no agent +
+ {% else %} +
+ + {{ r.host }} + {% if r.cpu_pct is not none %}{{ "%.0f" | format(r.cpu_pct) }}%{% endif %} +
+ {% endif %} + {% endfor %} +
+
+ {% endfor %} +
+
+{% endif %} + +{# ── Container tables, one per host ───────────────────────────────────────── #} +{% if host_groups %} +{% for g in host_groups %} +
+
+ {% if g.host %} + {{ g.host.name }} + {% else %} + {{ g.host_name }} + {% endif %} + + {{ g.running }} running{% if g.stopped %} · {{ g.stopped }} stopped{% endif %} + +
+ + + + + + + + + + + + + + {% for sub in g.subgroups %} + {% if g.grouped and sub.label %} + + {% endif %} + {% for item in sub.containers %} + {% set c = item.container %} + + + + + + + + + + {% endfor %} + {% endfor %} + +
ContainerImagePortsCPU %CPU historyMem %Mem history
{{ sub.label }}
+
+ +
+
+ {{ c.name }} + {% if c.health == 'healthy' %}● + {% elif c.health == 'unhealthy' %}● + {% elif c.health == 'starting' %}◐{% endif %} +
+
+ {{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %} + {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %} + · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %} + {% endif %} + {% if c.restart_count %} · ⟳{{ c.restart_count }}{% endif %} +
+ {% if c.service_name or c.compose_project %} +
+ {{ c.service_name or c.compose_project }} +
+ {% endif %} +
+
+ 		+ {{ c.image }} + + {% for p in item.ports %} +
{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}
+ {% endfor %} + 		+ {% if c.cpu_pct is not none %} + + {{ "%.1f" | format(c.cpu_pct) }}% + + {% else %} + — + {% endif %} + {{ item.sparkline_cpu | safe }} + {% if c.mem_pct is not none %} + + {{ "%.1f" | format(c.mem_pct) }}% + + {% else %} + — + {% endif %} + {{ item.sparkline_mem | safe }}
+
+{% endfor %} +{% elif not swarm_services %} +
+
+ No containers reported yet. Containers are collected by the Steward host agent — + deploy the agent to a host running Docker (Hosts → the host → agent panel) and + its containers will appear here under that host. +
+
+{% endif %} diff --git a/plugins/docker/templates/docker/swarm.html b/plugins/docker/templates/docker/swarm.html new file mode 100644 index 0000000..7fdbf44 --- /dev/null +++ b/plugins/docker/templates/docker/swarm.html @@ -0,0 +1,87 @@ +{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement #} +{% extends "base.html" %} +{% from "_macros.html" import crumbs %} +{% block title %}Swarm — Docker — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Swarm", "")]) }}{% endblock %} +{% block content %} + +

Swarm

+ +{% if swarms %} +{% for g in swarms %} +
+
+ Swarm + + reported by {{ g.managers | length }} manager{{ 's' if g.managers | length != 1 }} · {{ g.managers | join(", ") }} + +
+ + {# ── Services ─────────────────────────────────────────────────────────── #} +
+ + + + + + + + {% for s in g.services %} + + + + + + + + {% else %} + + {% endfor %} + +
ServiceModeReplicasImagePlacement
{{ s.name }}{{ s.mode }} + + {{ s.running }}/{{ s.desired }} + + {{ s.image or "—" }} + {% for p in s.placement %}{{ p.node }} ({{ p.running }}){% if not loop.last %}, {% endif %}{% else %}—{% endfor %} +
No services in this swarm.
+
+ + {# ── Nodes ────────────────────────────────────────────────────────────── #} +
+ + + + + + {% for n in g.nodes %} + + + + + + + {% else %} + + {% endfor %} + +
NodeRoleAvailabilityStatus
+ {{ n.hostname or n.node_id }} + {% if n.leader %}leader{% endif %} + {{ n.role }}{{ n.availability }} + + {{ n.status }} +
No nodes reported.
+
+
+{% endfor %} +{% else %} +
+
+ No Swarm topology reported. A Steward host agent running on a Swarm + manager reports services, nodes, and task placement here — + workers and non-swarm hosts contribute only their local containers. +
+
+{% endif %} +{% endblock %} diff --git a/plugins/docker/templates/docker/widget.html b/plugins/docker/templates/docker/widget.html new file mode 100644 index 0000000..ad86814 --- /dev/null +++ b/plugins/docker/templates/docker/widget.html @@ -0,0 +1,67 @@ +{# docker/widget.html — dashboard widget: container status overview, by host #} +{% if total_count == 0 %} +

No containers reported yet.

+{% else %} + +
+
+ {{ running_count }} + running +
+
+ {{ failed_24h }} + failed (24h) +
+
+ +{# Swarm services — collapsed, each with its replicas' host chips (dashed = a + node with no Steward agent, count-only from placement). #} +{% if swarm_services %} +
Swarm services
+
+ {% for s in swarm_services %} +
+
+ + {{ s.name }} + {{ s.running }}/{{ s.desired }} +
+
+ {% for r in s.replicas %} + {% if r.ghost %} + {{ r.host }} ×{{ r.count }} + {% else %} + {{ r.host }} + {% endif %} + {% endfor %} +
+
+ {% endfor %} +
+{% endif %} + +{% for g in host_groups %} +{% if multi_host %} +
{{ g.host_name }}
+{% endif %} +
+ {% for c in g.containers %} +
+ + + {{ c.name }}{% if c.health == 'unhealthy' %} ●{% elif c.health == 'healthy' %} ●{% endif %}{% if c.restart_count %} ⟳{{ c.restart_count }}{% endif %} + + {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %} + exit {{ c.exit_code }} + {% endif %} + {% if c.cpu_pct is not none %} + + {{ "%.1f" | format(c.cpu_pct) }}% + + {% endif %} +
+ {% endfor %} +
+{% endfor %} + +{% endif %} diff --git a/plugins/docker/templates/docker/widget_resources.html b/plugins/docker/templates/docker/widget_resources.html new file mode 100644 index 0000000..c4605fa --- /dev/null +++ b/plugins/docker/templates/docker/widget_resources.html @@ -0,0 +1,29 @@ +{# docker/widget_resources.html — dashboard widget: the busiest running containers + by CPU, each with labeled CPU + memory utilisation bars. #} +{% if not rows %} +
No running containers.
+{% else %} +{% macro bar(label, pct, warn, crit) %} +
+ {{ label }} +
+
+
+ {{ "%.1f" | format(pct) }}% +
+{% endmacro %} +
+ {% for r in rows %} + {% set c = r.c %} +
+
+ {{ c.name }}{% if multi_host %} · {{ r.host_name }}{% endif %} +
+ {% if c.cpu_pct is not none %}{{ bar("CPU", c.cpu_pct, 50, 80) }}{% endif %} + {% if c.mem_pct is not none %}{{ bar("MEM", c.mem_pct, 70, 90) }}{% endif %} +
+ {% endfor %} +
+{% endif %} diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py new file mode 100644 index 0000000..6a7b229 --- /dev/null +++ b/plugins/host_agent/__init__.py @@ -0,0 +1,23 @@ +# plugins/host_agent/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + + +def get_scheduled_tasks() -> list: + from .scheduler import make_task + return [make_task(_app)] + + +def get_blueprint(): + from .routes import host_agent_bp + return host_agent_bp diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py new file mode 100644 index 0000000..98a237e --- /dev/null +++ b/plugins/host_agent/agent.py @@ -0,0 +1,1088 @@ +# plugins/host_agent/agent.py +"""Steward host agent — pushes resource metrics to a Steward instance. + +Python 3.8+ stdlib only. Target ~300 lines. Served to targets at +GET /plugins/host_agent/agent.py. +""" +from __future__ import annotations + +import json +import os +import re +import shutil +import signal +import socket +import sys +import time +import urllib.error +import urllib.request +from collections import deque +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone + +AGENT_VERSION = "1.6.0" + +# Default path to the local Docker Engine socket. Overridable via the +# `docker_socket` config key; collection is silently skipped if it's absent or +# unreadable, so this never needs setting on a Docker-less host (zero-config). +DEFAULT_DOCKER_SOCKET = "/var/run/docker.sock" + + +class ConfigError(Exception): + pass + + +REQUIRED_KEYS = ("url", "token") +INT_KEYS = ("interval_seconds",) +LIST_KEYS = ("mounts",) + + +def read_config(path: str) -> dict: + """Parse a flat `key = value` config file.""" + cfg: dict = {} + try: + with open(path, "r", encoding="utf-8") as f: + for lineno, raw in enumerate(f, 1): + line = raw.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + raise ConfigError(f"{path}:{lineno}: expected 'key = value'") + key, _, value = line.partition("=") + key = key.strip() + value = value.strip() + if key in INT_KEYS: + try: + cfg[key] = int(value) + except ValueError: + raise ConfigError(f"{path}:{lineno}: {key} must be int") + elif key in LIST_KEYS: + cfg[key] = [v.strip() for v in value.split(",") if v.strip()] + else: + cfg[key] = value + except FileNotFoundError: + raise ConfigError(f"{path}: not found") + + missing = [k for k in REQUIRED_KEYS if k not in cfg] + if missing: + raise ConfigError(f"{path}: missing required key(s): {', '.join(missing)}") + + cfg.setdefault("interval_seconds", 30) + cfg.setdefault("docker_socket", DEFAULT_DOCKER_SOCKET) + return cfg + + +# ─── collectors ────────────────────────────────────────────────────────────── + +STAT_PATH = "/proc/stat" +MEMINFO_PATH = "/proc/meminfo" +LOADAVG_PATH = "/proc/loadavg" +UPTIME_PATH = "/proc/uptime" +OS_RELEASE_PATH = "/etc/os-release" +NETDEV_PATH = "/proc/net/dev" +DISKSTATS_PATH = "/proc/diskstats" +HWMON_DIR = "/sys/class/hwmon" +PSI_DIR = "/proc/pressure" + +# Partition names to drop from disk I/O (we report whole-disk throughput only). +_PARTITION_RE = re.compile( + r"^(?:sd[a-z]+\d+|vd[a-z]+\d+|hd[a-z]+\d+|nvme\d+n\d+p\d+|mmcblk\d+p\d+)$" +) + + +def _read_file(path: str) -> str: + with open(path, "r", encoding="utf-8") as f: + return f.read() + + +def _parse_cpu_lines(stat_text: str) -> dict: + """Return {name: (total_jiffies, idle_jiffies)} for every cpu line. + + 'cpu' is the aggregate; 'cpu0', 'cpu1', … are per-core. + """ + out: dict = {} + for line in stat_text.splitlines(): + if not line.startswith("cpu"): + continue + parts = line.split() + try: + fields = [int(x) for x in parts[1:]] + except ValueError: + continue + if len(fields) < 4: + continue + idle = fields[3] + (fields[4] if len(fields) > 4 else 0) # idle + iowait + out[parts[0]] = (sum(fields), idle) + return out + + +def collect_cpu(sample_window: float = 0.2) -> tuple[float, list]: + """Return (aggregate_pct, [per_core_pct, …]) over sample_window seconds.""" + a = _parse_cpu_lines(_read_file(STAT_PATH)) + time.sleep(sample_window) + b = _parse_cpu_lines(_read_file(STAT_PATH)) + + def _pct(name: str): + if name not in a or name not in b: + return None + dt = b[name][0] - a[name][0] + di = b[name][1] - a[name][1] + if dt <= 0: + return 0.0 + return round(100.0 * (dt - di) / dt, 2) + + agg = _pct("cpu") or 0.0 + cores: list = [] + i = 0 + while f"cpu{i}" in b: + cores.append(_pct(f"cpu{i}")) + i += 1 + return agg, cores + + +def collect_memory() -> dict: + info: dict[str, int] = {} + for line in _read_file(MEMINFO_PATH).splitlines(): + if ":" not in line: + continue + key, _, rest = line.partition(":") + parts = rest.strip().split() + if not parts: + continue + try: + value_kb = int(parts[0]) + except ValueError: + continue + info[key] = value_kb * 1024 # bytes + total = info.get("MemTotal", 0) + available = info.get("MemAvailable", 0) + swap_total = info.get("SwapTotal", 0) + swap_free = info.get("SwapFree", 0) + return { + "total_bytes": total, + "used_bytes": max(total - available, 0), + "available_bytes": available, + "swap_used_bytes": max(swap_total - swap_free, 0), + "cached_bytes": info.get("Cached", 0), + "buffers_bytes": info.get("Buffers", 0), + } + + +def collect_storage(mounts: list[str]) -> list[dict]: + out: list[dict] = [] + for m in mounts: + try: + u = shutil.disk_usage(m) + except (FileNotFoundError, PermissionError): + continue + out.append({"mount": m, "total_bytes": u.total, "used_bytes": u.used}) + return out + + +def collect_load() -> dict: + parts = _read_file(LOADAVG_PATH).split() + return {"1m": float(parts[0]), "5m": float(parts[1]), "15m": float(parts[2])} + + +def collect_uptime() -> int: + return int(float(_read_file(UPTIME_PATH).split()[0])) + + +def detect_primary_ip() -> str | None: + """Best-effort primary non-loopback IPv4 of this host. + + connect() on a SOCK_DGRAM socket only sets the default peer and selects the + outbound interface — no packets are sent — so getsockname() reveals the + source IP the kernel would use to reach the internet. This is the host's own + view of its address, so it survives reverse proxies / NAT (unlike the + server reading request.remote_addr). Returns None on any error (offline / no + route) and skips loopback. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect(("8.8.8.8", 80)) + ip = s.getsockname()[0] + except OSError: + return None + finally: + s.close() + if not ip or ip.startswith("127."): + return None + return ip + + +def collect_metadata() -> dict: + u = os.uname() + distro = "unknown" + try: + for line in _read_file(OS_RELEASE_PATH).splitlines(): + if line.startswith("PRETTY_NAME="): + distro = line.split("=", 1)[1].strip().strip('"') + break + except (OSError, FileNotFoundError): + pass + return { + "kernel": u.release, + "distro": distro, + "arch": u.machine, + "host_ip": detect_primary_ip(), + } + + +def default_mounts() -> list[str]: + """Return real mount points from /proc/mounts, skipping pseudo filesystems.""" + skip_types = {"tmpfs", "devtmpfs", "proc", "sysfs", "cgroup", "cgroup2", + "overlay", "squashfs", "ramfs", "devpts", "mqueue", "pstore", + "securityfs", "debugfs", "tracefs", "hugetlbfs", "fusectl", + "configfs", "bpf", "autofs", "efivarfs", "binfmt_misc"} + mounts: list[str] = [] + try: + with open("/proc/mounts", "r") as f: + for line in f: + parts = line.split() + if len(parts) < 3: + continue + if parts[2] in skip_types: + continue + mounts.append(parts[1]) + except OSError: + return ["/"] + return mounts or ["/"] + + +def collect_net_raw() -> dict: + """Return {iface: (rx_bytes, tx_bytes)} cumulative counters from /proc/net/dev. + + Skips loopback and veth* (container) interfaces to cut noise. Counters are + monotonic; per-second rates are derived from deltas in build_sample(). + """ + out: dict = {} + try: + text = _read_file(NETDEV_PATH) + except OSError: + return out + for line in text.splitlines(): + if ":" not in line: + continue + name, _, rest = line.partition(":") + name = name.strip() + if name == "lo" or name.startswith("veth"): + continue + f = rest.split() + if len(f) < 16: + continue + try: + out[name] = (int(f[0]), int(f[8])) # rx_bytes, tx_bytes + except ValueError: + continue + return out + + +def collect_diskio_raw() -> dict: + """Return {device: (read_bytes, write_bytes)} cumulative from /proc/diskstats. + + Whole disks only — loop/ram/sr/fd and partitions are skipped. Sectors are + 512 bytes. Counters are monotonic; rates derived from deltas. + """ + out: dict = {} + try: + text = _read_file(DISKSTATS_PATH) + except OSError: + return out + for line in text.splitlines(): + f = line.split() + if len(f) < 10: + continue + name = f[2] + if name.startswith(("loop", "ram", "sr", "fd")) or _PARTITION_RE.match(name): + continue + try: + out[name] = (int(f[5]) * 512, int(f[9]) * 512) # sectors_read, sectors_written + except ValueError: + continue + return out + + +def collect_temps() -> list: + """Return [{label, celsius}] from /sys/class/hwmon (best-effort, may be empty).""" + out: list = [] + try: + chips = os.listdir(HWMON_DIR) + except OSError: + return out + for hw in chips: + d = os.path.join(HWMON_DIR, hw) + chip = "" + try: + chip = _read_file(os.path.join(d, "name")).strip() + except OSError: + pass + try: + files = os.listdir(d) + except OSError: + continue + for fn in files: + if not (fn.startswith("temp") and fn.endswith("_input")): + continue + idx = fn[: -len("_input")] # e.g. "temp1" + try: + milli = int(_read_file(os.path.join(d, fn)).strip()) + except (OSError, ValueError): + continue + label = "" + try: + label = _read_file(os.path.join(d, idx + "_label")).strip() + except OSError: + pass + name = label or (f"{chip}_{idx}" if chip else idx) + out.append({"label": name, "celsius": round(milli / 1000.0, 1)}) + return out + + +def _parse_psi(text: str) -> dict: + """Parse a /proc/pressure/* file into {some_avg10, some_avg60, full_avg10, …}.""" + res: dict = {} + for line in text.splitlines(): + parts = line.split() + if not parts: + continue + kind = parts[0] # "some" | "full" + for p in parts[1:]: + for prefix, suffix in (("avg10=", "avg10"), ("avg60=", "avg60")): + if p.startswith(prefix): + try: + res[f"{kind}_{suffix}"] = float(p[len(prefix):]) + except ValueError: + pass + return res + + +def collect_psi() -> dict: + """Return pressure-stall (PSI) gauges, prefixed by resource (mem/cpu/io). + + Absent on kernels without CONFIG_PSI — returns only what exists. + """ + out: dict = {} + for res_name, fname in (("mem", "memory"), ("cpu", "cpu"), ("io", "io")): + try: + text = _read_file(os.path.join(PSI_DIR, fname)) + except OSError: + continue + for k, v in _parse_psi(text).items(): + out[f"{res_name}_{k}"] = v + return out + + +def _rates(cur: dict, prev: dict, dt: float) -> dict: + """Per-second rates for counters present in both snapshots over dt seconds. + + Skips names missing from prev and negative deltas (counter reset / iface or + disk hot-plug), so a reboot never emits a bogus spike. + """ + out: dict = {} + if dt <= 0: + return out + for name, cur_vals in cur.items(): + if name not in prev: + continue + prev_vals = prev[name] + deltas = [c - p for c, p in zip(cur_vals, prev_vals)] + if any(d < 0 for d in deltas): + continue + out[name] = tuple(d / dt for d in deltas) + return out + + +# ─── docker ────────────────────────────────────────────────────────────────── +# +# Per-host container collection over the local Docker socket. stdlib-only: a +# minimal HTTP/1.1 client over an AF_UNIX socket. We send `Connection: close` +# so the daemon closes the socket at end of response and we can read to EOF; +# Docker still frames the body with Transfer-Encoding: chunked, so we de-chunk +# when that header is present rather than assume Content-Length. + +DOCKER_API_TIMEOUT = 5.0 + + +def _dechunk(body: bytes) -> bytes: + """Decode an HTTP/1.1 chunked-transfer body into the raw payload.""" + out = bytearray() + while body: + line, sep, rest = body.partition(b"\r\n") + if not sep: + break + try: + size = int(line.split(b";", 1)[0], 16) # ignore chunk extensions + except ValueError: + break + if size == 0: + break + out += rest[:size] + body = rest[size + 2:] # skip the chunk data and its trailing CRLF + return bytes(out) + + +def _docker_request(socket_path: str, path: str, timeout: float = DOCKER_API_TIMEOUT): + """GET `path` from the Docker Engine API over a Unix socket; return JSON. + + Raises OSError on any socket/transport problem (absent socket, permission + denied, non-200) and ValueError on a non-JSON body — both are caught by the + caller and treated as "no docker here", so collection degrades silently. + """ + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.settimeout(timeout) + try: + sock.connect(socket_path) + req = ( + "GET " + path + " HTTP/1.1\r\n" + "Host: docker\r\n" + "Accept: application/json\r\n" + "Connection: close\r\n" + "\r\n" + ) + sock.sendall(req.encode("ascii")) + chunks = [] + while True: + buf = sock.recv(65536) + if not buf: + break + chunks.append(buf) + finally: + sock.close() + + raw = b"".join(chunks) + head, _, body = raw.partition(b"\r\n\r\n") + header_text = head.decode("latin-1") + status_line = header_text.split("\r\n", 1)[0] + parts = status_line.split(None, 2) # "HTTP/1.1 200 OK" + status = int(parts[1]) if len(parts) >= 2 and parts[1].isdigit() else 0 + if status != 200: + raise OSError(f"docker API {path} returned {status}") + if "transfer-encoding: chunked" in header_text.lower(): + body = _dechunk(body) + return json.loads(body.decode("utf-8")) + + +def _docker_cpu_pct(stats: dict) -> float: + """CPU % from a Docker stats snapshot (delta of container vs system CPU). + + Ported verbatim from the old central scraper's math; correct as long as both + cpu_stats and precpu_stats are populated (we request without `one-shot` so + the daemon fills precpu over one cycle). + """ + try: + cpu = stats.get("cpu_stats", {}) + precpu = stats.get("precpu_stats", {}) + cpu_delta = ( + cpu["cpu_usage"]["total_usage"] - precpu["cpu_usage"]["total_usage"] + ) + sys_delta = cpu.get("system_cpu_usage", 0) - precpu.get("system_cpu_usage", 0) + num_cpus = cpu.get("online_cpus") or len( + cpu["cpu_usage"].get("percpu_usage") or [None] + ) + if sys_delta <= 0 or cpu_delta < 0: + return 0.0 + return round((cpu_delta / sys_delta) * num_cpus * 100.0, 2) + except (KeyError, TypeError, ZeroDivisionError): + return 0.0 + + +def _docker_mem(stats: dict): + """Return (usage_bytes, limit_bytes, mem_pct); working set excludes cache.""" + try: + mem = stats.get("memory_stats", {}) + usage = mem.get("usage", 0) + limit = mem.get("limit", 0) + # Subtract page cache for working set (cgroup v1 "cache", v2 "inactive_file"). + cache = mem.get("stats", {}).get("cache", 0) or \ + mem.get("stats", {}).get("inactive_file", 0) + actual = max(0, usage - cache) + pct = round(actual / limit * 100.0, 2) if limit > 0 else 0.0 + return actual, limit, pct + except (KeyError, TypeError): + return 0, 0, 0.0 + + +def _docker_ports(ports: list) -> list: + """Normalise Docker port bindings to a compact published-ports list.""" + out = [] + for p in (ports or []): + if p.get("PublicPort"): + out.append({ + "host_port": p["PublicPort"], + "container_port": p.get("PrivatePort"), + "protocol": p.get("Type", "tcp"), + }) + return out + + +def _docker_grouping(labels: dict) -> dict: + """Pull compose project + swarm service/task/node out of container labels. + + These are set by Docker itself (compose / swarm), so reading them off the + container costs nothing extra and lets Steward group + place containers + without a manager-node query. + """ + labels = labels or {} + return { + "compose_project": labels.get("com.docker.compose.project"), + "service_name": labels.get("com.docker.swarm.service.name"), + "task_id": labels.get("com.docker.swarm.task.id"), + "node_id": labels.get("com.docker.swarm.node.id"), + } + + +def _docker_inspect_fields(insp: dict): + """Return (health, restart_count, exit_code, oom_killed) from an inspect. + + health is None for containers without a HEALTHCHECK (no State.Health). + """ + state = (insp or {}).get("State") or {} + health = (state.get("Health") or {}).get("Status") # healthy|unhealthy|starting + restart_count = insp.get("RestartCount", 0) if isinstance(insp, dict) else 0 + exit_code = state.get("ExitCode") + oom_killed = bool(state.get("OOMKilled", False)) + return health, restart_count, exit_code, oom_killed + + +def _docker_net_io(stats: dict): + """Return cumulative (net_rx, net_tx, blk_read, blk_write) bytes from stats. + + Counters are cumulative since container start (rates can be derived later); + block I/O comes from the cgroup io_service_bytes list, absent on some setups. + """ + net_rx = net_tx = blk_read = blk_write = 0 + for iface in (stats.get("networks") or {}).values(): + net_rx += iface.get("rx_bytes", 0) or 0 + net_tx += iface.get("tx_bytes", 0) or 0 + for e in ((stats.get("blkio_stats") or {}).get("io_service_bytes_recursive") or []): + op = (e.get("op") or "").lower() + if op == "read": + blk_read += e.get("value", 0) or 0 + elif op == "write": + blk_write += e.get("value", 0) or 0 + return net_rx, net_tx, blk_read, blk_write + + +def _collect_one_container(socket_path: str, c: dict) -> dict: + """Build one container's enriched record (runs in a worker thread). + + Does a per-container inspect (health/restart/exit/oom — only available there) + plus a stats read for running containers (cpu/mem/net/io). All best-effort: + a failed call just leaves the affected fields at their defaults. + """ + cid = c.get("Id", "") or "" + names = c.get("Names") or [] + name = names[0].lstrip("/") if names else cid[:12] + state = c.get("State", "unknown") + + # Inspect every container (incl. stopped) — exit codes + restart counts only + # live here, and stopped containers are exactly where exit_code matters. + health = exit_code = None + restart_count = 0 + oom_killed = False + try: + insp = _docker_request(socket_path, f"/containers/{cid}/json") + health, restart_count, exit_code, oom_killed = _docker_inspect_fields(insp) + except (OSError, ValueError): + pass + + cpu_pct = mem_usage = mem_limit = mem_pct = None + net_rx = net_tx = blk_read = blk_write = None + if state == "running": + try: + stats = _docker_request(socket_path, f"/containers/{cid}/stats?stream=false") + cpu_pct = _docker_cpu_pct(stats) + mem_usage, mem_limit, mem_pct = _docker_mem(stats) + net_rx, net_tx, blk_read, blk_write = _docker_net_io(stats) + except (OSError, ValueError): + pass + + created = c.get("Created") + started_at = ( + datetime.fromtimestamp(created, tz=timezone.utc).isoformat() + if isinstance(created, (int, float)) else None + ) + record = { + "name": name, + "container_id": cid[:12], + "image": c.get("Image", ""), + "status": state, + "cpu_pct": cpu_pct, + "mem_usage_bytes": mem_usage, + "mem_limit_bytes": mem_limit, + "mem_pct": mem_pct, + "ports": _docker_ports(c.get("Ports", [])), + "started_at": started_at, + "health": health, + "restart_count": restart_count, + "exit_code": exit_code, + "oom_killed": oom_killed, + "net_rx_bytes": net_rx, + "net_tx_bytes": net_tx, + "blk_read_bytes": blk_read, + "blk_write_bytes": blk_write, + } + record.update(_docker_grouping(c.get("Labels"))) + return record + + +def collect_docker(socket_path: str) -> list: + """Per-container state from the local Docker socket, or [] if unavailable. + + Absent socket / permission denied / non-Docker endpoint all yield [] — the + agent silently reports no containers rather than erroring, so a Docker-less + host needs no configuration. + """ + try: + containers = _docker_request(socket_path, "/containers/json?all=true") + except (OSError, ValueError): + return [] + if not isinstance(containers, list) or not containers: + return [] + + # Each container needs an inspect (+ a stats read if running), and the stats + # call blocks ~1s while the daemon computes the cpu delta. Done serially that + # stretches the whole sample on a busy host, so fan the per-container work out + # over a small bounded thread pool (I/O-bound → threads are enough). + workers = min(8, len(containers)) + with ThreadPoolExecutor(max_workers=workers) as ex: + return list(ex.map(lambda c: _collect_one_container(socket_path, c), containers)) + + +# ─── swarm (manager-only) ──────────────────────────────────────────────────── + + +def _swarm_is_manager(info: dict) -> bool: + """True only on a reachable Swarm manager. + + `/services`, `/tasks`, `/nodes` are manager-only — a worker (or a non-swarm + daemon) returns 503 for them. `Swarm.ControlAvailable` in `/info` is exactly + "this node can serve the control-plane API", so we gate on it and skip the + topology query everywhere else (no errors, near-zero cost on workers). + """ + return bool((info or {}).get("Swarm", {}).get("ControlAvailable")) + + +def _swarm_services(services: list, tasks: list) -> list: + """Roll desired-vs-running replicas per service up from the task list. + + Replica health isn't on the service object — it's derived by counting tasks: + `running` = tasks currently in state running; `desired` = the service's + configured replica count (replicated) or the count of tasks the scheduler + wants running (global, which has no fixed number). + """ + running_by_svc: dict = {} + desired_by_svc: dict = {} # only used for global-mode services + placement: dict = {} # svc_id -> {node_id: running_count} + for t in (tasks or []): + sid = t.get("ServiceID") + if not sid: + continue + if ((t.get("Status") or {}).get("State") or "").lower() == "running": + running_by_svc[sid] = running_by_svc.get(sid, 0) + 1 + node = t.get("NodeID") + if node: + placement.setdefault(sid, {})[node] = \ + placement.setdefault(sid, {}).get(node, 0) + 1 + if (t.get("DesiredState") or "").lower() == "running": + desired_by_svc[sid] = desired_by_svc.get(sid, 0) + 1 + + out = [] + for s in (services or []): + sid = s.get("ID", "") or "" + spec = s.get("Spec") or {} + name = spec.get("Name") or sid[:12] + mode_obj = spec.get("Mode") or {} + if "Global" in mode_obj: + mode = "global" + desired = desired_by_svc.get(sid, 0) + else: + mode = "replicated" + desired = (mode_obj.get("Replicated") or {}).get("Replicas", 0) or 0 + image = (((spec.get("TaskTemplate") or {}).get("ContainerSpec") or {}) + .get("Image") or "") + image = image.split("@", 1)[0] # drop the @sha256:… digest suffix + out.append({ + "service_name": name, "mode": mode, + "desired": desired, "running": running_by_svc.get(sid, 0), + "image": image, + # task→node placement of the running replicas (cross-node — a manager + # sees every task, even ones whose containers live on other hosts). + "placement": [{"node_id": n, "running": cnt} + for n, cnt in sorted(placement.get(sid, {}).items())], + }) + return out + + +def _swarm_nodes(nodes: list) -> list: + """Normalise node role / availability / status (+ manager leader flag).""" + out = [] + for n in (nodes or []): + spec = n.get("Spec") or {} + desc = n.get("Description") or {} + status = n.get("Status") or {} + mgr = n.get("ManagerStatus") or {} + out.append({ + "node_id": n.get("ID", "") or "", + "hostname": desc.get("Hostname", "") or "", + "role": (spec.get("Role") or "worker").lower(), + "availability": (spec.get("Availability") or "active").lower(), + "status": (status.get("State") or "unknown").lower(), + "leader": bool(mgr.get("Leader", False)), + }) + return out + + +def collect_swarm(socket_path: str): + """Swarm topology from a manager node, or None on workers / non-swarm hosts. + + Self-detects manager via `/info` (one cheap call); only then queries the + manager-only endpoints. Any transport failure degrades to None, same silent + contract as collect_docker — a non-swarm host adds nothing to the payload. + """ + try: + info = _docker_request(socket_path, "/info") + except (OSError, ValueError): + return None + if not _swarm_is_manager(info): + return None + try: + services = _docker_request(socket_path, "/services") + tasks = _docker_request(socket_path, "/tasks") + nodes = _docker_request(socket_path, "/nodes") + except (OSError, ValueError): + return None + return { + "services": _swarm_services( + services if isinstance(services, list) else [], + tasks if isinstance(tasks, list) else []), + "nodes": _swarm_nodes(nodes if isinstance(nodes, list) else []), + } + + +# ─── disk usage (image storage) ────────────────────────────────────────────── + + +def _disk_images(images: list) -> list: + """Normalise /system/df image rows → compact per-image storage records. + + Sorted largest-first and capped so a host with a huge image cache doesn't + bloat the payload; the panel only needs the heavy hitters. `containers` is + how many containers reference the image (0 ⇒ reclaimable). + """ + out = [] + for im in images: + if not isinstance(im, dict): + continue + tags = im.get("RepoTags") or [] + # Untagged/dangling images report [":"] or null. + tag = next((t for t in tags if t and t != ":"), None) + raw_id = (im.get("Id") or "").split(":", 1)[-1] + out.append({ + "image_id": raw_id[:12], + "repo_tag": tag or "", + "size": int(im.get("Size") or 0), + "shared_size": int(im.get("SharedSize") or 0) if im.get("SharedSize", -1) >= 0 else 0, + "containers": int(im.get("Containers") or 0) if (im.get("Containers") or 0) > 0 else 0, + }) + out.sort(key=lambda r: r["size"], reverse=True) + return out[:50] + + +def collect_disk_usage(socket_path: str): + """Docker disk usage from `/system/df`, or None if unavailable. + + One bounded API call (the daemon walks images/layers/volumes). Reclaimable = + bytes held by images no container references — the same intent as + `docker system df`'s RECLAIMABLE column. Silent-skip (None) on any transport + failure, same contract as collect_docker/collect_swarm. + """ + try: + df = _docker_request(socket_path, "/system/df") + except (OSError, ValueError): + return None + if not isinstance(df, dict): + return None + + images = [im for im in (df.get("Images") or []) if isinstance(im, dict)] + containers = [c for c in (df.get("Containers") or []) if isinstance(c, dict)] + volumes = [v for v in (df.get("Volumes") or []) if isinstance(v, dict)] + build_cache = [b for b in (df.get("BuildCache") or []) if isinstance(b, dict)] + + def _vol_size(v): + return int((v.get("UsageData") or {}).get("Size") or 0) + + return { + "layers_size": int(df.get("LayersSize") or 0), + "images_total": len(images), + "images_active": sum(1 for im in images if (im.get("Containers") or 0) > 0), + "images_size": sum(int(im.get("Size") or 0) for im in images), + "images_reclaimable": sum( + int(im.get("Size") or 0) for im in images if (im.get("Containers") or 0) <= 0), + "containers_count": len(containers), + "containers_size": sum(int(c.get("SizeRw") or 0) for c in containers), + "volumes_count": len(volumes), + "volumes_size": sum(_vol_size(v) for v in volumes if _vol_size(v) > 0), + "build_cache_size": sum(int(b.get("Size") or 0) for b in build_cache), + "images": _disk_images(images), + } + + +# ─── ring buffer ───────────────────────────────────────────────────────────── + + +class RingBuffer: + def __init__(self, maxlen: int = 20) -> None: + self._dq: deque = deque(maxlen=maxlen) + + def push(self, item) -> None: + self._dq.append(item) + + def drain(self) -> list: + out = list(self._dq) + self._dq.clear() + return out + + def __len__(self) -> int: + return len(self._dq) + + +# ─── payload ───────────────────────────────────────────────────────────────── + + +def build_sample(mounts: list[str], state: dict, + docker_socket: str = DEFAULT_DOCKER_SOCKET) -> dict: + """Collect one full sample. Partial samples allowed if a collector fails. + + `state` carries the previous network/disk counters + monotonic timestamp so + throughput rates can be derived from deltas; it is mutated in place. + `docker_socket` is probed best-effort — the `docker` key is omitted entirely + when no containers are found, so non-Docker hosts add nothing to the payload. + """ + sample: dict = {"ts": datetime.now(timezone.utc).isoformat()} + try: + agg, cores = collect_cpu() + sample["cpu_pct"] = agg + sample["cpu_cores"] = cores + except Exception: + sample["cpu_pct"] = None + try: + sample["mem"] = collect_memory() + except Exception: + sample["mem"] = None + try: + sample["load"] = collect_load() + except Exception: + sample["load"] = None + try: + sample["uptime_secs"] = collect_uptime() + except Exception: + sample["uptime_secs"] = None + try: + sample["storage"] = collect_storage(mounts) + except Exception: + sample["storage"] = [] + try: + sample["temps"] = collect_temps() + except Exception: + sample["temps"] = [] + try: + sample["psi"] = collect_psi() + except Exception: + sample["psi"] = {} + + # Throughput rates: diff cumulative counters against the previous sample. + now_mono = time.monotonic() + try: + net_raw = collect_net_raw() + except Exception: + net_raw = {} + try: + disk_raw = collect_diskio_raw() + except Exception: + disk_raw = {} + dt = now_mono - state["mono"] if state.get("mono") else 0.0 + sample["net"] = [ + {"iface": n, "rx_bps": round(rx, 1), "tx_bps": round(tx, 1)} + for n, (rx, tx) in _rates(net_raw, state.get("net", {}), dt).items() + ] + sample["diskio"] = [ + {"device": n, "read_bps": round(rd, 1), "write_bps": round(wr, 1)} + for n, (rd, wr) in _rates(disk_raw, state.get("disk", {}), dt).items() + ] + state["net"], state["disk"], state["mono"] = net_raw, disk_raw, now_mono + + try: + docker = collect_docker(docker_socket) + except Exception: + docker = [] + if docker: + sample["docker"] = docker + + # Swarm topology is only emitted by managers (collect_swarm returns None + # everywhere else), so the key is absent on workers / non-swarm hosts. + try: + swarm = collect_swarm(docker_socket) + except Exception: + swarm = None + if swarm is not None: + sample["swarm"] = swarm + + # Image/disk usage — host-level docker fact (one /system/df call), omitted on + # Docker-less hosts. Only meaningful when there's docker here at all, so skip + # the call entirely when collect_docker found nothing. + if docker: + try: + disk = collect_disk_usage(docker_socket) + except Exception: + disk = None + if disk is not None: + sample["docker_disk"] = disk + + return sample + + +def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict: + return { + "agent_version": AGENT_VERSION, + "hostname": hostname, + "metadata": metadata, + "samples": samples, + } + + +# ─── POST + backoff ────────────────────────────────────────────────────────── + + +BACKOFF_CAP = 300 + + +def next_backoff(current: int) -> int: + if current <= 0: + return 30 + return min(current * 2, BACKOFF_CAP) + + +def post_payload(url: str, token: str, payload: dict) -> tuple[bool, int | None]: + body = json.dumps(payload).encode("utf-8") + req = urllib.request.Request( + url.rstrip("/") + "/plugins/host_agent/ingest", + data=body, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + "User-Agent": f"steward-host-agent/{AGENT_VERSION}", + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + return (200 <= resp.status < 300, resp.status) + except urllib.error.HTTPError as e: + return (False, e.code) + except (urllib.error.URLError, TimeoutError, socket.timeout, OSError): + return (False, None) + + +# ─── main loop ─────────────────────────────────────────────────────────────── + + +_reload_requested = False +_shutdown_requested = False + + +def _handle_hup(_signum, _frame): + global _reload_requested + _reload_requested = True + + +def _handle_term(_signum, _frame): + global _shutdown_requested + _shutdown_requested = True + + +def _log(level: str, msg: str) -> None: + sys.stderr.write(f"[{level}] {msg}\n") + sys.stderr.flush() + + +def main_loop(conf_path: str) -> int: + global _reload_requested, _shutdown_requested + signal.signal(signal.SIGHUP, _handle_hup) + signal.signal(signal.SIGTERM, _handle_term) + + try: + cfg = read_config(conf_path) + except ConfigError as e: + _log("ERROR", str(e)) + return 2 + + metadata = collect_metadata() + hostname = cfg.get("hostname") or socket.gethostname() + mounts = cfg.get("mounts") or default_mounts() + docker_socket = cfg.get("docker_socket") or DEFAULT_DOCKER_SOCKET + buffer = RingBuffer(maxlen=20) + backoff = 0 + # Carries previous net/disk counters + monotonic ts for rate computation. + rate_state: dict = {} + + _log("INFO", f"steward-host-agent {AGENT_VERSION} starting " + f"(url={cfg['url']}, interval={cfg['interval_seconds']}s)") + + while not _shutdown_requested: + if _reload_requested: + try: + cfg = read_config(conf_path) + mounts = cfg.get("mounts") or default_mounts() + docker_socket = cfg.get("docker_socket") or DEFAULT_DOCKER_SOCKET + metadata = collect_metadata() # refresh host_ip/distro on reload + _log("INFO", "config reloaded") + except ConfigError as e: + _log("ERROR", f"reload failed: {e}") + _reload_requested = False + + sample = build_sample(mounts, rate_state, docker_socket) + buffered = buffer.drain() + payload = build_payload( + samples=buffered + [sample], + hostname=hostname, + metadata=metadata, + ) + ok, status = post_payload(cfg["url"], cfg["token"], payload) + + if ok: + if buffered: + _log("INFO", f"flushed {len(buffered)} buffered samples") + backoff = 0 + sleep_for = cfg["interval_seconds"] + else: + if status == 400: + _log("ERROR", "server rejected payload (400) — dropping sample") + elif status == 401: + _log("ERROR", "token rejected (401) — check config + UI") + buffer.push(sample) + else: + _log("WARN", f"POST failed (status={status}); buffering") + for s in buffered: + buffer.push(s) + buffer.push(sample) + backoff = next_backoff(backoff) + sleep_for = backoff + + slept = 0.0 + while slept < sleep_for and not _shutdown_requested and not _reload_requested: + time.sleep(min(1.0, sleep_for - slept)) + slept += 1.0 + + _log("INFO", "SIGTERM — flushing and exiting") + final = buffer.drain() + if final: + post_payload(cfg["url"], cfg["token"], + build_payload(final, hostname, metadata)) + return 0 + + +if __name__ == "__main__": + conf = os.environ.get("STEWARD_AGENT_CONFIG", "/etc/steward-agent.conf") + sys.exit(main_loop(conf)) diff --git a/plugins/host_agent/metrics_query.py b/plugins/host_agent/metrics_query.py new file mode 100644 index 0000000..1438443 --- /dev/null +++ b/plugins/host_agent/metrics_query.py @@ -0,0 +1,112 @@ +"""Host-metric read helpers (latest snapshot + bucketed history). + +Kept separate from routes.py so it imports only the core PluginMetric model — not +the host_agent ORM models — which lets integration tests import these helpers +without tripping the plugin-loader's double-registration ("Table already +defined") guard. +""" +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +from sqlalchemy import func, or_, select + +from steward.core.time_range import bucket_seconds +from steward.models.metrics import PluginMetric, PluginMetricHourly + +SOURCE_MODULE = "host_agent" + +# Host-level metrics charted on the detail page (sub-resources are shown as +# current-value lists, not time series, to keep the page readable). +HISTORY_METRICS = ( + "cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m", + "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps", + "temp_c_max", "psi_mem_some_avg10", +) + + +async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]: + """{resource_name: {metric: value}} — latest sample per (resource, metric) for + a host + its sub-resources. + + DISTINCT ON picks the newest row per group in one index-ordered pass over + ix_plugin_metrics_module_resource_metric_recorded, instead of a GROUP-BY-max + subquery self-joined back to the table (two passes over the whole history). + """ + rows = (await session.execute( + select(PluginMetric) + .where( + PluginMetric.source_module == SOURCE_MODULE, + or_( + PluginMetric.resource_name == host_name, + PluginMetric.resource_name.like(host_name + ":%"), + ), + ) + .distinct(PluginMetric.resource_name, PluginMetric.metric_name) + .order_by( + PluginMetric.resource_name, + PluginMetric.metric_name, + PluginMetric.recorded_at.desc(), + ) + )).scalars().all() + out: dict[str, dict[str, float]] = {} + for r in rows: + out.setdefault(r.resource_name, {})[r.metric_name] = r.value + return out + + +async def _history_for_host(session, host_name: str, since, *, raw_days: int = 7) -> dict[str, list]: + """{metric: [[epoch_ms, avg_value], …]} host-level series since `since`. + + Retention rolls raw plugin_metrics older than `raw_days` into hourly averages + (plugin_metrics_hourly) and deletes the raw rows. So we read the rollup for the + part of the range older than that boundary and raw (bucket-averaged in SQL, + capped to ≤1h to match the rollup) for the recent part — never shipping raw + samples to Python. The two windows don't overlap, so appending hourly-then-raw + keeps each metric's series time-ordered. Epoch-ms x feeds a linear chart axis. + """ + now = datetime.now(timezone.utc) + raw_cutoff = (now - timedelta(days=raw_days)).replace(minute=0, second=0, microsecond=0) + series: dict[str, list] = {m: [] for m in HISTORY_METRICS} + + # ── Older than the raw window: the hourly rollup ── + if since < raw_cutoff: + hourly = (await session.execute( + select(PluginMetricHourly.metric_name, PluginMetricHourly.bucket, + PluginMetricHourly.value_avg) + .where( + PluginMetricHourly.source_module == SOURCE_MODULE, + PluginMetricHourly.resource_name == host_name, + PluginMetricHourly.metric_name.in_(HISTORY_METRICS), + PluginMetricHourly.bucket >= since, + PluginMetricHourly.bucket < raw_cutoff, + ) + .order_by(PluginMetricHourly.bucket) + )).all() + for metric_name, bucket, avg in hourly: + series[metric_name].append([int(bucket.timestamp() * 1000), round(float(avg), 2)]) + + # ── Recent part: raw, bucket-averaged in SQL ── + raw_since = since if since >= raw_cutoff else raw_cutoff + width_s = bucket_seconds(raw_since, 120) + if since < raw_cutoff: + width_s = min(width_s, 3600) # ≤ 1h so it matches the hourly portion + rbucket = func.date_bin( + func.make_interval(0, 0, 0, 0, 0, 0, width_s), + PluginMetric.recorded_at, + func.to_timestamp(0), # epoch origin + ).label("bucket") + raw = (await session.execute( + select(PluginMetric.metric_name, rbucket, func.avg(PluginMetric.value)) + .where( + PluginMetric.source_module == SOURCE_MODULE, + PluginMetric.resource_name == host_name, + PluginMetric.metric_name.in_(HISTORY_METRICS), + PluginMetric.recorded_at >= raw_since, + ) + .group_by(PluginMetric.metric_name, rbucket) + .order_by(rbucket) + )).all() + for metric_name, b, avg in raw: + series[metric_name].append([int(b.timestamp() * 1000), round(float(avg), 2)]) + return series diff --git a/plugins/host_agent/migrations/__init__.py b/plugins/host_agent/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/host_agent/migrations/env.py b/plugins/host_agent/migrations/env.py new file mode 100644 index 0000000..6545c0e --- /dev/null +++ b/plugins/host_agent/migrations/env.py @@ -0,0 +1,70 @@ +# plugins/host_agent/migrations/env.py +"""Alembic env.py for the host_agent plugin.""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from steward.models.base import Base +import steward.models # noqa: F401 +from plugins.host_agent.models import HostAgentRegistration # noqa: F401 + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("STEWARD_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/host_agent/migrations/versions/__init__.py b/plugins/host_agent/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/host_agent/migrations/versions/host_agent_001_initial.py b/plugins/host_agent/migrations/versions/host_agent_001_initial.py new file mode 100644 index 0000000..d241257 --- /dev/null +++ b/plugins/host_agent/migrations/versions/host_agent_001_initial.py @@ -0,0 +1,36 @@ +# plugins/host_agent/migrations/versions/host_agent_001_initial.py +"""host_agent plugin initial tables + +Revision ID: host_agent_001_initial +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "host_agent_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "host_agent" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "host_agent_registrations", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), + nullable=False, unique=True), + sa.Column("token_hash", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("token_created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("agent_version", sa.String(32), nullable=True), + sa.Column("kernel", sa.String(128), nullable=True), + sa.Column("distro", sa.String(128), nullable=True), + sa.Column("arch", sa.String(32), nullable=True), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("host_agent_registrations") diff --git a/plugins/host_agent/migrations/versions/host_agent_002_host_ip.py b/plugins/host_agent/migrations/versions/host_agent_002_host_ip.py new file mode 100644 index 0000000..197dd2f --- /dev/null +++ b/plugins/host_agent/migrations/versions/host_agent_002_host_ip.py @@ -0,0 +1,24 @@ +# plugins/host_agent/migrations/versions/host_agent_002_host_ip.py +"""host_agent: add agent-reported host_ip column + +Revision ID: host_agent_002_host_ip +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "host_agent_002_host_ip" +down_revision: Union[str, None] = "host_agent_001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "host_agent_registrations", + sa.Column("host_ip", sa.String(45), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("host_agent_registrations", "host_ip") diff --git a/plugins/host_agent/models.py b/plugins/host_agent/models.py new file mode 100644 index 0000000..e8d87d0 --- /dev/null +++ b/plugins/host_agent/models.py @@ -0,0 +1,34 @@ +# plugins/host_agent/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import Column, String, DateTime, ForeignKey +from steward.models.base import Base + + +def _uuid() -> str: + return str(uuid.uuid4()) + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +class HostAgentRegistration(Base): + __tablename__ = "host_agent_registrations" + + id = Column(String(36), primary_key=True, default=_uuid) + host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"), + unique=True, nullable=False) + token_hash = Column(String(64), nullable=False, unique=True, index=True) + token_created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) + agent_version = Column(String(32), nullable=True) + kernel = Column(String(128), nullable=True) + distro = Column(String(128), nullable=True) + arch = Column(String(32), nullable=True) + # Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix). + host_ip = Column(String(45), nullable=True) + last_seen_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) + updated_at = Column(DateTime(timezone=True), nullable=False, + default=_utcnow, onupdate=_utcnow) diff --git a/plugins/host_agent/plugin.yaml b/plugins/host_agent/plugin.yaml new file mode 100644 index 0000000..ab23525 --- /dev/null +++ b/plugins/host_agent/plugin.yaml @@ -0,0 +1,19 @@ +# plugins/host_agent/plugin.yaml +name: host_agent +version: "1.2.0" +description: "Remote Linux host resource monitoring via a lightweight Python push agent (CPU incl. per-core, memory + PSI, storage, disk I/O, network throughput, load, temperatures, uptime)" +author: "Steward" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/host_agent" +tags: + - host + - monitoring + - cpu + - memory + - storage + +config: + stale_after_seconds: 180 + default_interval_seconds: 30 diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py new file mode 100644 index 0000000..b63ae1a --- /dev/null +++ b/plugins/host_agent/routes.py @@ -0,0 +1,1075 @@ +# plugins/host_agent/routes.py +from __future__ import annotations + +import hashlib +from datetime import datetime, timezone +from typing import Any + +from pathlib import Path + +import secrets +from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for, session +from steward.auth.middleware import require_role +from steward.models.users import UserRole +from sqlalchemy import select, func, or_, and_ +from datetime import timedelta + +from steward.core.settings import get_setting, public_base_url +from steward.core.time_range import parse_range, RANGE_OPTIONS +from steward.models.hosts import Host +from steward.models.metrics import PluginMetric +from .models import HostAgentRegistration +# Query helpers live in a model-free module so integration tests can import them +# without the plugin-loader double-registration guard tripping (see metrics_query). +from .metrics_query import ( + SOURCE_MODULE, + _history_for_host, + _latest_metrics_for_host, +) + +host_agent_bp = Blueprint("host_agent", __name__, template_folder="templates") + + +def _hash_token(raw: str) -> str: + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def _parse_ts(ts: str) -> datetime: + if ts.endswith("Z"): + ts = ts[:-1] + "+00:00" + return datetime.fromisoformat(ts) + + +def pick_host_address(current: str | None, reported: str | None) -> str | None: + """Return the address to store on the Host, or None for no change. + + The agent-reported IP fills Host.address only when the current value is + blank — an admin-typed address (DNS name, management IP) is never + overwritten. The reported IP is always kept on the registration regardless, + so admins can still see drift. + """ + if reported and not (current or "").strip(): + return reported + return None + + +def _expand_sample_to_metrics( + sample: dict, host_name: str, recorded_at: datetime +) -> list[PluginMetric]: + rows: list[PluginMetric] = [] + + def row(metric: str, resource: str, value: float) -> None: + rows.append(PluginMetric( + source_module=SOURCE_MODULE, + resource_name=resource, + metric_name=metric, + value=float(value), + recorded_at=recorded_at, + )) + + if sample.get("cpu_pct") is not None: + row("cpu_pct", host_name, sample["cpu_pct"]) + + mem = sample.get("mem") or {} + if mem.get("total_bytes"): + total = mem["total_bytes"] + available = mem.get("available_bytes", 0) + used_pct = 100.0 * (total - available) / total if total else 0.0 + row("mem_used_pct", host_name, used_pct) + row("mem_available_bytes", host_name, available) + row("swap_used_bytes", host_name, mem.get("swap_used_bytes", 0)) + + load = sample.get("load") or {} + for key in ("1m", "5m", "15m"): + if key in load: + row(f"load_{key}", host_name, load[key]) + + if sample.get("uptime_secs") is not None: + row("uptime_secs", host_name, sample["uptime_secs"]) + + worst_pct = 0.0 + for disk in sample.get("storage") or []: + total = disk.get("total_bytes", 0) + used = disk.get("used_bytes", 0) + pct = 100.0 * used / total if total else 0.0 + mount_res = f"{host_name}:{disk['mount']}" + row("disk_used_pct", mount_res, pct) + row("disk_used_bytes", mount_res, used) + row("disk_total_bytes", mount_res, total) + if pct > worst_pct: + worst_pct = pct + if sample.get("storage"): + row("disk_used_pct_worst", host_name, worst_pct) + + # Per-core CPU. Sub-resources carry a ':' so the fleet widget filters them out. + for i, core_pct in enumerate(sample.get("cpu_cores") or []): + if core_pct is not None: + row("cpu_pct", f"{host_name}:core{i}", core_pct) + + # Richer memory breakdown. + if mem.get("cached_bytes") is not None: + row("mem_cached_bytes", host_name, mem["cached_bytes"]) + if mem.get("buffers_bytes") is not None: + row("mem_buffers_bytes", host_name, mem["buffers_bytes"]) + + # Network throughput — per interface plus a host-level total for the fleet view. + net_rx_total = net_tx_total = 0.0 + for iface in sample.get("net") or []: + rx, tx = iface.get("rx_bps", 0.0), iface.get("tx_bps", 0.0) + res = f"{host_name}:net:{iface['iface']}" + row("net_rx_bps", res, rx) + row("net_tx_bps", res, tx) + net_rx_total += rx + net_tx_total += tx + if sample.get("net"): + row("net_rx_bps", host_name, net_rx_total) + row("net_tx_bps", host_name, net_tx_total) + + # Disk I/O — per device plus host-level total. + dr_total = dw_total = 0.0 + for dev in sample.get("diskio") or []: + rd, wr = dev.get("read_bps", 0.0), dev.get("write_bps", 0.0) + res = f"{host_name}:diskio:{dev['device']}" + row("disk_read_bps", res, rd) + row("disk_write_bps", res, wr) + dr_total += rd + dw_total += wr + if sample.get("diskio"): + row("disk_read_bps", host_name, dr_total) + row("disk_write_bps", host_name, dw_total) + + # Temperatures — per sensor plus the host max for at-a-glance. + max_temp: float | None = None + for t in sample.get("temps") or []: + c = t.get("celsius") + if c is None: + continue + row("temp_c", f"{host_name}:temp:{t['label']}", c) + if max_temp is None or c > max_temp: + max_temp = c + if max_temp is not None: + row("temp_c_max", host_name, max_temp) + + # Pressure stall information (PSI) — host-level gauges (mem/cpu/io some|full). + for k, v in (sample.get("psi") or {}).items(): + if isinstance(v, (int, float)): + row(f"psi_{k}", host_name, v) + + return rows + + +def _error(status: int, code: str, detail: str | None = None) -> tuple[Any, int]: + body: dict = {"ok": False, "error": code} + if detail: + body["detail"] = detail + return jsonify(body), status + + +@host_agent_bp.post("/ingest") +async def ingest(): + auth = request.headers.get("Authorization", "") + if not auth.startswith("Bearer "): + return _error(401, "invalid_token") + raw = auth[len("Bearer "):].strip() + if not raw: + return _error(401, "invalid_token") + + try: + payload = await request.get_json(force=True) + except Exception: + return _error(400, "malformed_payload", "invalid JSON") + if not isinstance(payload, dict) or "samples" not in payload: + return _error(400, "malformed_payload", "missing 'samples'") + + samples = payload.get("samples") or [] + if not isinstance(samples, list) or not samples: + return _error(400, "malformed_payload", "'samples' must be a non-empty list") + + token_hash = _hash_token(raw) + async with current_app.db_sessionmaker() as session: + reg = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.token_hash == token_hash) + )).scalar_one_or_none() + if reg is None: + return _error(401, "invalid_token") + host = (await session.execute( + select(Host).where(Host.id == reg.host_id) + )).scalar_one_or_none() + if host is None: + return _error(401, "invalid_token") + + accepted = 0 + latest_ts: datetime | None = None + docker_snapshots: list[tuple[datetime, list]] = [] + latest_swarm: dict | None = None + latest_swarm_ts: datetime | None = None + latest_disk: dict | None = None + latest_disk_ts: datetime | None = None + for sample in samples: + try: + recorded_at = _parse_ts(sample["ts"]) + except (KeyError, ValueError): + continue + metrics = _expand_sample_to_metrics(sample, host.name, recorded_at) + for m in metrics: + session.add(m) + docker = sample.get("docker") + if isinstance(docker, list) and docker: + docker_snapshots.append((recorded_at, docker)) + # Swarm is current-state, not time-series — keep only the newest + # sample's topology (a manager re-reports it every interval). + swarm = sample.get("swarm") + if isinstance(swarm, dict) and ( + latest_swarm_ts is None or recorded_at > latest_swarm_ts): + latest_swarm, latest_swarm_ts = swarm, recorded_at + # Disk usage is current-state too — keep only the newest sample's. + disk = sample.get("docker_disk") + if isinstance(disk, dict) and ( + latest_disk_ts is None or recorded_at > latest_disk_ts): + latest_disk, latest_disk_ts = disk, recorded_at + accepted += 1 + if latest_ts is None or recorded_at > latest_ts: + latest_ts = recorded_at + + if accepted == 0: + await session.rollback() + return _error(400, "malformed_payload", "no valid samples") + + # Hand host-scoped container data to the docker plugin if it's enabled + # (opportunistic synergy via the capability registry — no hard import, + # no-op when docker is disabled). A failure here must never sink the + # whole ingest, so the metrics above still land. + if docker_snapshots or latest_swarm is not None or latest_disk is not None: + from steward.core.capabilities import has_capability, invoke_capability + if has_capability("docker.persist_host_samples"): + try: + # SAVEPOINT so a docker-side failure rolls back only the + # docker writes, leaving the host metrics intact to commit. + async with session.begin_nested(): + await invoke_capability( + "docker.persist_host_samples", UserRole.admin, + session, host, docker_snapshots, latest_swarm, + latest_disk, + ) + except Exception: + current_app.logger.exception( + "host_agent ingest: docker persist failed for host=%s", + host.name) + + md = payload.get("metadata") or {} + changed = False + if latest_ts and (reg.last_seen_at is None or latest_ts > reg.last_seen_at): + reg.last_seen_at = latest_ts + changed = True + version = payload.get("agent_version") + if version and reg.agent_version != version: + reg.agent_version = version + changed = True + for field in ("kernel", "distro", "arch"): + if field in md and getattr(reg, field) != md[field]: + setattr(reg, field, md[field]) + changed = True + host_ip = md.get("host_ip") + if host_ip and reg.host_ip != host_ip: + reg.host_ip = host_ip + changed = True + # Mirror the reported IP into Host.address only when it's blank. + new_address = pick_host_address(host.address, host_ip) + if new_address is not None: + host.address = new_address + if changed: + reg.updated_at = datetime.now(timezone.utc) + + if latest_ts: + skew = abs((datetime.now(timezone.utc) - latest_ts).total_seconds()) + if skew > 300: + current_app.logger.warning( + "host_agent ingest: clock skew %.0fs for host=%s", skew, host.name) + + await session.commit() + + return jsonify({"ok": True, "samples_accepted": accepted}), 200 + + +AGENT_SOURCE_PATH = Path(__file__).parent / "agent.py" + + +def _agent_version() -> str: + try: + for line in AGENT_SOURCE_PATH.read_text().splitlines(): + if line.startswith("AGENT_VERSION"): + return line.split("=", 1)[1].strip().strip('"').strip("'") + except OSError: + pass + return "unknown" + + +@host_agent_bp.get("/install.sh") +async def install_script(): + token = request.args.get("token", "") + if not token: + return _error(401, "invalid_token") + async with current_app.db_sessionmaker() as session: + reg = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.token_hash == _hash_token(token)) + )).scalar_one_or_none() + if reg is None: + return _error(401, "invalid_token") + host = (await session.execute( + select(Host).where(Host.id == reg.host_id) + )).scalar_one_or_none() + if host is None: + return _error(401, "invalid_token") + + url = public_base_url(request) + + rendered = await render_template( + "install.sh.j2", + url=url, + token=token, + agent_version=_agent_version(), + host_name=host.name, + host_address=host.address, + ) + return Response(rendered, mimetype="text/plain") + + +@host_agent_bp.get("/agent.py") +async def agent_source(): + try: + body = AGENT_SOURCE_PATH.read_text(encoding="utf-8") + except OSError: + return _error(500, "agent_missing") + return Response(body, mimetype="text/x-python") + + +def _stale_after_seconds() -> int: + return int( + current_app.config.get("PLUGINS", {}) + .get(SOURCE_MODULE, {}) + .get("stale_after_seconds", 180) + ) + + +async def _fleet_rows(session) -> list[dict]: + """One row per registered host with its latest host-level metrics + stale flag. + + Only bare host-level resources are used here (sub-resources carry a ':' and + are skipped) so the fleet glance stays one line per host. + """ + regs = (await session.execute(select(HostAgentRegistration))).scalars().all() + host_ids = [r.host_id for r in regs] + hosts = { + h.id: h for h in (await session.execute( + select(Host).where(Host.id.in_(host_ids)) + )).scalars().all() + } if host_ids else {} + + subq = ( + select( + PluginMetric.resource_name, + PluginMetric.metric_name, + func.max(PluginMetric.recorded_at).label("max_ts"), + ) + .where(PluginMetric.source_module == SOURCE_MODULE) + .group_by(PluginMetric.resource_name, PluginMetric.metric_name) + ).subquery() + latest_rows = (await session.execute( + select(PluginMetric).join( + subq, + (PluginMetric.resource_name == subq.c.resource_name) & + (PluginMetric.metric_name == subq.c.metric_name) & + (PluginMetric.recorded_at == subq.c.max_ts), + ).where(PluginMetric.source_module == SOURCE_MODULE) + )).scalars().all() + + latest: dict[str, dict[str, float]] = {} + root_disk: dict[str, float] = {} # host_name → root (/) disk % + for row in latest_rows: + if row.resource_name.endswith(":/") and row.metric_name == "disk_used_pct": + root_disk[row.resource_name[:-2]] = row.value + if ":" in row.resource_name: + continue + latest.setdefault(row.resource_name, {})[row.metric_name] = row.value + + # Per-host recent series for the inline sparklines (cpu/mem/load host-level, + # disk from the root mount), so each metric shows its trend beside the value — + # the host-panel at-a-glance, on the fleet widget. 1h window keeps it cheap. + host_names = [h.name for h in hosts.values()] + spark_series: dict[str, dict[str, list]] = { + n: {"cpu": [], "mem": [], "disk": [], "load": []} for n in host_names + } + if host_names: + since = datetime.now(timezone.utc) - timedelta(hours=1) + sp_rows = (await session.execute( + select(PluginMetric).where( + PluginMetric.source_module == SOURCE_MODULE, + PluginMetric.recorded_at >= since, + or_( + and_(PluginMetric.resource_name.in_(host_names), + PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))), + and_(PluginMetric.resource_name.in_([n + ":/" for n in host_names]), + PluginMetric.metric_name == "disk_used_pct"), + ), + ).order_by(PluginMetric.recorded_at) + )).scalars().all() + for r in sp_rows: + if r.resource_name.endswith(":/"): + name = r.resource_name[:-2] + if name in spark_series: + spark_series[name]["disk"].append(r.value) + else: + key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name) + if key and r.resource_name in spark_series: + spark_series[r.resource_name][key].append(r.value) + + from steward.core.status import sparkline_svg + + stale_after = _stale_after_seconds() + now = datetime.now(timezone.utc) + rows = [] + for reg in regs: + host = hosts.get(reg.host_id) + if host is None: + continue + m = latest.get(host.name, {}) + ls = reg.last_seen_at + stale = ls is None or (now - ls).total_seconds() > stale_after + ser = spark_series.get(host.name, {}) + sparks = { + k: (sparkline_svg(v[-60:], width=72, height=16) if len(v) >= 2 else "") + for k, v in ser.items() + } + rows.append({ + "host": host, + "reg": reg, + "stale": stale, + "cpu_pct": m.get("cpu_pct"), + "mem_used_pct": m.get("mem_used_pct"), + "disk_worst": m.get("disk_used_pct_worst"), # health dot only + "disk_root": root_disk.get(host.name), # displayed at-a-glance + "load_1m": m.get("load_1m"), + "temp_max": m.get("temp_c_max"), + "net_rx_bps": m.get("net_rx_bps"), + "net_tx_bps": m.get("net_tx_bps"), + "disk_read_bps": m.get("disk_read_bps"), + "disk_write_bps": m.get("disk_write_bps"), + "sparks": sparks, + }) + rows.sort(key=lambda r: r["host"].name.lower()) + return rows + + +@host_agent_bp.get("/widget") +async def widget_table(): + """Fleet-glance dashboard widget: one row per monitored host, latest metrics.""" + async with current_app.db_sessionmaker() as session: + rows = await _fleet_rows(session) + return await render_template("widget_table.html", rows=rows) + + +@host_agent_bp.get("/widget/history") +async def widget_history(): + host_id = request.args.get("host_id", "") + try: + hours = max(1, min(168, int(request.args.get("hours", "6")))) + except ValueError: + hours = 6 + # wid (the dashboard widget row id) keeps the id unique when several + # history widgets share a page; fall back to the host id. + wid = request.args.get("wid", "") or host_id + cutoff = datetime.now(timezone.utc) - timedelta(hours=hours) + + series: dict[str, list[list]] = {"cpu_pct": [], "mem_used_pct": [], "disk_root": []} + host = None + if host_id: + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is not None: + points = (await session.execute( + select(PluginMetric).where( + PluginMetric.source_module == SOURCE_MODULE, + PluginMetric.recorded_at >= cutoff, + or_( + and_(PluginMetric.resource_name == host.name, + PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct"))), + and_(PluginMetric.resource_name == host.name + ":/", + PluginMetric.metric_name == "disk_used_pct"), + ), + ).order_by(PluginMetric.recorded_at) + )).scalars().all() + # Disk = root (/), consistent with the host panel — not "worst". + # Epoch-ms x values let the chart use a plain linear axis (no + # Chart.js date adapter needed), matching the host-detail charts. + for p in points: + key = "disk_root" if p.resource_name != host.name else p.metric_name + series[key].append([int(p.recorded_at.timestamp() * 1000), round(p.value, 2)]) + + # Downsample: the agent reports every few seconds, so a multi-hour window is + # hundreds–thousands of points — a noisy, unreadable line. Bucket-average to a + # readable point count, keeping the shape. + series = {k: _downsample(v) for k, v in series.items()} + + # Stable y-axis ceiling. All three series are percentages, so snap the top up + # to the next 10% band (floor 20, cap 100). Auto-scaling to the exact peak made + # the chart "zoom" jump on every poll as the peak drifted a few %; snapping to + # a band keeps the axis steady across refreshes while still filling the panel. + peak = max((p[1] for s in series.values() for p in s), default=0.0) + y_max = min(100, max(20, ((int(peak) // 10) + 1) * 10)) + return await render_template( + "widget_history.html", host=host, series=series, hours=hours, wid=wid, + y_max=y_max, + ) + + +def _downsample(series: list[list], target: int = 120) -> list[list]: + """Bucket-average a time series [[epoch_ms, value], …] down to ~target points. + + Consecutive samples are grouped into equal buckets and averaged (x = bucket + midpoint). Cheap, order-preserving, and smooths the high-frequency noise that + makes dense agent series hard to read. + """ + n = len(series) + if n <= target: + return series + bucket = (n + target - 1) // target + out: list[list] = [] + for i in range(0, n, bucket): + chunk = series[i:i + bucket] + mid = chunk[len(chunk) // 2][0] + avg = sum(p[1] for p in chunk) / len(chunk) + out.append([mid, round(avg, 2)]) + return out + + +# Full-metrics refresh cadences. The agent pushes every ~30s by default, so +# polling the current snapshot at 15s keeps it ≤ one cadence stale; the history +# charts move slowly over a multi-hour/day window, so they refresh less often. +_DETAIL_POLL_SECONDS = 15 +_CHART_POLL_SECONDS = 60 + + +def _split_host_metrics(host_name: str, latest: dict) -> tuple: + """Split a host's latest snapshot into + (hostlvl, cores, nets, disks_io, temps, mounts) for the current-state fragment.""" + hostlvl = latest.get(host_name, {}) + cores: list = [] + nets: dict = {} + disks_io: dict = {} + temps: dict = {} + mounts: dict = {} + plen = len(host_name) + 1 + for res, metrics in latest.items(): + if res == host_name: + continue + suffix = res[plen:] + if suffix.startswith("core"): + try: + idx = int(suffix[4:]) + except ValueError: + idx = 0 + cores.append((idx, metrics.get("cpu_pct"))) + elif suffix.startswith("net:"): + nets[suffix[len("net:"):]] = metrics + elif suffix.startswith("diskio:"): + disks_io[suffix[len("diskio:"):]] = metrics + elif suffix.startswith("temp:"): + temps[suffix[len("temp:"):]] = metrics.get("temp_c") + else: + mounts[suffix] = metrics + cores.sort(key=lambda c: c[0]) + return hostlvl, cores, nets, disks_io, temps, mounts + + +@host_agent_bp.get("//") +@require_role(UserRole.viewer) +async def host_detail(host_id: str): + """Full-metrics shell: header + range toggle only. The current-state and + history sections stream in via HTMX (host_detail_metrics / host_detail_charts) + so the heavy history query never blocks first paint and the data live-updates.""" + _, range_key = parse_range(request.args.get("range")) + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return _error(404, "not_found") + return await render_template( + "host_detail.html", + host=host, current_range=range_key, range_options=RANGE_OPTIONS, + poll_seconds=_DETAIL_POLL_SECONDS, chart_poll_seconds=_CHART_POLL_SECONDS, + ) + + +@host_agent_bp.get("//metrics") +@require_role(UserRole.viewer) +async def host_detail_metrics(host_id: str): + """Current-state fragment (gauges/cores/filesystems/IO/temps) — polled live so + the page shows the latest snapshot the server holds without a full reload.""" + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return "", 404 + reg = (await session.execute(select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + latest = await _latest_metrics_for_host(session, host.name) + hostlvl, cores, nets, disks_io, temps, mounts = _split_host_metrics(host.name, latest) + ls = reg.last_seen_at if reg else None + stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds() + return await render_template( + "_host_metrics.html", + host=host, reg=reg, stale=stale, hostlvl=hostlvl, + cores=cores, nets=nets, disks_io=disks_io, temps=temps, mounts=mounts, + ) + + +@host_agent_bp.get("//charts") +@require_role(UserRole.viewer) +async def host_detail_charts(host_id: str): + """History-charts fragment — lazy-loaded so the date_bin query never blocks + the page shell; refreshed on a slow cadence and on range change.""" + since, range_key = parse_range(request.args.get("range")) + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return "", 404 + raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7) + series = await _history_for_host(session, host.name, since, raw_days=raw_days) + return await render_template("_host_charts.html", series=series, range_key=range_key) + + +def _new_token_pair() -> tuple[str, str]: + raw = secrets.token_urlsafe(32) + return raw, _hash_token(raw) + + +@host_agent_bp.get("/vitals/") +@require_role(UserRole.viewer) +async def host_vitals(host_id: str): + """Compact live vitals strip (CPU/MEM/DISK/LOAD + pressure) for the Hosts hub. + + Self-contained fragment, polled by hosts/detail.html. Renders nothing until + the agent reports — the Agent panel below then carries provisioning. + """ + async with current_app.db_sessionmaker() as db: + host = (await db.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return "", 404 + reg = (await db.execute(select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + latest = await _latest_metrics_for_host(db, host.name) if reg else {} + + # Recent series for the at-a-glance sparklines (cpu/mem/load host-level, + # disk from the root mount sub-resource). + series_raw: dict[str, list[float]] = {"cpu": [], "mem": [], "disk": [], "load": []} + if reg: + since = datetime.now(timezone.utc) - timedelta(hours=6) + spark_rows = (await db.execute( + select(PluginMetric).where( + PluginMetric.source_module == SOURCE_MODULE, + PluginMetric.recorded_at >= since, + or_( + and_(PluginMetric.resource_name == host.name, + PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))), + and_(PluginMetric.resource_name == host.name + ":/", + PluginMetric.metric_name == "disk_used_pct"), + ), + ).order_by(PluginMetric.recorded_at) + )).scalars().all() + for r in spark_rows: + if r.resource_name == host.name: + key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name) + if key: + series_raw[key].append(r.value) + else: + series_raw["disk"].append(r.value) + + from steward.core.status import sparkline_svg + hostlvl = latest.get(host.name, {}) + # At-a-glance disk = the root filesystem (what people actually care about), + # not the "worst" mount which is opaque. Worst stays on the full-metrics page. + disk_root = (latest.get(host.name + ":/", {}) or {}).get("disk_used_pct") + sparks = {k: sparkline_svg(v[-120:]) for k, v in series_raw.items()} + # Normalize load by core count so it's comparable across hardware (load/core + # as %). Cores derived from the per-core CPU sub-resources already collected. + cores = sum(1 for k in latest if k.startswith(host.name + ":core")) + load1 = hostlvl.get("load_1m") + load_per_core = round(load1 / cores * 100) if (cores and load1 is not None) else None + psi = { + "cpu": hostlvl.get("psi_cpu_some_avg10"), + "mem": hostlvl.get("psi_mem_some_avg10"), + "io": hostlvl.get("psi_io_some_avg10"), + } + ls = reg.last_seen_at if reg else None + reporting = bool(reg) and ls is not None + stale = reporting and (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds() + return await render_template( + "_host_vitals.html", + reg=reg, cpu=hostlvl.get("cpu_pct"), mem=hostlvl.get("mem_used_pct"), + disk_root=disk_root, load_per_core=load_per_core, sparks=sparks, psi=psi, + reporting=reporting, stale=stale, + ) + + +@host_agent_bp.get("/panel/") +@require_role(UserRole.viewer) +async def host_panel(host_id: str): + """Per-host agent management panel embedded into the Hosts hub via HTMX. + + Lifecycle actions (update / rotate / remove / re-provision) when the host + reports, otherwise provisioning — tied to the host's linked Ansible target. + The live vitals are a separate strip (host_vitals); this panel is management + only. Self-contained fragment (no base.html) so it can be hx-swapped in. + """ + from steward.core.capabilities import has_capability + from steward.models.ansible_inventory import AnsibleTarget + + async with current_app.db_sessionmaker() as db: + host = (await db.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return "", 404 + reg = (await db.execute(select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + target = (await db.execute(select(AnsibleTarget).where( + AnsibleTarget.host_id == host_id))).scalar_one_or_none() + + ls = reg.last_seen_at if reg else None + # "Deployed" is gated on the agent actually checking in (last_seen_at), NOT on + # the registration row — that row is minted before the playbook runs, so a + # failed provision would otherwise look deployed. + reporting = bool(reg) and ls is not None + stale = reporting and (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds() + + ansible_cfg = current_app.config.get("ANSIBLE", {}) + return await render_template( + "panel.html", + host=host, reg=reg, reporting=reporting, stale=stale, target=target, + ansible_available=has_capability("ansible.run_playbook"), + managed_key_set=bool((ansible_cfg.get("ssh_public_key") or "").strip()), + ) + + +@host_agent_bp.get("/fleet/") +@require_role(UserRole.admin) +async def fleet(): + from steward.core.capabilities import has_capability + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + + ansible_available = has_capability("ansible.run_playbook") + async with current_app.db_sessionmaker() as db: + regs = (await db.execute(select(HostAgentRegistration))).scalars().all() + hosts_by_id = { + h.id: h for h in (await db.execute( + select(Host).where(Host.id.in_([r.host_id for r in regs])))).scalars().all() + } if regs else {} + targets, groups = [], [] + if ansible_available: + targets = (await db.execute( + select(AnsibleTarget).order_by(AnsibleTarget.name))).scalars().all() + groups = (await db.execute( + select(AnsibleGroup).order_by(AnsibleGroup.name))).scalars().all() + + new_token = request.args.get("new_token") + new_host_id = request.args.get("host_id") + install_url = None + if new_token and new_host_id: + install_url = f"{public_base_url(request)}/plugins/host_agent/install.sh?token={new_token}" + + managed_key_set = bool( + (current_app.config.get("ANSIBLE", {}).get("ssh_public_key") or "").strip()) + + return await render_template( + "settings_list.html", + registrations=[ + {"reg": r, "host": hosts_by_id.get(r.host_id)} for r in regs + ], + new_token=new_token, + install_url=install_url, + ansible_available=ansible_available, + deploy_targets=targets, + deploy_groups=groups, + managed_key_set=managed_key_set, + ) + + +@host_agent_bp.post("/fleet/add-host") +@require_role(UserRole.admin) +async def add_host(): + form = await request.form + name = (form.get("name") or "").strip() + address = (form.get("address") or "").strip() + if not name: + return _error(400, "missing_name") + + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.name == name))).scalar_one_or_none() + if host is None: + host = Host(name=name, address=address) + session.add(host) + await session.flush() + existing = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host.id))).scalar_one_or_none() + if existing is not None: + await session.rollback() + return _error(400, "already_registered") + + raw, hashed = _new_token_pair() + reg = HostAgentRegistration(host_id=host.id, token_hash=hashed) + session.add(reg) + await session.commit() + host_id = host.id + + return redirect(url_for("host_agent.fleet") + f"?new_token={raw}&host_id={host_id}") + + +@host_agent_bp.post("/fleet//rotate-token") +@require_role(UserRole.admin) +async def rotate_token(host_id: str): + async with current_app.db_sessionmaker() as session: + reg = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + if reg is None: + return _error(404, "not_found") + raw, hashed = _new_token_pair() + reg.token_hash = hashed + reg.token_created_at = datetime.now(timezone.utc) + await session.commit() + return redirect(url_for("host_agent.fleet") + f"?new_token={raw}&host_id={host_id}") + + +@host_agent_bp.post("/fleet//delete") +@require_role(UserRole.admin) +async def delete_registration(host_id: str): + async with current_app.db_sessionmaker() as session: + reg = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + if reg is not None: + await session.delete(reg) + await session.commit() + return redirect(url_for("host_agent.fleet")) + + +# ── Deploy via Ansible (plugin↔core synergy via the capability registry) ────── + +async def _ensure_host_for_target(db, target) -> "Host": + """Find or create the Host that an AnsibleTarget should report under.""" + if getattr(target, "host_id", None): + h = await db.get(Host, target.host_id) + if h: + return h + h = (await db.execute( + select(Host).where(Host.name == target.name))).scalar_one_or_none() + if h is None: + h = Host(name=target.name, address=getattr(target, "address", "") or "") + db.add(h) + await db.flush() + return h + + +async def _mint_registration_token(db, host) -> str: + """Create or rotate the host's agent registration; return the raw token. + + Tokens are stored hashed (unrecoverable), so deploying always mints a fresh + token — the run installs the agent with it. Existing agents get rotated. + """ + reg = (await db.execute(select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host.id))).scalar_one_or_none() + raw, hashed = _new_token_pair() + if reg is None: + db.add(HostAgentRegistration(host_id=host.id, token_hash=hashed)) + else: + reg.token_hash = hashed + reg.token_created_at = datetime.now(timezone.utc) + return raw + + +@host_agent_bp.post("/deploy") +@require_role(UserRole.admin) +async def deploy_via_ansible(): + """Install/update the agent on Ansible inventory targets via the bundled + install playbook — the 'curl | sh becomes a button' synergy. + + Uses the core "ansible.run_playbook" capability (no hard import of the + runner) and injects a freshly-minted per-host token as an inventory hostvar. + """ + from steward.core.capabilities import has_capability, invoke_capability + from steward.ansible.inventory_gen import ( + fetch_scope_targets, generate_inventory, inventory_to_yaml) + from steward.ansible.sources import BUILTIN_SOURCE_NAME + + if not has_capability("ansible.run_playbook"): + return _error(400, "ansible_unavailable", "Ansible is not available") + + form = await request.form + scope = (form.get("inventory_scope", "") or "").strip() + try: + interval = max(5, int(form.get("agent_interval", "30"))) + except (TypeError, ValueError): + interval = 30 + if not (scope.startswith("steward:target:") + or scope.startswith("steward:group:") + or scope == "steward:all"): + return _error(400, "bad_scope", "Choose a target or group") + + url = public_base_url(request) + async with current_app.db_sessionmaker() as db: + targets = await fetch_scope_targets(db, scope) + if not targets: + return _error(400, "no_targets", "No Ansible targets in that scope") + # The read above autobegins the session transaction (SQLAlchemy 2.0), so + # mint into it directly and commit — a nested db.begin() would double-begin. + tokens: dict[str, str] = {} + for t in targets: + host = await _ensure_host_for_target(db, t) + tokens[t.name] = await _mint_registration_token(db, host) + inv = generate_inventory(targets) # built while ORM objects are still live + # Per-host token as a host var; steward_url goes in as an extra-var below. + for name, tok in tokens.items(): + inv["_meta"]["hostvars"].setdefault(name, {})["steward_token"] = tok + inventory_content = inventory_to_yaml(inv) + await db.commit() + + actor_role = UserRole(session.get("user_role", "viewer")) + run, _source, err = await invoke_capability( + "ansible.run_playbook", actor_role, + current_app._get_current_object(), # type: ignore[attr-defined] + source_name=BUILTIN_SOURCE_NAME, + playbook_path="host_agent/install.yml", + inventory_content=inventory_content, + inventory_scope=scope, + # extra-vars outrank the playbook's `vars:` defaults; token stays per-host. + params={"extra_vars_map": {"steward_url": url, "agent_interval": interval}}, + triggered_by=session.get("user_id"), + ) + if err: + return _error(400, "deploy_failed", err) + return redirect(url_for("ansible.run_detail", run_id=run.id)) + + +@host_agent_bp.post("/provision") +@require_role(UserRole.admin) +async def provision_via_ansible(): + """First-contact provisioning: create the steward account + install the + managed key + NOPASSWD sudo, then install the agent — over bootstrap + (password) auth. After this, hosts are reachable with the managed key. + + The bootstrap password is passed to the runner as a connection override and + is NOT persisted on the run row. + """ + from steward.core.capabilities import has_capability, invoke_capability + from steward.ansible.inventory_gen import ( + fetch_scope_targets, generate_inventory, inventory_to_yaml) + from steward.ansible.sources import BUILTIN_SOURCE_NAME + + if not has_capability("ansible.run_playbook"): + return _error(400, "ansible_unavailable", "Ansible is not available") + + ansible_cfg = current_app.config.get("ANSIBLE", {}) + pubkey = (ansible_cfg.get("ssh_public_key") or "").strip() + if not pubkey: + return _error(400, "no_managed_key", + "Generate a managed SSH key in Settings → Ansible first") + steward_user = (ansible_cfg.get("ssh_user") or "steward").strip() or "steward" + + form = await request.form + scope = (form.get("inventory_scope", "") or "").strip() + boot_user = (form.get("bootstrap_user", "") or "").strip() + boot_password = form.get("bootstrap_password", "") or "" + try: + interval = max(5, int(form.get("agent_interval", "30"))) + except (TypeError, ValueError): + interval = 30 + if not (scope.startswith("steward:target:") + or scope.startswith("steward:group:") + or scope == "steward:all"): + return _error(400, "bad_scope", "Choose a target or group") + if not boot_user or not boot_password: + return _error(400, "missing_bootstrap", + "Bootstrap user and password are required for first contact") + + url = public_base_url(request) + async with current_app.db_sessionmaker() as db: + targets = await fetch_scope_targets(db, scope) + if not targets: + return _error(400, "no_targets", "No Ansible targets in that scope") + # The read above autobegins the session transaction (SQLAlchemy 2.0), so + # mint into it directly and commit — a nested db.begin() would double-begin. + tokens: dict[str, str] = {} + for t in targets: + host = await _ensure_host_for_target(db, t) + tokens[t.name] = await _mint_registration_token(db, host) + inv = generate_inventory(targets) # built while ORM objects are still live + # Only the per-host token is a host var; the globals go in as extra-vars + # below (extra-vars outrank play vars, host vars do not). + for name, tok in tokens.items(): + inv["_meta"]["hostvars"].setdefault(name, {})["steward_token"] = tok + inventory_content = inventory_to_yaml(inv) + await db.commit() + + actor_role = UserRole(session.get("user_role", "viewer")) + run, _source, err = await invoke_capability( + "ansible.run_playbook", actor_role, + current_app._get_current_object(), # type: ignore[attr-defined] + source_name=BUILTIN_SOURCE_NAME, + playbook_path="host_agent/provision.yml", + inventory_content=inventory_content, + inventory_scope=scope, + # extra_vars_map → a JSON -e @file: highest precedence (beats play vars) + # and space-safe (steward_pubkey carries a comment with a space). + params={"extra_vars_map": { + "steward_url": url, + "steward_pubkey": pubkey, + "steward_user": steward_user, + "agent_interval": interval, + }}, + triggered_by=session.get("user_id"), + connection={"user": boot_user, "password": boot_password}, + ) + if err: + return _error(400, "provision_failed", err) + return redirect(url_for("ansible.run_detail", run_id=run.id)) + + +@host_agent_bp.post("/update") +@require_role(UserRole.admin) +async def update_via_ansible(): + """Refresh agent.py on already-installed hosts via the bundled update + playbook. Connects as the managed steward account — no token rotation, no + config rewrite (the host keeps its existing identity).""" + from steward.core.capabilities import has_capability, invoke_capability + from steward.ansible.sources import BUILTIN_SOURCE_NAME + + if not has_capability("ansible.run_playbook"): + return _error(400, "ansible_unavailable", "Ansible is not available") + + form = await request.form + scope = (form.get("inventory_scope", "") or "").strip() + if not (scope.startswith("steward:target:") + or scope.startswith("steward:group:") + or scope == "steward:all"): + return _error(400, "bad_scope", "Choose a target or group") + + url = public_base_url(request) + actor_role = UserRole(session.get("user_role", "viewer")) + run, _source, err = await invoke_capability( + "ansible.run_playbook", actor_role, + current_app._get_current_object(), # type: ignore[attr-defined] + source_name=BUILTIN_SOURCE_NAME, + playbook_path="host_agent/update.yml", + inventory_scope=scope, + params={"extra_vars": [f"steward_url={url}"]}, + triggered_by=session.get("user_id"), + ) + if err: + return _error(400, "update_failed", err) + return redirect(url_for("ansible.run_detail", run_id=run.id)) diff --git a/plugins/host_agent/scheduler.py b/plugins/host_agent/scheduler.py new file mode 100644 index 0000000..85056c1 --- /dev/null +++ b/plugins/host_agent/scheduler.py @@ -0,0 +1,78 @@ +# plugins/host_agent/scheduler.py +from __future__ import annotations +import logging +from datetime import datetime, timedelta, timezone +from typing import Iterable + +from sqlalchemy import select + +from steward.core.scheduler import ScheduledTask +from steward.models.hosts import Host +from .models import HostAgentRegistration + +logger = logging.getLogger(__name__) + + +def _filter_stale( + regs: Iterable, + *, + now: datetime, + stale_after_seconds: int, +) -> list: + """Pure staleness filter: returns the subset with last_seen_at strictly + older than (now - stale_after_seconds). Rows with last_seen_at=None are + never stale (they are unregistered-in-practice).""" + cutoff = now - timedelta(seconds=stale_after_seconds) + return [ + r for r in regs + if r.last_seen_at is not None and r.last_seen_at < cutoff + ] + + +async def find_stale_registrations(app, stale_after_seconds: int = 180) -> list[dict]: + async with app.db_sessionmaker() as session: + all_regs = (await session.execute( + select(HostAgentRegistration) + )).scalars().all() + stale_rows = _filter_stale( + all_regs, + now=datetime.now(timezone.utc), + stale_after_seconds=stale_after_seconds, + ) + if not stale_rows: + return [] + hosts = { + h.id: h for h in (await session.execute( + select(Host).where(Host.id.in_([r.host_id for r in stale_rows])) + )).scalars().all() + } + return [ + { + "host_id": r.host_id, + "host_name": hosts[r.host_id].name if r.host_id in hosts else "?", + "last_seen_at": r.last_seen_at, + } + for r in stale_rows + ] + + +def make_task(app) -> ScheduledTask: + async def _check_stale(): + try: + stale = await find_stale_registrations(app) + except Exception: + logger.exception("host_agent stale check failed") + return + if stale: + logger.info( + "host_agent: %d stale agent(s): %s", + len(stale), + [s["host_name"] for s in stale], + ) + + return ScheduledTask( + name="host_agent_stale_check", + coro_factory=_check_stale, + interval_seconds=60, + run_on_startup=False, + ) diff --git a/plugins/host_agent/templates/_host_charts.html b/plugins/host_agent/templates/_host_charts.html new file mode 100644 index 0000000..72899ee --- /dev/null +++ b/plugins/host_agent/templates/_host_charts.html @@ -0,0 +1,6 @@ +{# History data-only fragment. Swapped into the hidden #hm-charts-data poller; + the script runs and feeds the persistent charts created by host_detail.html, + updating them in place (no canvas swap → no flicker / no re-animation). #} + diff --git a/plugins/host_agent/templates/_host_metrics.html b/plugins/host_agent/templates/_host_metrics.html new file mode 100644 index 0000000..c06c49b --- /dev/null +++ b/plugins/host_agent/templates/_host_metrics.html @@ -0,0 +1,182 @@ +{# Current-state fragment for the full-metrics page — polled live via HTMX. + Self-contained (defines its own format macros) so each poll re-renders the + latest snapshot the server has. #} +{% macro fmt_bps(v) %} +{%- if v is none -%}— +{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s +{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s +{%- else -%}{{ "%.0f"|format(v) }} B/s +{%- endif -%} +{% endmacro %} +{% macro fmt_bytes(v) %} +{%- if v is none -%}— +{%- elif v >= 1125899906842624 -%}{{ "%.1f"|format(v / 1125899906842624) }} PB +{%- elif v >= 1099511627776 -%}{{ "%.1f"|format(v / 1099511627776) }} TB +{%- elif v >= 1073741824 -%}{{ "%.1f"|format(v / 1073741824) }} GB +{%- elif v >= 1048576 -%}{{ "%.0f"|format(v / 1048576) }} MB +{%- else -%}{{ "%.0f"|format(v / 1024) }} KB +{%- endif -%} +{% endmacro %} +{% macro bar(pct) %} +{%- set p = pct if pct is not none else 0 -%} +
+
+
+{% endmacro %} + +{# ── Identity / metadata ─────────────────────────────────────────────────── #} +
+ {% if stale %}stale{% else %}live{% endif %} + Address {{ host.address or "—" }} + {% if reg %} + {% if reg.distro %}OS {{ reg.distro }}{% endif %} + {% if reg.kernel %}Kernel {{ reg.kernel }}{% endif %} + {% if reg.arch %}Arch {{ reg.arch }}{% endif %} + {% if reg.agent_version %}Agent v{{ reg.agent_version }}{% endif %} + {% set up = hostlvl.get('uptime_secs') %} + {% if up %}Uptime {{ (up // 86400)|int }}d {{ ((up % 86400) // 3600)|int }}h{% endif %} + Last seen {{ reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") ~ " UTC" if reg.last_seen_at else "never" }} + {% else %} + No agent registration found for this host. + {% endif %} +
+ +{% if not hostlvl %} +
No metrics reported yet. Once the agent checks in, CPU, memory, network, disk I/O, temperatures, and pressure will appear here.
+{% else %} + +{# ── Current headline gauges ─────────────────────────────────────────────── #} +
+
+
CPU
+ {% set c = hostlvl.get('cpu_pct') %} + {% if c is not none %}{{ "%.0f"|format(c) }}%{% else %}—{% endif %} +
{{ bar(c) }}
+
+
+
Memory
+ {% set mp = hostlvl.get('mem_used_pct') %} + {% if mp is not none %}{{ "%.0f"|format(mp) }}%{% else %}—{% endif %} +
{{ bar(mp) }}
+
+ avail {{ fmt_bytes(hostlvl.get('mem_available_bytes')) }} · cache {{ fmt_bytes(hostlvl.get('mem_cached_bytes')) }} · swap {{ fmt_bytes(hostlvl.get('swap_used_bytes')) }} +
+
+
+
Load
+ {% if hostlvl.get('load_1m') is not none %}{{ "%.2f"|format(hostlvl.get('load_1m')) }}{% else %}—{% endif %} +
+ 5m {% if hostlvl.get('load_5m') is not none %}{{ "%.2f"|format(hostlvl.get('load_5m')) }}{% else %}—{% endif %} · + 15m {% if hostlvl.get('load_15m') is not none %}{{ "%.2f"|format(hostlvl.get('load_15m')) }}{% else %}—{% endif %} +
+
+
+
Network
+
+
↓ {{ fmt_bps(hostlvl.get('net_rx_bps')) }}
+
↑ {{ fmt_bps(hostlvl.get('net_tx_bps')) }}
+
+
+
+
Disk I/O
+
+
rd {{ fmt_bps(hostlvl.get('disk_read_bps')) }}
+
wr {{ fmt_bps(hostlvl.get('disk_write_bps')) }}
+
+
+ {% if hostlvl.get('temp_c_max') is not none %} +
+
Temp (max)
+ {% set t = hostlvl.get('temp_c_max') %} + {{ "%.0f"|format(t) }}°C +
+ {% endif %} + {% if hostlvl.get('psi_mem_some_avg10') is not none %} +
+
Pressure (10s)
+
+
mem {{ "%.1f"|format(hostlvl.get('psi_mem_some_avg10')) }}%
+ {% if hostlvl.get('psi_cpu_some_avg10') is not none %}
cpu {{ "%.1f"|format(hostlvl.get('psi_cpu_some_avg10')) }}%
{% endif %} + {% if hostlvl.get('psi_io_some_avg10') is not none %}
io {{ "%.1f"|format(hostlvl.get('psi_io_some_avg10')) }}%
{% endif %} +
+
+ {% endif %} +
+ +{# ── Per-core CPU ─────────────────────────────────────────────────────────── #} +{% if cores %} +
+
Per-core CPU
+
+ {% for idx, pct in cores %} +
+
+ core {{ idx }}{% if pct is not none %}{{ "%.0f"|format(pct) }}%{% else %}—{% endif %} +
+ {{ bar(pct) }} +
+ {% endfor %} +
+
+{% endif %} + +{# ── Filesystems ──────────────────────────────────────────────────────────── #} +{% if mounts %} +
+
Filesystems
+ {% for mount, m in mounts.items() %} +
+
+ {{ mount }} + {{ fmt_bytes(m.get('disk_used_bytes')) }} / {{ fmt_bytes(m.get('disk_total_bytes')) }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %} +
+ {{ bar(m.get('disk_used_pct')) }} +
+ {% endfor %} +
+{% endif %} + +{# ── Interfaces / disks (short panels, side by side at natural height) ─────── #} +{% if nets or disks_io %} +
+ {% if nets %} +
+
Interfaces
+ {% for iface, m in nets.items() %} +
+ {{ iface }} + ↓ {{ fmt_bps(m.get('net_rx_bps')) }} · ↑ {{ fmt_bps(m.get('net_tx_bps')) }} +
+ {% endfor %} +
+ {% endif %} + {% if disks_io %} +
+
Disks
+ {% for dev, m in disks_io.items() %} +
+ {{ dev }} + rd {{ fmt_bps(m.get('disk_read_bps')) }} · wr {{ fmt_bps(m.get('disk_write_bps')) }} +
+ {% endfor %} +
+ {% endif %} +
+{% endif %} + +{# ── Temperatures (full width; cores flow into a compact multi-column grid) ── #} +{% if temps %} +
+
Temperatures
+
+ {% for label, c in temps.items() %} +
+ {{ label }} + {% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %} +
+ {% endfor %} +
+
+{% endif %} + +{% endif %} diff --git a/plugins/host_agent/templates/_host_vitals.html b/plugins/host_agent/templates/_host_vitals.html new file mode 100644 index 0000000..eea0c37 --- /dev/null +++ b/plugins/host_agent/templates/_host_vitals.html @@ -0,0 +1,35 @@ +{# Host vitals strip — compact CPU/MEM/DISK/LOAD + pressure, live-polled into the + host hub. Renders nothing until the agent reports (the Agent panel below then + shows provisioning). #} +{% if reporting %} +{% set lbl = "font-size:0.68rem;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em;" %} +{% macro vital(label, value, text, kind, spark) %} +
+
{{ label }}
+
{{ text }}
+
{{ spark | safe }}
+
+{% endmacro %} +
+ {{ vital("CPU", cpu, ('%.0f%%'|format(cpu) if cpu is not none else '—'), 'cpu', sparks.cpu) }} + {{ vital("Memory", mem, ('%.0f%%'|format(mem) if mem is not none else '—'), 'mem', sparks.mem) }} + {{ vital("Disk /", disk_root, ('%.0f%%'|format(disk_root) if disk_root is not none else '—'), 'disk', sparks.disk) }} + {{ vital("Load /core", load_per_core, ('%d%%'|format(load_per_core) if load_per_core is not none else '—'), 'load', sparks.load) }} + +
+ + {% if stale %}stale{% else %}live{% endif %} + {% if reg and reg.agent_version %}v{{ reg.agent_version }}{% endif %} + + {% if psi.cpu is not none or psi.mem is not none or psi.io is not none %} + + Pressure 10s + cpu {{ '%.0f%%'|format(psi.cpu) if psi.cpu is not none else '—' }} + · mem {{ '%.0f%%'|format(psi.mem) if psi.mem is not none else '—' }} + · io {{ '%.0f%%'|format(psi.io) if psi.io is not none else '—' }} + + {% endif %} + {% if reg and reg.last_seen_at %}seen {{ reg.last_seen_at.strftime("%H:%M:%S") }} UTC{% endif %} +
+
+{% endif %} diff --git a/plugins/host_agent/templates/host_detail.html b/plugins/host_agent/templates/host_detail.html new file mode 100644 index 0000000..359b1fc --- /dev/null +++ b/plugins/host_agent/templates/host_detail.html @@ -0,0 +1,108 @@ +{% extends "base.html" %} +{% from "_macros.html" import crumbs %} +{% block title %}{{ host.name }} — Metrics — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), (host.name, "/hosts/" ~ host.id), ("Metrics", "")]) }}{% endblock %} +{% block content %} +{# Shell only: current-state + history data stream in via HTMX so the heavy + history query never blocks first paint and the data refreshes live (≈ the + agent's push cadence). The chart CANVASES live here permanently — only their + data is polled and applied in place, so refreshes never re-create/flicker. #} +
+

{{ host.name }}

+ {% include "_time_range.html" %} +
+ +{# Current state — lazy + polled live. Atomic innerHTML swap of fixed-height + cards, so values update without a layout jump. #} +
+
Loading metrics…
+
+ +{# History charts — persistent canvases (created once below); only data is polled. #} +
+
+
Utilization % — last {{ current_range }}
+
+
+
+
Throughput (B/s) — last {{ current_range }}
+
+
+
+
Load & pressure — last {{ current_range }}
+
+
+
+{# Hidden poller: fetches the history series and applies it to the charts above + (no visible swap). Lazy on load → never blocks paint; refreshed slowly + on + range change. #} + + + +{% endblock %} diff --git a/plugins/host_agent/templates/install.sh.j2 b/plugins/host_agent/templates/install.sh.j2 new file mode 100644 index 0000000..02d55be --- /dev/null +++ b/plugins/host_agent/templates/install.sh.j2 @@ -0,0 +1,81 @@ +#!/bin/sh +# Steward host agent installer +# Generated for: {{ host_name }} ({{ host_address }}) +# Steward URL: {{ url }} +set -e + +STEWARD_URL="{{ url }}" +AGENT_TOKEN="{{ token }}" +AGENT_VERSION="{{ agent_version }}" + +AGENT_USER="steward-agent" +AGENT_DIR="/usr/local/lib/steward-agent" +CONF_FILE="/etc/steward-agent.conf" +UNIT_FILE="/etc/systemd/system/steward-agent.service" + +# ── preflight ──────────────────────────────────────────────────────────────── +[ "$(id -u)" = "0" ] || { echo "must run as root (use sudo)"; exit 1; } +command -v systemctl >/dev/null 2>&1 || { echo "systemd not found — unsupported init system"; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo "python3 not found — install python3 first"; exit 1; } + +# Handle --uninstall +if [ "${1:-}" = "--uninstall" ]; then + systemctl disable --now steward-agent.service 2>/dev/null || true + rm -f "$UNIT_FILE" "$CONF_FILE" + rm -rf "$AGENT_DIR" + systemctl daemon-reload + # keep the system user — removing it risks orphaned files elsewhere + echo "uninstalled" + exit 0 +fi + +# ── create system user ─────────────────────────────────────────────────────── +if ! id "$AGENT_USER" >/dev/null 2>&1; then + useradd --system --no-create-home --shell /usr/sbin/nologin "$AGENT_USER" +fi + +# ── drop the agent file ────────────────────────────────────────────────────── +mkdir -p "$AGENT_DIR" +curl -sSL "${STEWARD_URL}/plugins/host_agent/agent.py" -o "$AGENT_DIR/agent.py" +chmod 0755 "$AGENT_DIR/agent.py" +chown root:root "$AGENT_DIR/agent.py" + +# ── write config ───────────────────────────────────────────────────────────── +cat > "$CONF_FILE" < "$UNIT_FILE" < via HTMX. Self-contained. #} +{% set scope = "steward:target:" ~ target.id if target else "" %} +
+
+

Agent

+ {% if reporting %} + + {{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %} + · last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }} + + {% elif reg %} + pending — no check-in yet + {% endif %} +
+ + {% if reporting %} + {# Reporting: live vitals are shown by the vitals strip above; this panel is + lifecycle/management only. #} +
+ Full metrics → + {% if session.user_role == 'admin' %} + {% if ansible_available and target %} +
+ + +
+ {% endif %} +
+ +
+
+ +
+ {% endif %} +
+ {% if ansible_available and not target %} +

+ Link an Ansible target (above) to enable one-click agent updates. +

+ {% endif %} + + {% if session.user_role == 'admin' and ansible_available and target and managed_key_set %} +
+ Re-provision (reinstall the steward account + managed key, then the agent) +

+ Use after regenerating the managed key, or if SSH auth as steward breaks. + Connects over a one-time bootstrap user + password (not stored). +

+
+ +
+ + +
+
+ + +
+ +
+
+ {% endif %} + + {% else %} + {# ── Not reporting: pending (token minted, no check-in) or never installed ── #} + {% if reg %} +
+ A token was minted for this host but no metrics have arrived yet — the + deploy may still be running, or it failed. Open the latest + Ansible run to check, then retry below. Live metrics appear here + once the agent checks in. +
+ {% else %} +

+ No agent installed. The agent reports CPU, memory, disk, network and more back to Steward. +

+ {% endif %} + + {% if session.user_role != 'admin' %} +

An admin can install the agent here.

+ + {% elif not ansible_available %} +

+ Ansible isn't available, so the agent can't be deployed from here. Install it manually with the + curl install command. +

+ + {% elif not managed_key_set %} +
+ ⚠ + No managed SSH key yet — Steward needs one to log into hosts. +
+ + +
+
+ + {% elif not target %} +

+ Link or create an Ansible target for this host (in the Ansible section below) first — provisioning + needs an SSH connection. +

+ + {% else %} +

+ Deploys the agent to {{ target.name }} by running an Ansible playbook. Steward mints + the host's API token automatically; you'll watch the run live. +

+ {# Provision: brand-new host (creates steward account + key over a one-time password) #} +
+ +
Provision (first contact — fresh host):
+
+ + +
+
+ + +
+ +
+ {# Install: host already has the steward account (managed key works) #} +
+ + Already provisioned? + +
+ {% endif %} + + {% if reg and session.user_role == 'admin' %} +
+ +
+ {% endif %} + {% endif %} +
diff --git a/plugins/host_agent/templates/settings_list.html b/plugins/host_agent/templates/settings_list.html new file mode 100644 index 0000000..93cf0d4 --- /dev/null +++ b/plugins/host_agent/templates/settings_list.html @@ -0,0 +1,175 @@ +{% extends "base.html" %} +{% from "_macros.html" import crumbs %} +{% block title %}Agent fleet — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), ("Agent fleet", "")]) }}{% endblock %} +{% block content %} +
+

Agent fleet

+
+

+ Bulk agent operations across your inventory. For a single host, use its + host page — provisioning and metrics live there. +

+ +{% if new_token %} +
+

New token — copy this install command now

+

+ This is the only time the raw token is shown. Rotate if you lose it. +

+ 
curl -sSL '{{ install_url }}' | sudo sh
+
+{% endif %} + +
+
+
+ + +
+
+ + +
+ +
+
+ +{% if ansible_available %} +{% set scope_select %} + + +{% endset %} + +
+

Agent lifecycle via Ansible

+

+ These actions run bundled Ansible playbooks to deploy and maintain the + Steward monitoring agent on your inventory hosts. + Steward generates each host's API token automatically and connects over SSH as the managed + steward account — you'll be dropped into the live Ansible run to watch it. +

+ + {% if not managed_key_set %} +
+ ⚠ + + No managed SSH key yet — Steward needs one to log into hosts as steward. + +
+ + +
+
+ {% elif not (deploy_targets or deploy_groups) %} +

+ No Ansible inventory targets yet. Add some under Ansible → Inventory. +

+ {% endif %} +
+ +{% if managed_key_set and (deploy_targets or deploy_groups) %} +
+

1 · Provision a fresh host

+

+ First contact for a brand-new host. Connects with a one-time bootstrap + user + password (used for this run only, never stored), creates the steward + login account with passwordless sudo, installs the managed SSH key, then installs the + agent. Run this once per host. +

+
+
{{ scope_select }}
+
+ + +
+
+ + +
+
+ + +
+ +
+
+ +
+

2 · Install / enroll agent

+

+ For a host that's already provisioned (has the steward account) but has no + agent yet — or to re-enroll one. Connects as steward with the managed key, + mints a fresh token, and installs the agent. No curl | sh needed. +

+
+
{{ scope_select }}
+
+ + +
+ +
+
+ +
+

3 · Update agents

+

+ Refresh the agent binary on hosts already running it. Connects as steward + with the managed key and restarts the service — the token and config are left untouched, + so identity is preserved. +

+
+
{{ scope_select }}
+ +
+
+{% endif %} +{% endif %} + +
+ + + + + + + + + {% for item in registrations %} + + + + + + + + + {% else %} + + {% endfor %} + +
HostReported IPAgent versionDistroLast seenActions
{{ item.host.name if item.host else item.reg.host_id }}{{ item.reg.host_ip or "—" }}{{ item.reg.agent_version or "—" }}{{ item.reg.distro or "—" }}{{ item.reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") if item.reg.last_seen_at else "never" }} +
+ +
+
+ +
+
No hosts registered. Add one above.
+
+{% endblock %} diff --git a/plugins/host_agent/templates/widget_history.html b/plugins/host_agent/templates/widget_history.html new file mode 100644 index 0000000..1c13783 --- /dev/null +++ b/plugins/host_agent/templates/widget_history.html @@ -0,0 +1,88 @@ +{# host_agent history-graph widget — the host-view utilization chart, embedded + on a dashboard. Fills the (resizable) panel; epoch-ms linear axis so no + Chart.js date adapter is needed. #} +{% set has_data = host and (series.cpu_pct or series.mem_used_pct or series.disk_root) %} +{% if not host %} +
+ No host selected. Edit this dashboard and pick a host for this graph. +
+{% elif not has_data %} +
+ No agent metrics for {{ host.name }} in the last {{ hours }}h yet. +
+{% else %} +
+ {# Name the host the graph represents (panel title is generic) + a live range + toggle that re-requests this widget without entering edit mode. #} +
+ {% if session.user_id %} + {{ host.name }} + {% else %} + {{ host.name }} + {% endif %} + + {% for opt in [1, 6, 24] %} + + {% endfor %} + +
+
+ +
+
+ +{% endif %} diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html new file mode 100644 index 0000000..163823f --- /dev/null +++ b/plugins/host_agent/templates/widget_table.html @@ -0,0 +1,77 @@ +{# host_agent widget — fleet glance. One horizontal row per host, zebra-striped + so adjacent hosts read as distinct groupings: health dot + host name on the + left, metric cells (cpu/mem/disk/load) each with a trend sparkline laid out + horizontally on the right. The name wraps (never hard-truncated) but the row + stays horizontal. #} +{% if rows %} +{% macro metric_cell(label, value, fmt, svg, style="", title="") %} +
+
{{ label }}
+
+ {% if value is not none %}{{ fmt|format(value) }}{% else %}—{% endif %} +
+
{{ svg | safe }}
+
+{% endmacro %} +{# Human-readable throughput, matching the host-detail page. #} +{% macro fmt_bps(v) %} +{%- if v is none -%}— +{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s +{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s +{%- else -%}{{ "%.0f"|format(v) }} B/s +{%- endif -%} +{% endmacro %} +{# Two-line down/up (or read/write) throughput cell. #} +{% macro io_cell(label, down_sym, down_val, up_sym, up_val, title="") %} +
+
{{ label }}
+
+
{{ down_sym }} {{ fmt_bps(down_val) }}
+
{{ up_sym }} {{ fmt_bps(up_val) }}
+
+
+{% endmacro %} +
+ {% for r in rows %} + {% set stale = r.stale %} +
+ {# Left — health dot + full host name (wraps, never truncated) + last seen #} +
+ + + {% if session.user_id %} + {{ r.host.name }} + {% else %} + {{ r.host.name }} + {% endif %} + + {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }} + + +
+ {# Right — metric cells with value + trend, laid out horizontally #} +
+ {{ metric_cell("cpu", r.cpu_pct, "%.0f%%", r.sparks.cpu, threshold_style(r.cpu_pct, 'cpu'), "Average CPU utilization") }} + {{ metric_cell("mem", r.mem_used_pct, "%.0f%%", r.sparks.mem, threshold_style(r.mem_used_pct, 'mem'), "Memory in use") }} + {{ metric_cell("disk /", r.disk_root, "%.0f%%", r.sparks.disk, threshold_style(r.disk_root, 'disk'), "Root filesystem (/) usage") }} + {{ metric_cell("load", r.load_1m, "%.2f", r.sparks.load, "", "Load average (1m)") }} + {# Extra agent metrics — shown only when the host reports them (VMs/containers + often have no temp sensor, etc.) so absent data doesn't clutter the row. #} + {% if r.net_rx_bps is not none or r.net_tx_bps is not none %} + {{ io_cell("net", "↓", r.net_rx_bps, "↑", r.net_tx_bps, "Network throughput (down / up)") }} + {% endif %} + {% if r.disk_read_bps is not none or r.disk_write_bps is not none %} + {{ io_cell("disk i/o", "rd", r.disk_read_bps, "wr", r.disk_write_bps, "Disk I/O (read / write)") }} + {% endif %} + {% if r.temp_max is not none %} + {{ metric_cell("temp", r.temp_max, "%.0f°C", "", threshold_style(r.temp_max, 'temp'), "Hottest sensor") }} + {% endif %} +
+
+ {% endfor %} +
+{% else %} +

No hosts with agent data yet.

+{% endif %} diff --git a/plugins/index.yaml b/plugins/index.yaml new file mode 100644 index 0000000..3d5fe8e --- /dev/null +++ b/plugins/index.yaml @@ -0,0 +1,74 @@ +# steward-plugins / index.yaml +# +# Plugin catalog for Steward. +# Fetched by the app at: Settings → Plugins → Plugin Catalog +# +# After updating an entry, commit and push — changes are live within 5 minutes +# (the app caches this file in-process for 300 seconds). +# +# See docs/plugins/index.yaml.example in the main app repo for the full schema. + +version: 1 +updated: "2026-03-22" + +plugins: + + - name: docker + version: "2.0.0" + description: "Per-host Docker container status + resource usage, collected by the Steward host agent" + author: "Steward" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker" + download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v2.0.0/docker.zip" + checksum_sha256: "" + tags: + - containers + - docker + - infrastructure + + - name: traefik + version: "1.0.0" + description: "Traefik reverse proxy metrics and access log integration" + author: "Steward" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik" + download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip" + checksum_sha256: "" + tags: + - proxy + - metrics + - access-log + + - name: unifi + version: "1.0.0" + description: "UniFi Network controller integration — WAN health, devices, clients, DPI" + author: "Steward" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi" + download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/unifi-v1.0.0/unifi.zip" + checksum_sha256: "" + tags: + - network + - unifi + - ubiquiti + + - name: ups + version: "1.0.0" + description: "UPS monitoring via NUT (Network UPS Tools) with Ansible shutdown automation" + author: "Steward" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/ups" + download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/ups-v1.0.0/ups.zip" + checksum_sha256: "" + tags: + - ups + - power + - nut diff --git a/plugins/snmp/__init__.py b/plugins/snmp/__init__.py new file mode 100644 index 0000000..0222aee --- /dev/null +++ b/plugins/snmp/__init__.py @@ -0,0 +1,28 @@ +# plugins/snmp/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + + +def get_scheduled_tasks() -> list: + from .scheduler import make_poll_task + return [make_poll_task(_app)] + + +def get_blueprint(): + from .routes import snmp_bp + return snmp_bp + + +def get_nav() -> list[dict]: + # Surfaces in the sidebar's Infrastructure group (SNMP device readings). + return [{"label": "SNMP", "href": "/plugins/snmp/"}] diff --git a/plugins/snmp/plugin.yaml b/plugins/snmp/plugin.yaml new file mode 100644 index 0000000..7ef3683 --- /dev/null +++ b/plugins/snmp/plugin.yaml @@ -0,0 +1,59 @@ +# plugins/snmp/plugin.yaml +name: snmp +version: "1.0.0" +description: "SNMP polling for network devices — switches, routers, printers, UPSes, etc." +author: "Steward" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/snmp" +tags: + - snmp + - network + - monitoring + +config: + poll_interval_seconds: 60 + # Each device is polled independently. OIDs must resolve to numeric values. + # + # `host` is the SNMP poll target (IP/hostname we query). A device also shows + # up on a Steward Host's detail page when it maps to that host. By default + # that mapping is implicit: `host` is matched against the Host's address or + # name. To bind explicitly instead — e.g. when you poll by IP but the Host is + # recorded by DNS name, or for an SNMP-only switch/PDU/UPS — add ONE of: + # host_id: "" # exact Host.id match + # steward_host: "switch01" # Host name OR address (case-insensitive) + # An explicit binding is exclusive: the device then maps ONLY to that host, + # never implicitly to another by a coincidental address string. + devices: + - name: "core-switch" + host: "192.168.1.1" + port: 161 + community: "public" + version: "2c" # "1", "2c", or "3" + oids: + - oid: "1.3.6.1.2.1.1.3.0" + label: "uptime_centisecs" + - oid: "1.3.6.1.2.1.2.2.1.10.1" + label: "if1_in_octets" + - oid: "1.3.6.1.2.1.2.2.1.16.1" + label: "if1_out_octets" + # Example: network UPS with SNMP management card (RFC 1628 UPS-MIB). + # upsBatteryStatus values: 1=unknown, 2=batteryNormal, 3=batteryLow, 4=batteryDepleted. + # Uncomment and point at your UPS to enable. + # - name: "ups" + # host: "192.168.1.10" + # port: 161 + # community: "public" + # version: "2c" + # oids: + # - oid: "1.3.6.1.2.1.33.1.2.1.0" + # label: "battery_status" + # - oid: "1.3.6.1.2.1.33.1.2.2.0" + # label: "seconds_on_battery" + # - oid: "1.3.6.1.2.1.33.1.2.3.0" + # label: "runtime_minutes_remaining" + # - oid: "1.3.6.1.2.1.33.1.2.4.0" + # label: "battery_charge_pct" + # - oid: "1.3.6.1.2.1.33.1.3.3.1.3.1" + # label: "input_voltage_dV" diff --git a/plugins/snmp/poller.py b/plugins/snmp/poller.py new file mode 100644 index 0000000..c84de2c --- /dev/null +++ b/plugins/snmp/poller.py @@ -0,0 +1,171 @@ +# plugins/snmp/poller.py +""" +Asynchronous SNMP GET helper. + +Requires pysnmp-lextudio (the maintained pysnmp fork), bundled into the Docker +image via the `snmp` extra (`pip install .[snmp]`): + pip install 'steward[snmp]' + +If pysnmp is not installed, poll_device() returns an empty dict and logs a +warning — SNMP polling is then simply disabled, nothing else breaks. +""" +from __future__ import annotations +import logging + +logger = logging.getLogger(__name__) + + +def _pysnmp_available() -> bool: + try: + import pysnmp # noqa: F401 + return True + except ImportError: + return False + + +def _mp_model(version: str) -> int: + """Map an SNMP version string to pysnmp's mpModel int (0 = v1, 1 = v2c).""" + return 0 if version == "1" else 1 + + +def _close_engine(engine) -> None: + """Release the engine's UDP transport socket. + + pysnmp opens a UDP socket per ``SnmpEngine`` and never closes it on its own. + Since we build a fresh engine for every poll, an unclosed engine leaks one + file descriptor each scheduler tick; over hours of polling that exhausts the + process fd limit (``OSError: [Errno 24] Too many open files``), which also + takes down the app's listening socket. So close it explicitly here. + + The close method/attribute names differ across pysnmp majors (6.2.x lextudio + is camelCase, canonical 7.x is snake_case), so probe for whatever exists. + All paths are best-effort — a failed close must never break the poll loop. + """ + # 7.x exposes a convenience close directly on the engine. + for meth in ("close_dispatcher", "closeDispatcher"): + fn = getattr(engine, meth, None) + if callable(fn): + try: + fn() + except Exception: + pass + return + # 6.2.x: go through the transport dispatcher. + for attr in ("transport_dispatcher", "transportDispatcher"): + dispatcher = getattr(engine, attr, None) + if dispatcher is None: + continue + for meth in ("close_dispatcher", "closeDispatcher"): + fn = getattr(dispatcher, meth, None) + if callable(fn): + try: + fn() + except Exception: + pass + return + return + + +async def poll_device( + host: str, + port: int, + community: str, + version: str, + oids: list[dict], +) -> dict[str, float]: + """Perform an SNMP GET for each OID and return ``{label: float_value}``. + + Non-numeric OIDs (strings, etc.) are skipped. Returns an empty dict on any + error (unreachable host, wrong community, …) so a flaky device never breaks + the poll loop. + + pysnmp's HLAPI is asyncio-only as of v6. The import path moved between major + versions, so we support both rather than let a dependency bump silently + re-break polling: + • pysnmp-lextudio 6.2.x → ``pysnmp.hlapi.asyncio`` (``getCmd`` + a directly + constructed ``UdpTransportTarget``). + • canonical pysnmp 7.x → ``pysnmp.hlapi.v3arch.asyncio`` (``get_cmd`` + the + async ``UdpTransportTarget.create``). + """ + if not _pysnmp_available(): + logger.warning("pysnmp not installed — SNMP polling disabled. " + "Install with: pip install 'steward[snmp]'") + return {} + + try: + # canonical pysnmp 7.x + from pysnmp.hlapi.v3arch.asyncio import ( + CommunityData, + ContextData, + ObjectIdentity, + ObjectType, + SnmpEngine, + UdpTransportTarget, + get_cmd as _get_cmd, + ) + _transport_is_async = True + except ImportError: + # pysnmp-lextudio 6.2.x + from pysnmp.hlapi.asyncio import ( + CommunityData, + ContextData, + ObjectIdentity, + ObjectType, + SnmpEngine, + UdpTransportTarget, + getCmd as _get_cmd, + ) + _transport_is_async = False + + engine = SnmpEngine() + + # Always release the engine's UDP socket — see _close_engine for why. + try: + # Same host/port for every OID on this device, so build the transport once. + try: + if _transport_is_async: + transport = await UdpTransportTarget.create((host, port), timeout=5, retries=1) + else: + transport = UdpTransportTarget((host, port), timeout=5, retries=1) + except Exception as exc: + logger.debug("SNMP transport setup failed for %s:%s: %s", host, port, exc) + return {} + + results: dict[str, float] = {} + + for oid_cfg in oids: + oid = oid_cfg["oid"] + label = oid_cfg.get("label") or oid + scale = float(oid_cfg.get("scale", 1.0)) + + try: + error_indication, error_status, error_index, var_binds = await _get_cmd( + engine, + CommunityData(community, mpModel=_mp_model(version)), + transport, + ContextData(), + ObjectType(ObjectIdentity(oid)), + ) + except Exception as exc: + logger.debug("SNMP GET %s@%s OID %s failed: %s", host, port, oid, exc) + continue + + if error_indication: + logger.debug("SNMP error %s@%s OID %s: %s", host, port, oid, error_indication) + continue + if error_status: + logger.debug("SNMP status %s@%s OID %s: %s at %s", + host, port, oid, error_status.prettyPrint(), + error_index and var_binds[int(error_index) - 1][0] or "?") + continue + + for _, val in var_binds: + try: + results[label] = float(val) * scale + except (TypeError, ValueError): + # Non-numeric type (e.g. OctetString description) — skip. + logger.debug("SNMP non-numeric value for %s label=%s: %r", oid, label, val) + + return results + finally: + _close_engine(engine) diff --git a/plugins/snmp/routes.py b/plugins/snmp/routes.py new file mode 100644 index 0000000..6157338 --- /dev/null +++ b/plugins/snmp/routes.py @@ -0,0 +1,215 @@ +# plugins/snmp/routes.py +from __future__ import annotations +from datetime import datetime, timedelta, timezone + +from quart import Blueprint, current_app, render_template, request +from sqlalchemy import func, select + +from steward.auth.middleware import require_role +from steward.models.users import UserRole +from steward.models.metrics import PluginMetric + +snmp_bp = Blueprint("snmp", __name__, template_folder="templates") + + +async def _latest_readings(db, device_names: list[str]) -> dict[str, dict[str, float]]: + """Return {device_name: {label: latest_value}} for all configured devices.""" + if not device_names: + return {} + + # Latest value per (resource_name, metric_name) pair + subq = ( + select( + PluginMetric.resource_name, + PluginMetric.metric_name, + func.max(PluginMetric.recorded_at).label("latest_at"), + ) + .where(PluginMetric.source_module == "snmp") + .where(PluginMetric.resource_name.in_(device_names)) + .group_by(PluginMetric.resource_name, PluginMetric.metric_name) + .subquery() + ) + result = await db.execute( + select(PluginMetric).join( + subq, + (PluginMetric.resource_name == subq.c.resource_name) + & (PluginMetric.metric_name == subq.c.metric_name) + & (PluginMetric.recorded_at == subq.c.latest_at), + ).where(PluginMetric.source_module == "snmp") + ) + rows = result.scalars().all() + + out: dict[str, dict[str, float]] = {} + for row in rows: + out.setdefault(row.resource_name, {})[row.metric_name] = row.value + return out + + +async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]: + """Return {label: [(recorded_at, value), ...]} for a single device.""" + since = datetime.now(timezone.utc) - timedelta(hours=hours) + result = await db.execute( + select(PluginMetric) + .where( + PluginMetric.source_module == "snmp", + PluginMetric.resource_name == device_name, + PluginMetric.recorded_at >= since, + ) + .order_by(PluginMetric.recorded_at) + ) + rows = result.scalars().all() + + out: dict[str, list] = {} + for row in rows: + out.setdefault(row.metric_name, []).append( + (row.recorded_at.isoformat(), row.value) + ) + return out + + +@snmp_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) + device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg] + + async with current_app.db_sessionmaker() as db: + latest = await _latest_readings(db, device_names) + + # Merge config + latest readings for the template + devices = [] + for cfg in devices_cfg: + name = cfg.get("name") or cfg.get("host", "?") + devices.append({ + "name": name, + "host": cfg.get("host", ""), + "oids": cfg.get("oids", []), + "readings": latest.get(name, {}), + }) + + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + return await render_template("snmp/index.html", + devices=devices, + poll_interval=poll_interval) + + +def _device_binds_to_host(d: dict, keys: set[str], host_id: str) -> bool: + """Does config device `d` belong on the host identified by `keys` + (lowercased {address, name}) / `host_id`? + + A device may declare an **explicit** binding that decouples "which Steward + host this belongs to" from "where to poll" (the `host` field): + • `host_id` — the Steward Host UUID (exact match), the explicit link. + • `steward_host` — a friendly bind by Host name or address (case-insensitive). + An explicit binding is **exclusive**: a device bound to host A must not also + match host B by a coincidental poll-target string. Only when no explicit + binding is present do we fall back to the implicit `host`-string match + (backward compatible with configs that only set `host`). + """ + explicit_id = str(d.get("host_id", "")).strip() + explicit_name = str(d.get("steward_host", "")).strip().lower() + if explicit_id or explicit_name: + if explicit_id and host_id and explicit_id == host_id: + return True + return bool(explicit_name and explicit_name in keys) + return str(d.get("host", "")).strip().lower() in keys + + +def _devices_for_host(devices_cfg: list, address: str | None, name: str | None, + host_id: str | None = None) -> list[dict]: + """SNMP devices that map onto a Steward host's page. Prefers an explicit + per-device binding (`host_id` / `steward_host`); otherwise falls back to + matching the device's `host` (poll target) against the host's address or + name (case-insensitive). See `_device_binds_to_host` for the precedence.""" + keys = {(address or "").strip().lower(), (name or "").strip().lower()} + keys.discard("") + hid = (host_id or "").strip() + return [ + d for d in devices_cfg + if isinstance(d, dict) and _device_binds_to_host(d, keys, hid) + ] + + +@snmp_bp.get("/host/") +@require_role(UserRole.viewer) +async def host_panel(host_id: str): + """Per-host SNMP fragment for the Hosts hub. Surfaces any configured SNMP + device that maps to this host (by address or name) with its latest readings. + Renders nothing when no device maps, so hosts without SNMP carry no empty card. + """ + from steward.models.hosts import Host + + devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) + async with current_app.db_sessionmaker() as db: + host = (await db.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return "" + matched = _devices_for_host(devices_cfg, host.address, host.name, host.id) + if not matched: + return "" + names = [d.get("name") or d.get("host", "?") for d in matched] + latest = await _latest_readings(db, names) + + devices = [{ + "name": d.get("name") or d.get("host", "?"), + "host": d.get("host", ""), + "oids": d.get("oids", []), + "readings": latest.get(d.get("name") or d.get("host", "?"), {}), + "bound": bool(str(d.get("host_id", "")).strip() + or str(d.get("steward_host", "")).strip()), + } for d in matched] + return await render_template("snmp/host_panel.html", devices=devices) + + +@snmp_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) + device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg] + + async with current_app.db_sessionmaker() as db: + latest = await _latest_readings(db, device_names[:10]) + + devices = [] + for cfg in devices_cfg[:10]: + name = cfg.get("name") or cfg.get("host", "?") + devices.append({ + "name": name, + "oids": cfg.get("oids", []), + "readings": latest.get(name, {}), + }) + + return await render_template("snmp/widget.html", + devices=devices, + total=len(devices_cfg)) + + +@snmp_bp.get("/device/") +@require_role(UserRole.viewer) +async def device_detail(device_name: str): + devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) + device_cfg = next( + (d for d in devices_cfg if (d.get("name") or d.get("host")) == device_name), + None, + ) + if device_cfg is None: + from quart import abort + abort(404) + + hours = int(request.args.get("hours", 24)) + async with current_app.db_sessionmaker() as db: + hist = await _history(db, device_name, hours=hours) + + import json + history_json = json.dumps(hist) + oid_labels = [o.get("label") or o.get("oid") for o in device_cfg.get("oids", [])] + + return await render_template( + "snmp/device.html", + device=device_cfg, + device_name=device_name, + oid_labels=oid_labels, + history_json=history_json, + hours=hours, + ) diff --git a/plugins/snmp/scheduler.py b/plugins/snmp/scheduler.py new file mode 100644 index 0000000..e0d4df7 --- /dev/null +++ b/plugins/snmp/scheduler.py @@ -0,0 +1,69 @@ +# plugins/snmp/scheduler.py +from __future__ import annotations +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +from steward.core.scheduler import ScheduledTask + +logger = logging.getLogger(__name__) + + +def make_poll_task(app: "Quart") -> ScheduledTask: + interval = app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60) + + async def poll() -> None: + await _do_poll(app) + + return ScheduledTask( + name="snmp_poll", + coro_factory=poll, + interval_seconds=int(interval), + run_on_startup=True, + ) + + +async def _do_poll(app: "Quart") -> None: + from .poller import poll_device + from steward.core.alerts import record_metric + + devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", []) + if not devices: + return + + async with app.db_sessionmaker() as session: + async with session.begin(): + for device in devices: + if not isinstance(device, dict): + continue + name = device.get("name") or device.get("host", "unknown") + host = device.get("host", "") + port = int(device.get("port", 161)) + community = device.get("community", "public") + version = str(device.get("version", "2c")) + oids = device.get("oids", []) + + if not host or not oids: + continue + + try: + readings = await poll_device(host, port, community, version, oids) + except Exception: + logger.exception("SNMP poll failed for device %s (%s)", name, host) + continue + + for label, value in readings.items(): + await record_metric( + session=session, + source_module="snmp", + resource_name=name, + metric_name=label, + value=value, + ) + + if readings: + logger.debug("SNMP polled %s (%s): %d OID(s)", name, host, len(readings)) + else: + logger.debug("SNMP polled %s (%s): no readings", name, host) diff --git a/plugins/snmp/templates/snmp/device.html b/plugins/snmp/templates/snmp/device.html new file mode 100644 index 0000000..b0b1a18 --- /dev/null +++ b/plugins/snmp/templates/snmp/device.html @@ -0,0 +1,85 @@ +{# plugins/snmp/templates/snmp/device.html #} +{% extends "base.html" %} +{% from "_macros.html" import crumbs %} +{% block title %}SNMP — {{ device_name }} — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("SNMP", "/plugins/snmp/"), (device_name, "")]) }}{% endblock %} +{% block content %} +
+

{{ device_name }}

+ {{ device.host }} +
+ {% for h in [1, 6, 24, 168] %} + + {% if h < 24 %}{{ h }}h{% elif h == 24 %}24h{% else %}7d{% endif %} + + {% endfor %} +
+
+ +{% if oid_labels %} +
+ +
+ + +{% else %} +
+ No OIDs configured for this device. +
+{% endif %} +{% endblock %} diff --git a/plugins/snmp/templates/snmp/host_panel.html b/plugins/snmp/templates/snmp/host_panel.html new file mode 100644 index 0000000..42d471e --- /dev/null +++ b/plugins/snmp/templates/snmp/host_panel.html @@ -0,0 +1,51 @@ +{# Per-host SNMP fragment — embedded on the Hosts hub via HTMX. Self-contained. + Shows the SNMP device(s) whose address maps to this host + latest readings. #} +
+
+

SNMP

+ All devices → +
+ + {% for device in devices %} +
+
+ {{ device.name }} + {{ device.host }} + {% if device.bound %} + bound + {% endif %} + {% if device.readings %} + ● reachable + {% else %} + ○ no data yet + {% endif %} + History +
+ + {% if device.readings %} +
+ {% for oid_cfg in device.oids %} + {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %} + {% set val = device.readings.get(label) %} +
+
{{ label }}
+ {% if val is not none %} +
+ {% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}M + {% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}K + {% else %}{{ "%.4g"|format(val) }}{% endif %} +
+ {% else %} +
—
+ {% endif %} +
+ {% endfor %} +
+ {% else %} +

No readings yet — waiting for the next poll.

+ {% endif %} +
+ {% endfor %} +
diff --git a/plugins/snmp/templates/snmp/index.html b/plugins/snmp/templates/snmp/index.html new file mode 100644 index 0000000..ef5979c --- /dev/null +++ b/plugins/snmp/templates/snmp/index.html @@ -0,0 +1,57 @@ +{# plugins/snmp/templates/snmp/index.html #} +{% extends "base.html" %} +{% block title %}SNMP — Steward{% endblock %} +{% block content %} +
+

SNMP Devices

+ {{ devices|length }} device(s) configured +
+ +{% if not devices %} +
+ No SNMP devices configured. Add devices to plugin.yaml. +
+{% else %} + +{% for device in devices %} +
+
+

{{ device.name }}

+ {{ device.host }} + {% if device.readings %} + ● reachable + {% else %} + ○ no data yet + {% endif %} + History +
+ + {% if device.readings %} +
+ {% for oid_cfg in device.oids %} + {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %} + {% set val = device.readings.get(label) %} +
+
{{ label }}
+ {% if val is not none %} +
+ {% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}M + {% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}K + {% else %}{{ "%.4g"|format(val) }}{% endif %} +
+ {% else %} +
—
+ {% endif %} +
+ {% endfor %} +
+ {% else %} +

+ No readings yet — waiting for next poll cycle. +

+ {% endif %} +
+{% endfor %} + +{% endif %} +{% endblock %} diff --git a/plugins/snmp/templates/snmp/widget.html b/plugins/snmp/templates/snmp/widget.html new file mode 100644 index 0000000..fc4d593 --- /dev/null +++ b/plugins/snmp/templates/snmp/widget.html @@ -0,0 +1,49 @@ +{# plugins/snmp/templates/snmp/widget.html #} +{% if devices %} +{% for device in devices %} +
+
+ + {{ device.name }} + + {% if device.readings %} + ● + {% else %} + ○ + {% endif %} + detail → +
+ {% if device.readings %} +
+ {% for oid_cfg in device.oids[:4] %} + {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %} + {% set val = device.readings.get(label) %} + {% if val is not none %} + + {{ label }} + + {% if val >= 1000000 %}{{ "%.1f"|format(val / 1000000) }}M + {% elif val >= 1000 %}{{ "%.0f"|format(val / 1000) }}K + {% else %}{{ "%.4g"|format(val) }}{% endif %} + + + {% endif %} + {% endfor %} + {% if device.oids|length > 4 %} + +{{ device.oids|length - 4 }} more + {% endif %} +
+ {% endif %} +
+{% if not loop.last %} +
+{% endif %} +{% endfor %} +{% if total > devices|length %} +
+ +{{ total - devices|length }} more — view all → +
+{% endif %} +{% else %} +

No SNMP devices configured.

+{% endif %} diff --git a/plugins/traefik/__init__.py b/plugins/traefik/__init__.py new file mode 100644 index 0000000..19f2982 --- /dev/null +++ b/plugins/traefik/__init__.py @@ -0,0 +1,35 @@ +# plugins/traefik/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + from .models import TraefikMetric, TraefikCert, TraefikGlobalStat, TraefikAccessSummary # noqa: F401 + + +def get_scheduled_tasks() -> list: + from .scheduler import make_scrape_task, make_access_log_task + tasks = [make_scrape_task(_app)] + access_log_cfg = _app.config["PLUGINS"]["traefik"].get("access_log", {}) + if not isinstance(access_log_cfg, dict): + access_log_cfg = {} + if access_log_cfg.get("enabled", False): + tasks.append(make_access_log_task(_app)) + return tasks + + +def get_blueprint(): + from .routes import traefik_bp + return traefik_bp + + +def get_nav() -> list[dict]: + # Surfaces in the sidebar's Infrastructure group (Traefik services view). + return [{"label": "Traefik", "href": "/plugins/traefik/"}] diff --git a/plugins/traefik/access_log.py b/plugins/traefik/access_log.py new file mode 100644 index 0000000..909b493 --- /dev/null +++ b/plugins/traefik/access_log.py @@ -0,0 +1,199 @@ +# plugins/traefik/access_log.py +"""Traefik JSON access log tailer and traffic-origin aggregator. + +Reads new lines from Traefik's JSON access log on each scheduler tick, +classifies source IPs as internal (RFC1918) or external, optionally +looks up country codes via MaxMind GeoLite2, and returns an aggregated +summary ready to persist in TraefikAccessSummary. +""" +from __future__ import annotations + +import ipaddress +import json +from collections import defaultdict +from pathlib import Path + +# ── Module-level file-position state ───────────────────────────────────────── +_log_position: int = 0 # byte offset of last read +_log_inode: int = 0 # detect rotation by inode change + +# ── Private IP ranges (RFC1918 + loopback + link-local) ────────────────────── +_PRIVATE_NETS = [ + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("127.0.0.0/8"), + ipaddress.ip_network("169.254.0.0/16"), + ipaddress.ip_network("::1/128"), + ipaddress.ip_network("fc00::/7"), + ipaddress.ip_network("fe80::/10"), +] + + +def _strip_port(raw: str) -> str: + """Strip trailing :port from an IPv4 address string. Leaves IPv6 untouched.""" + if not raw: + return raw + # Bracketed IPv6 with port: [::1]:12345 + if raw.startswith("["): + return raw[1:raw.index("]")] if "]" in raw else raw + # IPv4:port has exactly one colon + if raw.count(":") == 1: + return raw.rsplit(":", 1)[0] + return raw + + +def is_internal(raw_ip: str) -> bool: + """Return True if the address falls within a private/loopback range.""" + ip_str = _strip_port(raw_ip) + try: + addr = ipaddress.ip_address(ip_str) + return any(addr in net for net in _PRIVATE_NETS) + except ValueError: + return False + + +def _country(ip_str: str, geoip_db) -> str | None: + """Return ISO country code via an open maxminddb reader, or None.""" + if geoip_db is None: + return None + clean = _strip_port(ip_str) + try: + record = geoip_db.get(clean) + if record: + return record.get("country", {}).get("iso_code") + except Exception: + pass + return None + + +def read_new_lines(log_path: str) -> list[dict]: + """Return all new JSON-parsed access log entries since the last call. + + Handles log rotation: if the file shrinks or its inode changes, we + reset to the beginning of the new file. + """ + global _log_position, _log_inode + + path = Path(log_path) + if not path.exists(): + return [] + + try: + stat = path.stat() + except OSError: + return [] + + current_inode = stat.st_ino + current_size = stat.st_size + + # Rotation detected + if current_inode != _log_inode or current_size < _log_position: + _log_position = 0 + _log_inode = current_inode + + if current_size == _log_position: + return [] + + entries: list[dict] = [] + try: + with open(log_path, "rb") as f: + f.seek(_log_position) + raw = f.read(current_size - _log_position) + _log_position = current_size + except OSError: + return [] + + for raw_line in raw.split(b"\n"): + raw_line = raw_line.strip() + if not raw_line: + continue + try: + entries.append(json.loads(raw_line)) + except json.JSONDecodeError: + continue + + return entries + + +def aggregate(entries: list[dict], geoip_db=None, top_n: int = 10) -> dict | None: + """Aggregate a batch of access log entries into a summary dict. + + Returns None if there are no entries. + Result keys map directly to TraefikAccessSummary columns (excluding + id and period_start/period_end, which the caller provides). + """ + if not entries: + return None + + total = 0 + internal = 0 + external = 0 + total_bytes = 0.0 + + by_router: dict[str, int] = defaultdict(int) + ip_counts: dict[str, int] = defaultdict(int) + ip_internal: dict[str, bool] = {} + ip_country: dict[str, str | None] = {} + country_counts: dict[str, int] = defaultdict(int) + + for entry in entries: + total += 1 + raw_ip = entry.get("ClientHost", "") + router = entry.get("RouterName") or entry.get("ServiceName") or "unknown" + total_bytes += entry.get("DownstreamContentSize") or 0 + + by_router[router] += 1 + + internal_flag = is_internal(raw_ip) + if internal_flag: + internal += 1 + else: + external += 1 + if raw_ip not in ip_country: + ip_country[raw_ip] = _country(raw_ip, geoip_db) + c = ip_country[raw_ip] + if c: + country_counts[c] += 1 + + ip_counts[raw_ip] += 1 + ip_internal[raw_ip] = internal_flag + + top_ips = [ + { + "ip": ip, + "count": cnt, + "internal": ip_internal[ip], + "country": ip_country.get(ip), + } + for ip, cnt in sorted(ip_counts.items(), key=lambda x: x[1], reverse=True)[:top_n] + ] + top_countries = [ + {"country": c, "count": n} + for c, n in sorted(country_counts.items(), key=lambda x: x[1], reverse=True)[:top_n] + ] + top_routers = [ + {"router": r, "count": n} + for r, n in sorted(by_router.items(), key=lambda x: x[1], reverse=True)[:top_n] + ] + + return { + "total_requests": total, + "internal_requests": internal, + "external_requests": external, + "total_bytes": total_bytes, + "top_ips_json": json.dumps(top_ips), + "top_countries_json": json.dumps(top_countries), + "top_routers_json": json.dumps(top_routers), + } + + +def open_geoip(db_path: str | None): + """Open a MaxMind mmdb file if available. Returns None if not configured or missing.""" + if not db_path: + return None + try: + import maxminddb # type: ignore[import] + return maxminddb.open_database(db_path) + except Exception: + return None diff --git a/plugins/traefik/migrations/__init__.py b/plugins/traefik/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/traefik/migrations/env.py b/plugins/traefik/migrations/env.py new file mode 100644 index 0000000..a523e27 --- /dev/null +++ b/plugins/traefik/migrations/env.py @@ -0,0 +1,75 @@ +# plugins/traefik/migrations/env.py +"""Alembic env.py for the Traefik plugin. + +For standalone use during plugin development. +At app startup, migration_runner.py uses the core env.py with version_locations. +""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +# Make steward importable +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from steward.models.base import Base +import steward.models # noqa: F401 — core models +from plugins.traefik.models import TraefikMetric # noqa: F401 — plugin model + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("STEWARD_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/traefik/migrations/versions/__init__.py b/plugins/traefik/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/traefik/migrations/versions/traefik_001_initial.py b/plugins/traefik/migrations/versions/traefik_001_initial.py new file mode 100644 index 0000000..aa53f78 --- /dev/null +++ b/plugins/traefik/migrations/versions/traefik_001_initial.py @@ -0,0 +1,37 @@ +# plugins/traefik/migrations/versions/traefik_001_initial.py +"""Traefik metrics table + +Revision ID: traefik_001_initial +Revises: (none — this is a branch off the core head via depends_on) +Create Date: 2026-03-17 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "traefik_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "traefik" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "traefik_metrics", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("router_name", sa.String(255), nullable=False), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("request_rate", sa.Float, nullable=False), + sa.Column("error_rate_4xx_pct", sa.Float, nullable=False), + sa.Column("error_rate_5xx_pct", sa.Float, nullable=False), + sa.Column("latency_p50_ms", sa.Float, nullable=False), + sa.Column("latency_p95_ms", sa.Float, nullable=False), + sa.Column("latency_p99_ms", sa.Float, nullable=False), + ) + op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics", + ["router_name", "scraped_at"]) + + +def downgrade() -> None: + op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics") + op.drop_table("traefik_metrics") diff --git a/plugins/traefik/migrations/versions/traefik_002_stats.py b/plugins/traefik/migrations/versions/traefik_002_stats.py new file mode 100644 index 0000000..8320d53 --- /dev/null +++ b/plugins/traefik/migrations/versions/traefik_002_stats.py @@ -0,0 +1,48 @@ +"""Traefik extended stats: bandwidth, certs, global stats + +Revision ID: traefik_002_stats +Revises: traefik_001_initial +Create Date: 2026-03-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "traefik_002_stats" +down_revision: Union[str, None] = "traefik_001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "traefik_metrics", + sa.Column("response_bytes_rate", sa.Float, nullable=False, server_default="0"), + ) + + op.create_table( + "traefik_certs", + sa.Column("serial", sa.String(255), primary_key=True), + sa.Column("cn", sa.String(512), nullable=True), + sa.Column("sans", sa.Text, nullable=True), + sa.Column("not_after", sa.DateTime(timezone=True), nullable=False), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "traefik_global_stats", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("open_conns_total", sa.Float, nullable=True), + sa.Column("config_reloads_total", sa.Float, nullable=True), + sa.Column("config_last_reload_success", sa.Float, nullable=True), + sa.Column("process_memory_bytes", sa.Float, nullable=True), + ) + op.create_index("ix_traefik_global_stats_scraped", "traefik_global_stats", ["scraped_at"]) + + +def downgrade() -> None: + op.drop_index("ix_traefik_global_stats_scraped", "traefik_global_stats") + op.drop_table("traefik_global_stats") + op.drop_table("traefik_certs") + op.drop_column("traefik_metrics", "response_bytes_rate") diff --git a/plugins/traefik/migrations/versions/traefik_003_access_log.py b/plugins/traefik/migrations/versions/traefik_003_access_log.py new file mode 100644 index 0000000..fbc8009 --- /dev/null +++ b/plugins/traefik/migrations/versions/traefik_003_access_log.py @@ -0,0 +1,40 @@ +"""Traefik access log traffic summaries + +Revision ID: traefik_003_access_log +Revises: traefik_002_stats +Create Date: 2026-03-20 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "traefik_003_access_log" +down_revision: Union[str, None] = "traefik_002_stats" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "traefik_access_summaries", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("period_start", sa.DateTime(timezone=True), nullable=False), + sa.Column("period_end", sa.DateTime(timezone=True), nullable=False), + sa.Column("total_requests", sa.Integer, nullable=False, server_default="0"), + sa.Column("internal_requests", sa.Integer, nullable=False, server_default="0"), + sa.Column("external_requests", sa.Integer, nullable=False, server_default="0"), + sa.Column("total_bytes", sa.Float, nullable=False, server_default="0"), + sa.Column("top_ips_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("top_countries_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("top_routers_json", sa.Text, nullable=False, server_default="[]"), + ) + op.create_index( + "ix_traefik_access_summaries_period_end", + "traefik_access_summaries", + ["period_end"], + ) + + +def downgrade() -> None: + op.drop_index("ix_traefik_access_summaries_period_end", "traefik_access_summaries") + op.drop_table("traefik_access_summaries") diff --git a/plugins/traefik/models.py b/plugins/traefik/models.py new file mode 100644 index 0000000..d177d10 --- /dev/null +++ b/plugins/traefik/models.py @@ -0,0 +1,67 @@ +# plugins/traefik/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import DateTime, Float, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from steward.models.base import Base + + +class TraefikMetric(Base): + __tablename__ = "traefik_metrics" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + router_name: Mapped[str] = mapped_column(String(255), nullable=False) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) + ) + request_rate: Mapped[float] = mapped_column(Float, nullable=False) + error_rate_4xx_pct: Mapped[float] = mapped_column(Float, nullable=False) + error_rate_5xx_pct: Mapped[float] = mapped_column(Float, nullable=False) + latency_p50_ms: Mapped[float] = mapped_column(Float, nullable=False) + latency_p95_ms: Mapped[float] = mapped_column(Float, nullable=False) + latency_p99_ms: Mapped[float] = mapped_column(Float, nullable=False) + response_bytes_rate: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + + +class TraefikCert(Base): + """TLS certificate expiry tracking — upserted by serial each scrape.""" + __tablename__ = "traefik_certs" + + serial: Mapped[str] = mapped_column(String(255), primary_key=True) + cn: Mapped[str | None] = mapped_column(String(512), nullable=True) + sans: Mapped[str | None] = mapped_column(Text, nullable=True) + not_after: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) + ) + + +class TraefikAccessSummary(Base): + """Aggregated traffic-origin summary — one row per scheduler tick.""" + __tablename__ = "traefik_access_summaries" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + period_start: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + period_end: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + total_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + internal_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + external_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + total_bytes: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + top_ips_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + top_countries_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + top_routers_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + + +class TraefikGlobalStat(Base): + """One row per scrape — process-level and config stats.""" + __tablename__ = "traefik_global_stats" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) + ) + open_conns_total: Mapped[float | None] = mapped_column(Float, nullable=True) + config_reloads_total: Mapped[float | None] = mapped_column(Float, nullable=True) + config_last_reload_success: Mapped[float | None] = mapped_column(Float, nullable=True) + process_memory_bytes: Mapped[float | None] = mapped_column(Float, nullable=True) diff --git a/plugins/traefik/plugin.yaml b/plugins/traefik/plugin.yaml new file mode 100644 index 0000000..3f09ff4 --- /dev/null +++ b/plugins/traefik/plugin.yaml @@ -0,0 +1,21 @@ +# plugins/traefik/plugin.yaml +name: traefik +version: "1.0.0" +description: "Traefik reverse proxy metrics and access log integration" +author: "Steward" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik" +tags: + - proxy + - metrics + - access-log + +config: + metrics_url: "http://localhost:8080/metrics" + scrape_interval_seconds: 60 + access_log: + enabled: false + path: "/var/log/traefik/access.log" + geoip_db: "" # optional: path to MaxMind GeoLite2-Country.mmdb diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py new file mode 100644 index 0000000..80e6c93 --- /dev/null +++ b/plugins/traefik/routes.py @@ -0,0 +1,373 @@ +# plugins/traefik/routes.py +from __future__ import annotations +from collections import defaultdict +from datetime import datetime, timezone +from quart import Blueprint, current_app, render_template, request +from sqlalchemy import Integer, cast, func, select +import json + +from steward.auth.middleware import require_role +from steward.models.users import UserRole +from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample +from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric + +traefik_bp = Blueprint("traefik", __name__, template_folder="templates") + + +def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: + """Generate an inline SVG sparkline from a list of float values.""" + if len(values) < 2: + return f'' + mn, mx = min(values), max(values) + if mx == mn: + mx = mn + 1.0 + step = width / (len(values) - 1) + pts = [] + for i, v in enumerate(values): + x = i * step + y = height - (v - mn) / (mx - mn) * (height - 2) - 1 + pts.append(f"{x:.1f},{y:.1f}") + poly = " ".join(pts) + return ( + f'' + f'' + f'' + ) + + +@traefik_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + _al_cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {}) + if not isinstance(_al_cfg, dict): + _al_cfg = {} + access_log_enabled = _al_cfg.get("enabled", False) + current_range = request.args.get("range", DEFAULT_RANGE) + return await render_template( + "traefik/index.html", + poll_interval=poll_interval, + access_log_enabled=access_log_enabled, + current_range=current_range, + ) + + +@traefik_bp.get("/rows") +@require_role(UserRole.viewer) +async def rows(): + """HTMX fragment: service cards with sparklines scoped to selected time range.""" + since, range_key = parse_range(request.args.get("range")) + now = datetime.now(timezone.utc) + + async with current_app.db_sessionmaker() as db: + # All known routers + result = await db.execute( + select(TraefikMetric.router_name) + .distinct() + .order_by(TraefikMetric.router_name) + ) + routers = [row[0] for row in result.all()] + + b_secs = bucket_seconds(since) + bucket_col = ( + cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs + ).label("bucket") + + router_data = [] + for router in routers: + # Bucketed history for sparklines — always ~80 points regardless of range + result = await db.execute( + select( + func.avg(TraefikMetric.request_rate).label("request_rate"), + func.avg(TraefikMetric.latency_p95_ms).label("latency_p95_ms"), + func.avg(TraefikMetric.error_rate_5xx_pct).label("error_rate_5xx_pct"), + func.min(TraefikMetric.scraped_at).label("scraped_at"), + bucket_col, + ) + .where(TraefikMetric.router_name == router) + .where(TraefikMetric.scraped_at >= since) + .group_by(bucket_col) + .order_by(bucket_col) + ) + history = result.all() + + # Latest value — always the most recent raw row + result = await db.execute( + select(TraefikMetric) + .where(TraefikMetric.router_name == router) + .order_by(TraefikMetric.scraped_at.desc()) + .limit(1) + ) + latest = result.scalar_one_or_none() + if not latest: + continue + + router_data.append({ + "name": router, + "latest": latest, + "sparkline_req": _sparkline([r.request_rate for r in history]), + "sparkline_p95": _sparkline([r.latency_p95_ms for r in history]), + "sparkline_5xx": _sparkline([r.error_rate_5xx_pct for r in history]), + }) + + # Latest global stat (always most recent, not range-filtered) + result = await db.execute( + select(TraefikGlobalStat) + .order_by(TraefikGlobalStat.scraped_at.desc()) + .limit(1) + ) + global_stat = result.scalar_one_or_none() + + # All certs, sorted by soonest expiry + result = await db.execute( + select(TraefikCert).order_by(TraefikCert.not_after) + ) + raw_certs = result.scalars().all() + + certs = [] + for c in raw_certs: + days = (c.not_after - now).total_seconds() / 86400.0 + certs.append({ + "serial": c.serial, + "cn": c.cn or c.serial, + "sans": c.sans, + "not_after": c.not_after, + "days_remaining": days, + }) + + return await render_template( + "traefik/rows.html", + router_data=router_data, + global_stat=global_stat, + certs=certs, + range_key=range_key, + ) + + +@traefik_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """HTMX dashboard widget fragment.""" + now = datetime.now(timezone.utc) + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(TraefikMetric.router_name).distinct().order_by(TraefikMetric.router_name) + ) + routers = [row[0] for row in result.all()] + router_data = [] + for router in routers: + result = await db.execute( + select(TraefikMetric) + .where(TraefikMetric.router_name == router) + .order_by(TraefikMetric.scraped_at.desc()) + .limit(1) + ) + latest = result.scalar_one_or_none() + if latest: + router_data.append({"name": router, "latest": latest}) + + router_data.sort(key=lambda r: ( + -(1 if r["latest"].error_rate_5xx_pct > 0.1 else 0), + -(1 if r["latest"].error_rate_4xx_pct > 5.0 else 0), + -r["latest"].request_rate, + )) + total_routers = len(router_data) + router_data = router_data[:5] + + result = await db.execute( + select(TraefikCert).order_by(TraefikCert.not_after) + ) + raw_certs = result.scalars().all() + + expiring_certs = [] + for c in raw_certs: + days = (c.not_after - now).total_seconds() / 86400.0 + if days < 30: + expiring_certs.append({"cn": c.cn or c.serial, "days_remaining": days}) + + return await render_template( + "traefik/widget.html", + router_data=router_data, + expiring_certs=expiring_certs, + total_routers=total_routers, + ) + + +@traefik_bp.get("/widget/access_log") +@require_role(UserRole.viewer) +async def widget_access_log(): + """HTMX dashboard widget: traffic origin summary with IP filter.""" + from collections import defaultdict + ip_filter = request.args.get("ip_filter", "all") + limit = max(1, min(50, int(request.args.get("limit", 10) or 10))) + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(TraefikAccessSummary) + .order_by(TraefikAccessSummary.period_end.desc()) + .limit(12) # last ~1 hour of 5-min windows + ) + rows = result.scalars().all() + + if not rows: + return await render_template("traefik/widget_access_log.html", + summary=None, widget_id=widget_id) + + total = sum(r.total_requests for r in rows) + internal = sum(r.internal_requests for r in rows) + external = sum(r.external_requests for r in rows) + + ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True}) + for row in rows: + for entry in json.loads(row.top_ips_json): + k = entry["ip"] + ip_counts[k]["count"] += entry["count"] + ip_counts[k]["internal"] = entry.get("internal", True) + + all_ips = sorted( + [{"ip": k, **v} for k, v in ip_counts.items()], + key=lambda x: x["count"], reverse=True, + ) + if ip_filter == "internal": + all_ips = [e for e in all_ips if e["internal"]] + elif ip_filter == "external": + all_ips = [e for e in all_ips if not e["internal"]] + + summary = { + "total": total, + "internal": internal, + "external": external, + "external_pct": (external / total * 100.0) if total else 0.0, + "top_ips": all_ips[:limit], + "ip_filter": ip_filter, + } + return await render_template("traefik/widget_access_log.html", + summary=summary, widget_id=widget_id) + + +@traefik_bp.get("/widget/request_chart") +@require_role(UserRole.viewer) +async def widget_request_chart(): + """HTMX dashboard widget: Chart.js request rate + error % over time.""" + from datetime import timedelta + hours = max(1, min(24, int(request.args.get("hours", 6) or 6))) + widget_id = request.args.get("wid", "0") + + since = datetime.now(timezone.utc) - timedelta(hours=hours) + b_secs = max(60, (hours * 3600) // 80) # ~80 buckets + + bucket_col = ( + cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs + ).label("bucket") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select( + func.avg(TraefikMetric.request_rate).label("req_rate"), + func.avg(TraefikMetric.error_rate_5xx_pct).label("err_pct"), + func.min(TraefikMetric.scraped_at).label("scraped_at"), + bucket_col, + ) + .where(TraefikMetric.scraped_at >= since) + .group_by(bucket_col) + .order_by(bucket_col) + ) + history = result.all() + + labels = list(range(len(history))) + req_rates = [round(r.req_rate or 0, 3) for r in history] + err_pcts = [round(r.err_pct or 0, 2) for r in history] + + return await render_template( + "traefik/widget_request_chart.html", + labels_json=json.dumps(labels), + req_rate_json=json.dumps(req_rates), + error_rate_json=json.dumps(err_pcts), + widget_id=widget_id, + hours=hours, + ) + + +@traefik_bp.get("/traffic") +@require_role(UserRole.viewer) +async def traffic(): + """HTMX fragment: traffic origin summary scoped to selected time range.""" + since, range_key = parse_range(request.args.get("range")) + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(TraefikAccessSummary) + .where(TraefikAccessSummary.period_end >= since) + .order_by(TraefikAccessSummary.period_end.asc()) + # No LIMIT — all summaries in range needed for accurate totals; + # sparkline subsamples to 80 points afterwards + ) + rows = result.scalars().all() + + if not rows: + return await render_template( + "traefik/traffic.html", summary=None, history=[], range_key=range_key + ) + + total = sum(r.total_requests for r in rows) + internal = sum(r.internal_requests for r in rows) + external = sum(r.external_requests for r in rows) + total_bytes = sum(r.total_bytes for r in rows) + + ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True, "country": None}) + country_counts: dict[str, int] = defaultdict(int) + router_counts: dict[str, int] = defaultdict(int) + + for row in rows: + for entry in json.loads(row.top_ips_json): + k = entry["ip"] + ip_counts[k]["count"] += entry["count"] + ip_counts[k]["internal"] = entry["internal"] + ip_counts[k]["country"] = entry.get("country") + for entry in json.loads(row.top_countries_json): + country_counts[entry["country"]] += entry["count"] + for entry in json.loads(row.top_routers_json): + router_counts[entry["router"]] += entry["count"] + + top_ips = sorted( + [{"ip": k, **v} for k, v in ip_counts.items()], + key=lambda x: x["count"], reverse=True + )[:10] + top_countries = sorted( + [{"country": k, "count": v} for k, v in country_counts.items()], + key=lambda x: x["count"], reverse=True + )[:10] + top_routers = sorted( + [{"router": k, "count": v} for k, v in router_counts.items()], + key=lambda x: x["count"], reverse=True + )[:10] + + history = [ + { + "period_end": r.period_end, + "external_pct": (r.external_requests / r.total_requests * 100.0) + if r.total_requests else 0.0, + } + for r in rows + ] + + summary = { + "total": total, + "internal": internal, + "external": external, + "total_bytes": total_bytes, + "external_pct": (external / total * 100.0) if total else 0.0, + "top_ips": top_ips, + "top_countries": top_countries, + "top_routers": top_routers, + "sparkline_external": _sparkline( + [h["external_pct"] for h in subsample(history, 80)] + ), + } + + return await render_template( + "traefik/traffic.html", summary=summary, history=history, range_key=range_key + ) diff --git a/plugins/traefik/scheduler.py b/plugins/traefik/scheduler.py new file mode 100644 index 0000000..678e788 --- /dev/null +++ b/plugins/traefik/scheduler.py @@ -0,0 +1,203 @@ +# plugins/traefik/scheduler.py +from __future__ import annotations +import logging +import time +from typing import TYPE_CHECKING + +from datetime import datetime +from steward.core.scheduler import ScheduledTask + +if TYPE_CHECKING: + from quart import Quart + +logger = logging.getLogger(__name__) + +# Module-level state: previous scrape data for delta computation +_prev_metrics: dict = {} +_prev_time: float = 0.0 + +# Access log state +_geoip_db = None +_access_log_last_tick: "datetime | None" = None + + +def make_scrape_task(app: "Quart") -> ScheduledTask: + """Return a ScheduledTask that scrapes Traefik metrics on the configured interval.""" + interval = app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"] + + async def scrape() -> None: + await _do_scrape(app) + + return ScheduledTask( + name="traefik_scrape", + coro_factory=scrape, + interval_seconds=int(interval), + run_on_startup=True, + ) + + +async def _do_scrape(app: "Quart") -> None: + """Fetch Traefik metrics, write to DB, and emit plugin_metrics.""" + global _prev_metrics, _prev_time + + from datetime import datetime, timezone + from .models import TraefikCert, TraefikGlobalStat, TraefikMetric + from .scraper import ( + compute_global_stats, + compute_router_metrics, + extract_certs, + fetch_metrics, + ) + from steward.core.alerts import record_metric + + metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"] + + try: + current = await fetch_metrics(metrics_url) + except Exception: + logger.exception("Traefik scrape failed (url=%s)", metrics_url) + return + + now = time.monotonic() + elapsed = now - _prev_time if _prev_time else 60.0 + router_metrics = compute_router_metrics(current, _prev_metrics or None, elapsed) + global_stats = compute_global_stats(current, _prev_metrics or None) + cert_list = extract_certs(current) + + _prev_metrics = current + _prev_time = now + + scraped_at = datetime.now(timezone.utc) + + async with app.db_sessionmaker() as session: + async with session.begin(): + # ── Per-service metrics ─────────────────────────────────────────── + for router_name, metrics in router_metrics.items(): + row = TraefikMetric( + router_name=router_name, + scraped_at=scraped_at, + request_rate=metrics["request_rate"], + error_rate_4xx_pct=metrics["error_rate_4xx_pct"], + error_rate_5xx_pct=metrics["error_rate_5xx_pct"], + latency_p50_ms=metrics["latency_p50_ms"], + latency_p95_ms=metrics["latency_p95_ms"], + latency_p99_ms=metrics["latency_p99_ms"], + response_bytes_rate=metrics["response_bytes_rate"], + ) + session.add(row) + + for metric_name, value in [ + ("request_rate", metrics["request_rate"]), + ("error_rate_4xx_pct", metrics["error_rate_4xx_pct"]), + ("error_rate_5xx_pct", metrics["error_rate_5xx_pct"]), + ("latency_p50_ms", metrics["latency_p50_ms"]), + ("latency_p95_ms", metrics["latency_p95_ms"]), + ("latency_p99_ms", metrics["latency_p99_ms"]), + ("response_bytes_rate", metrics["response_bytes_rate"]), + ]: + await record_metric( + session=session, + source_module="traefik", + resource_name=router_name, + metric_name=metric_name, + value=value, + ) + + # ── Global stats ────────────────────────────────────────────────── + session.add(TraefikGlobalStat(scraped_at=scraped_at, **global_stats)) + + # ── TLS certs (upsert by serial) ────────────────────────────────── + for cert in cert_list: + existing = await session.get(TraefikCert, cert["serial"]) + if existing: + existing.cn = cert["cn"] + existing.sans = cert["sans"] + existing.not_after = cert["not_after"] + existing.scraped_at = scraped_at + else: + session.add(TraefikCert( + serial=cert["serial"], + cn=cert["cn"], + sans=cert["sans"], + not_after=cert["not_after"], + scraped_at=scraped_at, + )) + + # Emit cert_expiry_days for alert rules + days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0 + await record_metric( + session=session, + source_module="traefik", + resource_name=cert["cn"] or cert["serial"], + metric_name="cert_expiry_days", + value=max(0.0, days_remaining), + ) + + logger.debug( + "Traefik scrape: %d router(s), %d cert(s)", + len(router_metrics), + len(cert_list), + ) + + +def make_access_log_task(app: "Quart") -> ScheduledTask: + """Return a ScheduledTask that processes new Traefik access log lines.""" + global _geoip_db + cfg = app.config["PLUGINS"]["traefik"].get("access_log", {}) + if not isinstance(cfg, dict): + cfg = {} + log_path: str = cfg.get("path", "/var/log/traefik/access.log") + geoip_path: str = cfg.get("geoip_db", "") + + from .access_log import open_geoip + _geoip_db = open_geoip(geoip_path or None) + if _geoip_db: + logger.info("Traefik access log: GeoIP database loaded from %s", geoip_path) + else: + logger.info("Traefik access log: GeoIP not configured, country lookup disabled") + + interval = app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"] + + async def process() -> None: + await _do_access_log(app, log_path) + + return ScheduledTask( + name="traefik_access_log", + coro_factory=process, + interval_seconds=int(interval), + run_on_startup=True, + ) + + +async def _do_access_log(app: "Quart", log_path: str) -> None: + """Read new access log lines, aggregate, and persist a summary.""" + global _access_log_last_tick + + from datetime import timezone + from .access_log import read_new_lines, aggregate + from .models import TraefikAccessSummary + + now = datetime.now(timezone.utc) + period_start = _access_log_last_tick or now + _access_log_last_tick = now + + entries = read_new_lines(log_path) + summary = aggregate(entries, geoip_db=_geoip_db) + + if summary is None: + return + + async with app.db_sessionmaker() as session: + async with session.begin(): + session.add(TraefikAccessSummary( + period_start=period_start, + period_end=now, + **summary, + )) + + logger.debug( + "Traefik access log: %d requests (%d internal, %d external)", + summary["total_requests"], + summary["internal_requests"], + summary["external_requests"], + ) diff --git a/plugins/traefik/scraper.py b/plugins/traefik/scraper.py new file mode 100644 index 0000000..6e06dd2 --- /dev/null +++ b/plugins/traefik/scraper.py @@ -0,0 +1,276 @@ +# plugins/traefik/scraper.py +"""Prometheus text-format scraper and per-service metrics calculator for Traefik. + +Traefik exposes Prometheus metrics at /metrics. This module: +1. Fetches the endpoint via httpx +2. Parses the Prometheus text format (counters + histograms) +3. Computes per-service request rate (delta/elapsed), error rates (% 4xx, % 5xx), + latency percentiles (p50, p95, p99) via linear histogram interpolation, + and bandwidth (bytes/sec from response bytes counter delta) +4. Extracts TLS certificate expiry info +5. Computes process-level and config health stats +""" +from __future__ import annotations +import math +import re +from collections import defaultdict +from datetime import datetime, timezone + +import httpx + + +# ────────────────────────────────────────────────────────────────────────────── +# Prometheus text format parser +# ────────────────────────────────────────────────────────────────────────────── + +# Parsed sample: {metric_name: {frozenset(label_items): float}} +ParsedMetrics = dict[str, dict[frozenset, float]] + +_SAMPLE_RE = re.compile( + r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+' + r'([+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?|[+-]?Inf|NaN)' +) +_LABEL_RE = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"\\]*(?:\\.[^"\\]*)*)"') + + +def parse_prometheus(text: str) -> ParsedMetrics: + """Parse Prometheus text exposition format into nested dicts. + + Returns: {metric_name: {frozenset({(label, value), ...}): float}} + """ + metrics: ParsedMetrics = {} + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + m = _SAMPLE_RE.match(line) + if not m: + continue + name, labels_str, value_str = m.groups() + try: + value = float(value_str) + except ValueError: + continue + labels = frozenset( + (lm.group(1), lm.group(2)) + for lm in _LABEL_RE.finditer(labels_str or "") + ) + metrics.setdefault(name, {})[labels] = value + return metrics + + +# ────────────────────────────────────────────────────────────────────────────── +# Per-router metric computation +# ────────────────────────────────────────────────────────────────────────────── + +def compute_router_metrics( + current: ParsedMetrics, + previous: ParsedMetrics | None, + elapsed_seconds: float, +) -> dict[str, dict[str, float]]: + """Compute per-service derived metrics from two consecutive scrapes. + + Uses traefik_service_* metrics (available by default). + + Returns: + {service_name: { + "request_rate": float, # requests/sec + "error_rate_4xx_pct": float, # % 4xx of total + "error_rate_5xx_pct": float, # % 5xx of total + "latency_p50_ms": float, + "latency_p95_ms": float, + "latency_p99_ms": float, + "response_bytes_rate": float, # bytes/sec + }} + """ + prev = previous or {} + elapsed = max(elapsed_seconds, 1.0) + + # ── Request counters ────────────────────────────────────────────────────── + req_samples = current.get("traefik_service_requests_total", {}) + prev_req = prev.get("traefik_service_requests_total", {}) + + service_total: dict[str, float] = defaultdict(float) + service_4xx: dict[str, float] = defaultdict(float) + service_5xx: dict[str, float] = defaultdict(float) + + for labels, value in req_samples.items(): + label_dict = dict(labels) + service = label_dict.get("service", "") + if not service: + continue + code = label_dict.get("code", "") + prev_value = prev_req.get(labels, value) # no delta on first scrape + delta = max(0.0, value - prev_value) + service_total[service] += delta + if code.startswith("4"): + service_4xx[service] += delta + elif code.startswith("5"): + service_5xx[service] += delta + + # ── Response bytes counters ─────────────────────────────────────────────── + bytes_samples = current.get("traefik_service_responses_bytes_total", {}) + prev_bytes = prev.get("traefik_service_responses_bytes_total", {}) + + service_bytes: dict[str, float] = defaultdict(float) + for labels, value in bytes_samples.items(): + service = dict(labels).get("service", "") + if not service: + continue + prev_value = prev_bytes.get(labels, value) + service_bytes[service] += max(0.0, value - prev_value) + + # ── Histogram buckets ───────────────────────────────────────────────────── + hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {}) + + # Collect all known services + all_services: set[str] = set(service_total.keys()) | set(service_bytes.keys()) + for labels in hist_buckets: + service = dict(labels).get("service", "") + if service: + all_services.add(service) + + result: dict[str, dict[str, float]] = {} + + for service in all_services: + total = service_total.get(service, 0.0) + result[service] = { + "request_rate": total / elapsed, + "error_rate_4xx_pct": (service_4xx.get(service, 0.0) / total * 100.0) + if total > 0 else 0.0, + "error_rate_5xx_pct": (service_5xx.get(service, 0.0) / total * 100.0) + if total > 0 else 0.0, + "latency_p50_ms": _percentile_ms(hist_buckets, service, 0.50), + "latency_p95_ms": _percentile_ms(hist_buckets, service, 0.95), + "latency_p99_ms": _percentile_ms(hist_buckets, service, 0.99), + "response_bytes_rate": service_bytes.get(service, 0.0) / elapsed, + } + + return result + + +def compute_global_stats( + current: ParsedMetrics, + previous: ParsedMetrics | None, +) -> dict[str, float | None]: + """Extract process-level and config health metrics from a scrape. + + Returns: + { + "open_conns_total": float | None, + "config_reloads_total": float | None, + "config_last_reload_success": float | None, # 1.0 = success, 0.0 = failure + "process_memory_bytes": float | None, + } + """ + stats: dict[str, float | None] = { + "open_conns_total": None, + "config_reloads_total": None, + "config_last_reload_success": None, + "process_memory_bytes": None, + } + + # Open connections — sum across all entrypoints/protocols + open_conns = current.get("traefik_open_connections", {}) + if open_conns: + stats["open_conns_total"] = sum(open_conns.values()) + + # Config reloads — sum across result labels (success + failure) + reloads = current.get("traefik_config_reloads_total", {}) + if reloads: + stats["config_reloads_total"] = sum(reloads.values()) + + # Last reload success gauge + last_success = current.get("traefik_config_last_reload_success", {}) + if last_success: + # Usually a single sample with no labels + stats["config_last_reload_success"] = next(iter(last_success.values())) + + # Process RSS memory + mem = current.get("process_resident_memory_bytes", {}) + if mem: + stats["process_memory_bytes"] = next(iter(mem.values())) + + return stats + + +def extract_certs(current: ParsedMetrics) -> list[dict]: + """Extract TLS certificate expiry info from traefik_tls_certs_not_after. + + Returns a list of dicts: + [{"serial": str, "cn": str, "sans": str, "not_after": datetime}, ...] + """ + certs = [] + samples = current.get("traefik_tls_certs_not_after", {}) + for labels, ts_value in samples.items(): + label_dict = dict(labels) + serial = label_dict.get("serial", "") + if not serial: + continue + try: + not_after = datetime.fromtimestamp(ts_value, tz=timezone.utc) + except (ValueError, OSError, OverflowError): + continue + certs.append({ + "serial": serial, + "cn": label_dict.get("cn") or None, + "sans": label_dict.get("sans") or None, + "not_after": not_after, + }) + return certs + + +def _percentile_ms( + buckets: dict[frozenset, float], + service: str, + pct: float, +) -> float: + """Linear interpolation of a Prometheus histogram percentile, returned in ms.""" + router_buckets: list[tuple[float, float]] = [] + for labels, count in buckets.items(): + label_dict = dict(labels) + if label_dict.get("service") != service: + continue + le_str = label_dict.get("le", "") + try: + le = float("inf") if le_str == "+Inf" else float(le_str) + router_buckets.append((le, count)) + except ValueError: + continue + + if not router_buckets: + return 0.0 + + router_buckets.sort(key=lambda t: t[0]) + total = router_buckets[-1][1] # +Inf bucket count == total requests + if total == 0.0: + return 0.0 + + target = total * pct + for i, (le, count) in enumerate(router_buckets): + if count < target: + continue + if i == 0: + ratio = target / count if count > 0 else 0.0 + return le * ratio * 1000.0 + prev_le, prev_count = router_buckets[i - 1] + if count == prev_count: + return prev_le * 1000.0 + if math.isinf(le): + return prev_le * 1000.0 + fraction = (target - prev_count) / (count - prev_count) + return (prev_le + fraction * (le - prev_le)) * 1000.0 + + return 0.0 + + +# ────────────────────────────────────────────────────────────────────────────── +# HTTP fetch +# ────────────────────────────────────────────────────────────────────────────── + +async def fetch_metrics(metrics_url: str) -> ParsedMetrics: + """Fetch and parse Prometheus metrics from the given URL.""" + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.get(metrics_url) + response.raise_for_status() + return parse_prometheus(response.text) diff --git a/plugins/traefik/templates/traefik/index.html b/plugins/traefik/templates/traefik/index.html new file mode 100644 index 0000000..18f50f0 --- /dev/null +++ b/plugins/traefik/templates/traefik/index.html @@ -0,0 +1,31 @@ +{# plugins/traefik/templates/traefik/index.html #} +{% extends "base.html" %} +{% block title %}Traefik — Steward{% endblock %} +{% block content %} +
+
+

Traefik Services

+ polling every {{ poll_interval }}s +
+ {% include "_time_range.html" %} +
+ +
+
+ +{% if access_log_enabled %} +
+

Traffic Origin

+
+
+
+{% endif %} +{% endblock %} diff --git a/plugins/traefik/templates/traefik/rows.html b/plugins/traefik/templates/traefik/rows.html new file mode 100644 index 0000000..c02b962 --- /dev/null +++ b/plugins/traefik/templates/traefik/rows.html @@ -0,0 +1,127 @@ +{# plugins/traefik/templates/traefik/rows.html #} + +{# ── Global stats bar ──────────────────────────────────────────────────────── #} +{% if global_stat %} +
+ + {% if global_stat.open_conns_total is not none %} + + Open conns + {{ global_stat.open_conns_total | int }} + + {% endif %} + + {% if global_stat.config_last_reload_success is not none %} + + Config + {% if global_stat.config_last_reload_success == 1.0 %} + ✓ OK + {% else %} + ✗ failed + {% endif %} + {% if global_stat.config_reloads_total is not none %} + ({{ global_stat.config_reloads_total | int }} reloads) + {% endif %} + + {% endif %} + + {% if global_stat.process_memory_bytes is not none %} + + Memory + {% set mb = (global_stat.process_memory_bytes / 1048576) %} + + {% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.0f"|format(mb) }} MB{% endif %} + + + {% endif %} + + + {{ global_stat.scraped_at.strftime("%H:%M:%S") }} UTC + +
+{% endif %} + +{# ── Per-service cards + cert card in shared columns layout ───────────────── #} +{% if router_data or certs %} +
+ +{% for r in router_data %} +
+

{{ r.name }}

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Req/s{{ "%.2f"|format(r.latest.request_rate) }}{{ r.sparkline_req | safe }}
Bandwidth + {% set bps = r.latest.response_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s + {% elif bps >= 1024 %}{{ "%.1f"|format(bps / 1024) }} KB/s + {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %} +
p95 ms{{ "%.1f"|format(r.latest.latency_p95_ms) }}{{ r.sparkline_p95 | safe }}
p99 ms{{ "%.1f"|format(r.latest.latency_p99_ms) }}
4xx % + {{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}% +
5xx % + {{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% + {{ r.sparkline_5xx | safe }}
+
+ Last scraped {{ r.latest.scraped_at.strftime("%H:%M:%S") }} UTC +
+
+{% endfor %} + +{% if certs %} +
+
TLS Certificates
+ + {% for cert in certs %} + + + + + + + + {% endfor %} +
{{ cert.cn }} + {{ cert.days_remaining | int }}d +
+ {{ cert.not_after.strftime("%Y-%m-%d") }} +
+
+{% endif %} + +
+{% else %} +
+

No Traefik metrics yet. Configure metrics_url in config.yaml under plugins.traefik.

+
+{% endif %} diff --git a/plugins/traefik/templates/traefik/traffic.html b/plugins/traefik/templates/traefik/traffic.html new file mode 100644 index 0000000..dcdcb96 --- /dev/null +++ b/plugins/traefik/templates/traefik/traffic.html @@ -0,0 +1,101 @@ +{# plugins/traefik/templates/traefik/traffic.html #} +{% if summary %} + +{# ── Overview bar ─────────────────────────────────────────────────────────── #} +
+ + Total + {{ summary.total }} + + + Internal + {{ summary.internal }} + + + External + + {{ summary.external }} ({{ "%.1f"|format(summary.external_pct) }}%) + + + + Bytes + + {% set mb = summary.total_bytes / 1048576 %} + {% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.1f"|format(mb) }} MB{% endif %} + + + + {# External % sparkline #} + + ext % over time + {{ summary.sparkline_external | safe }} + +
+ +{# ── Internal/external fill bar ───────────────────────────────────────────── #} +{% set ext_pct = summary.external_pct %} +
+
+
+ +{# ── Detail cards in columns ──────────────────────────────────────────────── #} +
+ + {# Top source IPs #} +
+
Top Source IPs
+ + {% for entry in summary.top_ips %} + + + {% if entry.country %} + + {% endif %} + + + {% endfor %} +
{{ entry.ip }}{{ entry.country }} + + {{ entry.count }} + +
+
+ + {# Top routers #} +
+
Top Services Hit
+ + {% for entry in summary.top_routers %} + + + + + {% endfor %} +
{{ entry.router }}{{ entry.count }}
+
+ + {# Top countries (only shown if GeoIP is active) #} + {% if summary.top_countries %} +
+
External Traffic by Country
+ + {% for entry in summary.top_countries %} + + + + + {% endfor %} +
{{ entry.country }}{{ entry.count }}
+
+ {% endif %} + +
+ +{% else %} +
+ No access log data yet. + {% if not access_log_enabled %} + Enable access_log.enabled: true in your plugin config and bind-mount the Traefik log directory. + {% endif %} +
+{% endif %} diff --git a/plugins/traefik/templates/traefik/widget.html b/plugins/traefik/templates/traefik/widget.html new file mode 100644 index 0000000..aa43085 --- /dev/null +++ b/plugins/traefik/templates/traefik/widget.html @@ -0,0 +1,56 @@ +{# plugins/traefik/templates/traefik/widget.html #} +{% if expiring_certs %} +
+{% for cert in expiring_certs %} +
+ ⚠ {{ cert.cn }} + {{ cert.days_remaining | int }}d +
+{% endfor %} +
+{% endif %} + +{% if router_data %} +{% for r in router_data %} +
+
+ {{ r.name }} +
+
+ + req/s + {{ "%.2f"|format(r.latest.request_rate) }} + + + bw + + {% set bps = r.latest.response_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s + {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %} + + + + p95 + + {{ "%.0f"|format(r.latest.latency_p95_ms) }}ms + + + {% if r.latest.error_rate_5xx_pct > 0 %} + + {{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% 5xx + + {% endif %} +
+
+{% endfor %} +{% if total_routers > 5 %} +
+ +{{ total_routers - 5 }} more — view all → +
+{% endif %} +{% else %} +

No Traefik data yet.

+{% endif %} diff --git a/plugins/traefik/templates/traefik/widget_access_log.html b/plugins/traefik/templates/traefik/widget_access_log.html new file mode 100644 index 0000000..899e09c --- /dev/null +++ b/plugins/traefik/templates/traefik/widget_access_log.html @@ -0,0 +1,40 @@ +{# traefik/widget_access_log.html — dashboard widget: traffic origin summary #} +{% if not summary %} +
No access log data yet.
+{% else %} + +
+
+ {{ summary.total | int }} + total req +
+
+ + {{ "%.1f" | format(summary.external_pct) }}% + + external +
+
+ +{% if summary.top_ips %} +
+ Top IPs + {% if summary.ip_filter != "all" %} + {{ summary.ip_filter }} + {% endif %} +
+
+ {% for ip in summary.top_ips %} +
+ + + {{ ip.ip }} + + {{ ip.count }} +
+ {% endfor %} +
+{% endif %} + +{% endif %} diff --git a/plugins/traefik/templates/traefik/widget_request_chart.html b/plugins/traefik/templates/traefik/widget_request_chart.html new file mode 100644 index 0000000..272ff39 --- /dev/null +++ b/plugins/traefik/templates/traefik/widget_request_chart.html @@ -0,0 +1,75 @@ +{# traefik/widget_request_chart.html — Chart.js request rate + error % over time #} +{% if not labels_json or labels_json == '[]' %} +
No data for last {{ hours }}h.
+{% else %} +
+ +
+ +
+ last {{ hours }}h +
+{% endif %} diff --git a/plugins/unifi/__init__.py b/plugins/unifi/__init__.py new file mode 100644 index 0000000..5dbedbb --- /dev/null +++ b/plugins/unifi/__init__.py @@ -0,0 +1,33 @@ +# plugins/unifi/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + from .models import ( # noqa: F401 + UnifiWanStat, UnifiClientSnapshot, UnifiWlanStat, UnifiDpiSnapshot, + UnifiDevice, UnifiKnownClient, UnifiEvent, UnifiAlarm, + UnifiSpeedtestResult, UnifiNetwork, UnifiPortForward, UnifiFwRule, + ) + + +def get_scheduled_tasks() -> list: + from .scheduler import make_poll_task + return [make_poll_task(_app)] + + +def get_blueprint(): + from .routes import unifi_bp + return unifi_bp + + +def get_nav() -> list[dict]: + # Surfaces in the sidebar's Infrastructure group (UniFi network overview). + return [{"label": "UniFi", "href": "/plugins/unifi/"}] diff --git a/plugins/unifi/client.py b/plugins/unifi/client.py new file mode 100644 index 0000000..d18a7d8 --- /dev/null +++ b/plugins/unifi/client.py @@ -0,0 +1,170 @@ +# plugins/unifi/client.py +"""Async UniFi Network controller API client. + +Supports UDM/UDM-Pro (firmware 2+) using the /proxy/network/ path prefix +and Bearer-token auth via the /api/auth/login endpoint. + +On 401, re-authenticates automatically once before raising. +""" +from __future__ import annotations +import logging + +import httpx + +logger = logging.getLogger(__name__) + + +class UnifiClient: + def __init__( + self, + host: str, + username: str, + password: str, + site: str = "default", + verify_ssl: bool = False, + ) -> None: + self._host = host.rstrip("/") + self._username = username + self._password = password + self._site = site + self._verify_ssl = verify_ssl + self._http: httpx.AsyncClient | None = None + self._csrf_token: str = "" + + # ── Lifecycle ───────────────────────────────────────────────────────────── + + async def _ensure_http(self) -> None: + if self._http is None: + self._http = httpx.AsyncClient( + verify=self._verify_ssl, + timeout=15.0, + follow_redirects=True, + ) + + async def login(self) -> None: + await self._ensure_http() + resp = await self._http.post( + f"{self._host}/api/auth/login", + json={"username": self._username, "password": self._password, "rememberMe": False}, + ) + resp.raise_for_status() + self._csrf_token = resp.headers.get("X-CSRF-Token", "") + logger.debug("UniFi login OK (csrf=%s…)", self._csrf_token[:8] if self._csrf_token else "none") + + async def close(self) -> None: + if self._http: + await self._http.aclose() + self._http = None + + # ── API helpers ─────────────────────────────────────────────────────────── + + def _api_url(self, path: str) -> str: + return f"{self._host}/proxy/network/api/s/{self._site}{path}" + + def _headers(self) -> dict: + return {"X-CSRF-Token": self._csrf_token} if self._csrf_token else {} + + async def _get(self, path: str, params: dict | None = None) -> list | dict: + await self._ensure_http() + resp = await self._http.get(self._api_url(path), headers=self._headers(), params=params) + if resp.status_code == 401: + await self.login() + resp = await self._http.get(self._api_url(path), headers=self._headers(), params=params) + resp.raise_for_status() + data = resp.json() + return data.get("data", data) + + # ── Core monitoring ─────────────────────────────────────────────────────── + + async def get_health(self) -> list[dict]: + """Subsystem health (wan, wlan, lan, www).""" + result = await self._get("/stat/health") + return result if isinstance(result, list) else [] + + async def get_active_clients(self) -> list[dict]: + """Currently connected clients.""" + result = await self._get("/stat/sta") + return result if isinstance(result, list) else [] + + async def get_devices(self) -> list[dict]: + """Managed network devices (APs, switches, gateways) with full detail.""" + result = await self._get("/stat/device") + return result if isinstance(result, list) else [] + + # ── Events & alarms ─────────────────────────────────────────────────────── + + async def get_events(self, limit: int = 200) -> list[dict]: + """Recent site events (client connects, IDS alerts, device state changes, etc.).""" + result = await self._get("/stat/event", params={"_limit": limit, "_sort": "-time"}) + return result if isinstance(result, list) else [] + + async def get_alarms(self, archived: bool = False) -> list[dict]: + """Active (or archived) alarms.""" + result = await self._get("/stat/alarm") + if not isinstance(result, list): + return [] + return [a for a in result if a.get("archived", False) == archived] + + # ── Traffic & DPI ───────────────────────────────────────────────────────── + + async def get_dpi_stats(self) -> list[dict]: + """DPI per-client application/category traffic stats (cumulative counters).""" + result = await self._get("/stat/dpi") + return result if isinstance(result, list) else [] + + async def get_dpi_site_stats(self) -> list[dict]: + """DPI site-level category totals (not per-client).""" + result = await self._get("/dpi") + return result if isinstance(result, list) else [] + + async def get_speedtest_status(self) -> dict | None: + """Most recent WAN speed test result.""" + result = await self._get("/stat/speedtest-status") + if isinstance(result, list) and result: + return result[0] + if isinstance(result, dict): + return result + return None + + # ── Network inventory ───────────────────────────────────────────────────── + + async def get_known_clients(self) -> list[dict]: + """All historically seen clients (not just currently connected).""" + result = await self._get("/rest/user") + return result if isinstance(result, list) else [] + + async def get_networks(self) -> list[dict]: + """Network/VLAN configurations.""" + result = await self._get("/rest/networkconf") + return result if isinstance(result, list) else [] + + async def get_wlan_configs(self) -> list[dict]: + """WLAN (SSID) configurations.""" + result = await self._get("/rest/wlanconf") + return result if isinstance(result, list) else [] + + async def get_port_forwards(self) -> list[dict]: + """Port forwarding rules.""" + result = await self._get("/rest/portforwarding") + return result if isinstance(result, list) else [] + + async def get_firewall_rules(self) -> list[dict]: + """Firewall rules.""" + result = await self._get("/rest/firewallrule") + return result if isinstance(result, list) else [] + + async def get_firewall_groups(self) -> list[dict]: + """Firewall groups (IP sets / port sets referenced by rules).""" + result = await self._get("/rest/firewallgroup") + return result if isinstance(result, list) else [] + + # ── System ──────────────────────────────────────────────────────────────── + + async def get_sysinfo(self) -> dict | None: + """Controller system information (version, uptime, hostname).""" + result = await self._get("/stat/sysinfo") + if isinstance(result, list) and result: + return result[0] + if isinstance(result, dict): + return result + return None diff --git a/plugins/unifi/migrations/__init__.py b/plugins/unifi/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/unifi/migrations/env.py b/plugins/unifi/migrations/env.py new file mode 100644 index 0000000..7fa8252 --- /dev/null +++ b/plugins/unifi/migrations/env.py @@ -0,0 +1,72 @@ +# plugins/unifi/migrations/env.py +"""Alembic env.py for the UniFi plugin (standalone dev use). +At app startup, migration_runner.py uses the core env.py with version_locations. +""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from steward.models.base import Base +import steward.models # noqa: F401 +from plugins.unifi.models import UnifiWanStat, UnifiClientSnapshot, UnifiDevice # noqa: F401 + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("STEWARD_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/unifi/migrations/versions/__init__.py b/plugins/unifi/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/unifi/migrations/versions/unifi_001_initial.py b/plugins/unifi/migrations/versions/unifi_001_initial.py new file mode 100644 index 0000000..36158a3 --- /dev/null +++ b/plugins/unifi/migrations/versions/unifi_001_initial.py @@ -0,0 +1,62 @@ +"""UniFi plugin initial tables + +Revision ID: unifi_001_initial +Revises: (none — branch off core via depends_on) +Create Date: 2026-03-20 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "unifi_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "unifi" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "unifi_wan_stats", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("is_up", sa.Boolean, nullable=False, server_default="false"), + sa.Column("wan_ip", sa.String(64), nullable=True), + sa.Column("latency_ms", sa.Float, nullable=True), + sa.Column("rx_bytes_rate", sa.Float, nullable=True), + sa.Column("tx_bytes_rate", sa.Float, nullable=True), + sa.Column("uptime_seconds", sa.Integer, nullable=True), + ) + op.create_index("ix_unifi_wan_stats_scraped", "unifi_wan_stats", ["scraped_at"]) + + op.create_table( + "unifi_client_snapshots", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("total_clients", sa.Integer, nullable=False, server_default="0"), + sa.Column("wireless_clients", sa.Integer, nullable=False, server_default="0"), + sa.Column("wired_clients", sa.Integer, nullable=False, server_default="0"), + sa.Column("guest_clients", sa.Integer, nullable=False, server_default="0"), + sa.Column("top_clients_json", sa.Text, nullable=False, server_default="[]"), + ) + op.create_index("ix_unifi_client_snapshots_scraped", "unifi_client_snapshots", ["scraped_at"]) + + op.create_table( + "unifi_devices", + sa.Column("mac", sa.String(32), primary_key=True), + sa.Column("name", sa.String(255), nullable=True), + sa.Column("model", sa.String(64), nullable=True), + sa.Column("device_type", sa.String(16), nullable=True), + sa.Column("ip", sa.String(64), nullable=True), + sa.Column("state", sa.Integer, nullable=False, server_default="0"), + sa.Column("uptime_seconds", sa.Integer, nullable=True), + sa.Column("last_seen", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("unifi_devices") + op.drop_index("ix_unifi_client_snapshots_scraped", "unifi_client_snapshots") + op.drop_table("unifi_client_snapshots") + op.drop_index("ix_unifi_wan_stats_scraped", "unifi_wan_stats") + op.drop_table("unifi_wan_stats") diff --git a/plugins/unifi/migrations/versions/unifi_002_expanded.py b/plugins/unifi/migrations/versions/unifi_002_expanded.py new file mode 100644 index 0000000..7336517 --- /dev/null +++ b/plugins/unifi/migrations/versions/unifi_002_expanded.py @@ -0,0 +1,156 @@ +"""UniFi expanded tables: wlan stats, dpi, events, alarms, known clients, +speedtest results, networks, port forwards, firewall rules. +Also adds missing 'version' column to unifi_devices. + +Revision ID: unifi_002_expanded +Revises: unifi_001_initial +Create Date: 2026-03-20 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "unifi_002_expanded" +down_revision: Union[str, None] = "unifi_001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ── Patch unifi_devices to add missing version column ───────────────────── + op.add_column("unifi_devices", sa.Column("version", sa.String(64), nullable=True)) + + # ── WLAN stats (time-series per SSID+radio) ─────────────────────────────── + op.create_table( + "unifi_wlan_stats", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("ssid", sa.String(255), nullable=False), + sa.Column("radio", sa.String(8), nullable=True), + sa.Column("num_sta", sa.Integer, nullable=False, server_default="0"), + sa.Column("tx_bytes", sa.Float, nullable=True), + sa.Column("rx_bytes", sa.Float, nullable=True), + sa.Column("channel", sa.Integer, nullable=True), + sa.Column("satisfaction", sa.Integer, nullable=True), + ) + op.create_index("ix_unifi_wlan_stats_scraped", "unifi_wlan_stats", ["scraped_at"]) + + # ── DPI snapshots (site-level category totals) ──────────────────────────── + op.create_table( + "unifi_dpi_snapshots", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("by_category_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("by_client_json", sa.Text, nullable=False, server_default="[]"), + ) + op.create_index("ix_unifi_dpi_scraped", "unifi_dpi_snapshots", ["scraped_at"]) + + # ── Known clients (upserted by MAC) ─────────────────────────────────────── + op.create_table( + "unifi_known_clients", + sa.Column("mac", sa.String(32), primary_key=True), + sa.Column("hostname", sa.String(255), nullable=True), + sa.Column("name", sa.String(255), nullable=True), + sa.Column("ip", sa.String(64), nullable=True), + sa.Column("mac_vendor", sa.String(128), nullable=True), + sa.Column("note", sa.Text, nullable=True), + sa.Column("is_blocked", sa.Boolean, nullable=False, server_default="false"), + sa.Column("first_seen", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_seen", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + # ── Events (upserted by UniFi event ID) ─────────────────────────────────── + op.create_table( + "unifi_events", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("event_time", sa.DateTime(timezone=True), nullable=False), + sa.Column("event_key", sa.String(64), nullable=False), + sa.Column("category", sa.String(16), nullable=False), + sa.Column("message", sa.Text, nullable=False), + sa.Column("source_mac", sa.String(32), nullable=True), + sa.Column("source_ip", sa.String(64), nullable=True), + sa.Column("details_json", sa.Text, nullable=False, server_default="{}"), + ) + op.create_index("ix_unifi_events_time", "unifi_events", ["event_time"]) + + # ── Alarms (upserted by UniFi alarm ID) ─────────────────────────────────── + op.create_table( + "unifi_alarms", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("alarm_time", sa.DateTime(timezone=True), nullable=False), + sa.Column("alarm_key", sa.String(64), nullable=False), + sa.Column("message", sa.Text, nullable=False), + sa.Column("archived", sa.Boolean, nullable=False, server_default="false"), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index("ix_unifi_alarms_time", "unifi_alarms", ["alarm_time"]) + + # ── Speedtest results (upserted by run timestamp) ───────────────────────── + op.create_table( + "unifi_speedtest_results", + sa.Column("run_at", sa.DateTime(timezone=True), primary_key=True), + sa.Column("download_mbps", sa.Float, nullable=True), + sa.Column("upload_mbps", sa.Float, nullable=True), + sa.Column("latency_ms", sa.Float, nullable=True), + sa.Column("server_name", sa.String(255), nullable=True), + ) + + # ── Networks/VLANs (upserted by UniFi ID) ──────────────────────────────── + op.create_table( + "unifi_networks", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("purpose", sa.String(32), nullable=True), + sa.Column("vlan", sa.Integer, nullable=True), + sa.Column("ip_subnet", sa.String(64), nullable=True), + sa.Column("dhcp_enabled", sa.Boolean, nullable=False, server_default="false"), + sa.Column("is_guest", sa.Boolean, nullable=False, server_default="false"), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + # ── Port forwards (upserted by UniFi ID) ───────────────────────────────── + op.create_table( + "unifi_port_forwards", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("name", sa.String(255), nullable=True), + sa.Column("proto", sa.String(8), nullable=False, server_default="tcp"), + sa.Column("dst_port", sa.String(32), nullable=False), + sa.Column("fwd_ip", sa.String(64), nullable=False), + sa.Column("fwd_port", sa.String(32), nullable=False), + sa.Column("src_filter", sa.String(255), nullable=True), + sa.Column("enabled", sa.Boolean, nullable=False, server_default="true"), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + # ── Firewall rules (upserted by UniFi ID) ──────────────────────────────── + op.create_table( + "unifi_fw_rules", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("ruleset", sa.String(16), nullable=False), + sa.Column("rule_index", sa.Integer, nullable=True), + sa.Column("action", sa.String(16), nullable=False), + sa.Column("protocol", sa.String(16), nullable=True), + sa.Column("enabled", sa.Boolean, nullable=False, server_default="true"), + sa.Column("src_address", sa.String(255), nullable=True), + sa.Column("dst_address", sa.String(255), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("unifi_fw_rules") + op.drop_table("unifi_port_forwards") + op.drop_table("unifi_networks") + op.drop_table("unifi_speedtest_results") + op.drop_index("ix_unifi_alarms_time", "unifi_alarms") + op.drop_table("unifi_alarms") + op.drop_index("ix_unifi_events_time", "unifi_events") + op.drop_table("unifi_events") + op.drop_table("unifi_known_clients") + op.drop_index("ix_unifi_dpi_scraped", "unifi_dpi_snapshots") + op.drop_table("unifi_dpi_snapshots") + op.drop_index("ix_unifi_wlan_stats_scraped", "unifi_wlan_stats") + op.drop_table("unifi_wlan_stats") + op.drop_column("unifi_devices", "version") diff --git a/plugins/unifi/models.py b/plugins/unifi/models.py new file mode 100644 index 0000000..bb36410 --- /dev/null +++ b/plugins/unifi/models.py @@ -0,0 +1,182 @@ +# plugins/unifi/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from steward.models.base import Base + + +# ── Time-series (one row per scrape) ────────────────────────────────────────── + +class UnifiWanStat(Base): + """WAN interface health — one row per scrape.""" + __tablename__ = "unifi_wan_stats" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + wan_ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True) + rx_bytes_rate: Mapped[float | None] = mapped_column(Float, nullable=True) + tx_bytes_rate: Mapped[float | None] = mapped_column(Float, nullable=True) + uptime_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) + + +class UnifiClientSnapshot(Base): + """Aggregated client counts + top talkers — one row per scrape.""" + __tablename__ = "unifi_client_snapshots" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + total_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + wireless_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + wired_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + guest_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + top_clients_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + # JSON: [{"mac","hostname","ip","type","tx_rate","rx_rate","signal","essid"}] + + +class UnifiWlanStat(Base): + """Per-SSID/radio stats — one row per scrape per SSID+radio combination.""" + __tablename__ = "unifi_wlan_stats" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + ssid: Mapped[str] = mapped_column(String(255), nullable=False) + radio: Mapped[str | None] = mapped_column(String(8), nullable=True) # ng=2.4GHz, na=5GHz, 6e=6GHz + num_sta: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + tx_bytes: Mapped[float | None] = mapped_column(Float, nullable=True) + rx_bytes: Mapped[float | None] = mapped_column(Float, nullable=True) + channel: Mapped[int | None] = mapped_column(Integer, nullable=True) + satisfaction: Mapped[int | None] = mapped_column(Integer, nullable=True) # 0-100 + + +class UnifiDpiSnapshot(Base): + """DPI application/category traffic snapshot — one row per scrape (JSON blob).""" + __tablename__ = "unifi_dpi_snapshots" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + # JSON: [{"cat_id", "cat_name", "rx_bytes", "tx_bytes"}] aggregated across all clients + by_category_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + # JSON: [{"mac", "hostname", "top_cats": [{"cat_name", "bytes"}]}] + by_client_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + + +# ── Upserted inventory (keyed by UniFi ID or MAC) ───────────────────────────── + +class UnifiDevice(Base): + """Network infrastructure devices — upserted by MAC each scrape.""" + __tablename__ = "unifi_devices" + + mac: Mapped[str] = mapped_column(String(32), primary_key=True) + name: Mapped[str | None] = mapped_column(String(255), nullable=True) + model: Mapped[str | None] = mapped_column(String(64), nullable=True) + device_type: Mapped[str | None] = mapped_column(String(16), nullable=True) + ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + state: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + uptime_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) + last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + version: Mapped[str | None] = mapped_column(String(64), nullable=True) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiKnownClient(Base): + """All historically seen network clients — upserted by MAC.""" + __tablename__ = "unifi_known_clients" + + mac: Mapped[str] = mapped_column(String(32), primary_key=True) + hostname: Mapped[str | None] = mapped_column(String(255), nullable=True) + name: Mapped[str | None] = mapped_column(String(255), nullable=True) # user-set alias + ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + mac_vendor: Mapped[str | None] = mapped_column(String(128), nullable=True) + note: Mapped[str | None] = mapped_column(Text, nullable=True) + is_blocked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + first_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiEvent(Base): + """Site event log — upserted by UniFi event ID to avoid duplicates.""" + __tablename__ = "unifi_events" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + event_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + event_key: Mapped[str] = mapped_column(String(64), nullable=False) # e.g. EVT_WU_Connected + category: Mapped[str] = mapped_column(String(16), nullable=False) # wlan, wired, ap, ids, wan, system + message: Mapped[str] = mapped_column(Text, nullable=False) + source_mac: Mapped[str | None] = mapped_column(String(32), nullable=True) + source_ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + details_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}") + + +class UnifiAlarm(Base): + """Active alarms — upserted by UniFi alarm ID.""" + __tablename__ = "unifi_alarms" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + alarm_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + alarm_key: Mapped[str] = mapped_column(String(64), nullable=False) + message: Mapped[str] = mapped_column(Text, nullable=False) + archived: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiSpeedtestResult(Base): + """WAN speed test results — upserted by run timestamp.""" + __tablename__ = "unifi_speedtest_results" + + run_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), primary_key=True) + download_mbps: Mapped[float | None] = mapped_column(Float, nullable=True) + upload_mbps: Mapped[float | None] = mapped_column(Float, nullable=True) + latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True) + server_name: Mapped[str | None] = mapped_column(String(255), nullable=True) + + +# ── Config inventory (upserted by UniFi ID) ─────────────────────────────────── + +class UnifiNetwork(Base): + """VLAN/network configurations — upserted by UniFi ID.""" + __tablename__ = "unifi_networks" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + purpose: Mapped[str | None] = mapped_column(String(32), nullable=True) # corporate, guest, vlan-only + vlan: Mapped[int | None] = mapped_column(Integer, nullable=True) + ip_subnet: Mapped[str | None] = mapped_column(String(64), nullable=True) + dhcp_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_guest: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiPortForward(Base): + """Port forwarding rules — upserted by UniFi ID.""" + __tablename__ = "unifi_port_forwards" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + name: Mapped[str | None] = mapped_column(String(255), nullable=True) + proto: Mapped[str] = mapped_column(String(8), nullable=False, default="tcp") + dst_port: Mapped[str] = mapped_column(String(32), nullable=False) + fwd_ip: Mapped[str] = mapped_column(String(64), nullable=False) + fwd_port: Mapped[str] = mapped_column(String(32), nullable=False) + src_filter: Mapped[str | None] = mapped_column(String(255), nullable=True) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiFwRule(Base): + """Firewall rules — upserted by UniFi ID.""" + __tablename__ = "unifi_fw_rules" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + ruleset: Mapped[str] = mapped_column(String(16), nullable=False) # WAN_IN, LAN_IN, etc. + rule_index: Mapped[int | None] = mapped_column(Integer, nullable=True) + action: Mapped[str] = mapped_column(String(16), nullable=False) # accept, drop, reject + protocol: Mapped[str | None] = mapped_column(String(16), nullable=True) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + src_address: Mapped[str | None] = mapped_column(String(255), nullable=True) + dst_address: Mapped[str | None] = mapped_column(String(255), nullable=True) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) diff --git a/plugins/unifi/plugin.yaml b/plugins/unifi/plugin.yaml new file mode 100644 index 0000000..76b64ed --- /dev/null +++ b/plugins/unifi/plugin.yaml @@ -0,0 +1,21 @@ +name: unifi +version: "1.0.0" +description: "UniFi Network controller integration — WAN health, devices, clients, DPI" +author: "Steward" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi" +tags: + - network + - unifi + - ubiquiti + +config: + host: "https://192.168.1.1" + username: "admin" + password: "" + site: "default" + verify_ssl: false + poll_interval_seconds: 60 + top_clients: 10 # number of top-talker clients to store per snapshot diff --git a/plugins/unifi/routes.py b/plugins/unifi/routes.py new file mode 100644 index 0000000..e5e00a6 --- /dev/null +++ b/plugins/unifi/routes.py @@ -0,0 +1,356 @@ +# plugins/unifi/routes.py +from __future__ import annotations +import json +from quart import Blueprint, current_app, render_template, request +from sqlalchemy import Integer, cast, func, select + +from steward.auth.middleware import require_role +from steward.models.users import UserRole +from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds +from .models import ( + UnifiAlarm, UnifiClientSnapshot, UnifiDevice, UnifiDpiSnapshot, + UnifiEvent, UnifiFwRule, UnifiKnownClient, UnifiNetwork, + UnifiPortForward, UnifiSpeedtestResult, UnifiWanStat, UnifiWlanStat, +) + +unifi_bp = Blueprint("unifi", __name__, template_folder="templates") + + +def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: + if len(values) < 2: + return f'' + mn, mx = min(values), max(values) + if mx == mn: + mx = mn + 1.0 + step = width / (len(values) - 1) + pts = [] + for i, v in enumerate(values): + x = i * step + y = height - (v - mn) / (mx - mn) * (height - 2) - 1 + pts.append(f"{x:.1f},{y:.1f}") + poly = " ".join(pts) + return ( + f'' + f'' + f'' + ) + + +@unifi_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + current_range = request.args.get("range", DEFAULT_RANGE) + return await render_template( + "unifi/index.html", + poll_interval=poll_interval, + current_range=current_range, + ) + + +@unifi_bp.get("/rows") +@require_role(UserRole.viewer) +async def rows(): + """HTMX fragment: WAN stats, devices, clients, WLAN — scoped to selected range.""" + since, range_key = parse_range(request.args.get("range")) + + b_secs = bucket_seconds(since) + + async with current_app.db_sessionmaker() as db: + # WAN history — bucketed for accurate full-range sparklines + wan_bucket = ( + cast(func.strftime('%s', UnifiWanStat.scraped_at), Integer) / b_secs + ).label("bucket") + result = await db.execute( + select( + func.avg(UnifiWanStat.latency_ms).label("latency_ms"), + func.avg(UnifiWanStat.rx_bytes_rate).label("rx_bytes_rate"), + func.avg(UnifiWanStat.tx_bytes_rate).label("tx_bytes_rate"), + func.min(UnifiWanStat.scraped_at).label("scraped_at"), + wan_bucket, + ) + .where(UnifiWanStat.scraped_at >= since) + .group_by(wan_bucket) + .order_by(wan_bucket) + ) + wan_history = result.all() + + # Latest WAN — always most recent raw row for current displayed values + result = await db.execute( + select(UnifiWanStat).order_by(UnifiWanStat.scraped_at.desc()).limit(1) + ) + latest_wan = result.scalar_one_or_none() + + # Latest client snapshot (always most recent) + result = await db.execute( + select(UnifiClientSnapshot) + .order_by(UnifiClientSnapshot.scraped_at.desc()) + .limit(1) + ) + latest_clients = result.scalar_one_or_none() + + # All devices + result = await db.execute( + select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name) + ) + devices = result.scalars().all() + + # Client count history — bucketed + cs_bucket = ( + cast(func.strftime('%s', UnifiClientSnapshot.scraped_at), Integer) / b_secs + ).label("bucket") + result = await db.execute( + select( + func.avg(UnifiClientSnapshot.total_clients).label("total_clients"), + func.min(UnifiClientSnapshot.scraped_at).label("scraped_at"), + cs_bucket, + ) + .where(UnifiClientSnapshot.scraped_at >= since) + .group_by(cs_bucket) + .order_by(cs_bucket) + ) + client_history = result.all() + + # Latest WLAN stats (always most recent per SSID) + result = await db.execute( + select(UnifiWlanStat.ssid).distinct().order_by(UnifiWlanStat.ssid) + ) + ssids = [row[0] for row in result.all()] + wlan_latest: list[UnifiWlanStat] = [] + for ssid in ssids: + result = await db.execute( + select(UnifiWlanStat) + .where(UnifiWlanStat.ssid == ssid) + .order_by(UnifiWlanStat.scraped_at.desc()) + .limit(1) + ) + w = result.scalar_one_or_none() + if w: + wlan_latest.append(w) + + # Latest speedtest + result = await db.execute( + select(UnifiSpeedtestResult) + .order_by(UnifiSpeedtestResult.run_at.desc()) + .limit(1) + ) + speedtest = result.scalar_one_or_none() + + top_clients = json.loads(latest_clients.top_clients_json) if latest_clients else [] + + sparkline_latency = _sparkline([r.latency_ms or 0 for r in wan_history]) + sparkline_rx = _sparkline([r.rx_bytes_rate or 0 for r in wan_history]) + sparkline_tx = _sparkline([r.tx_bytes_rate or 0 for r in wan_history]) + sparkline_clients = _sparkline([r.total_clients for r in client_history]) + + return await render_template( + "unifi/rows.html", + latest_wan=latest_wan, + latest_clients=latest_clients, + devices=devices, + top_clients=top_clients, + wlan_latest=wlan_latest, + speedtest=speedtest, + sparkline_latency=sparkline_latency, + sparkline_rx=sparkline_rx, + sparkline_tx=sparkline_tx, + sparkline_clients=sparkline_clients, + range_key=range_key, + ) + + +@unifi_bp.get("/events") +@require_role(UserRole.viewer) +async def events(): + """HTMX fragment: site events within selected range.""" + since, range_key = parse_range(request.args.get("range")) + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiEvent) + .where(UnifiEvent.event_time >= since) + .order_by(UnifiEvent.event_time.desc()) + .limit(500) + ) + event_rows = result.scalars().all() + + return await render_template( + "unifi/events.html", events=event_rows, range_key=range_key + ) + + +@unifi_bp.get("/dpi") +@require_role(UserRole.viewer) +async def dpi(): + """HTMX fragment: DPI traffic breakdown (latest snapshot).""" + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiDpiSnapshot) + .order_by(UnifiDpiSnapshot.scraped_at.desc()) + .limit(1) + ) + snapshot = result.scalar_one_or_none() + + categories = [] + top_clients = [] + if snapshot: + categories = json.loads(snapshot.by_category_json) + top_clients = json.loads(snapshot.by_client_json) + + return await render_template( + "unifi/dpi.html", categories=categories, top_clients=top_clients, snapshot=snapshot + ) + + +@unifi_bp.get("/security") +@require_role(UserRole.viewer) +async def security(): + """HTMX fragment: active and archived alarms.""" + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiAlarm) + .where(UnifiAlarm.archived == False) # noqa: E712 + .order_by(UnifiAlarm.alarm_time.desc()) + ) + active_alarms = result.scalars().all() + + result = await db.execute( + select(UnifiAlarm) + .where(UnifiAlarm.archived == True) # noqa: E712 + .order_by(UnifiAlarm.alarm_time.desc()) + .limit(20) + ) + archived_alarms = result.scalars().all() + + return await render_template( + "unifi/security.html", + active_alarms=active_alarms, + archived_alarms=archived_alarms, + ) + + +@unifi_bp.get("/inventory") +@require_role(UserRole.viewer) +async def inventory(): + """HTMX fragment: known clients, networks, port forwards, firewall rules.""" + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiKnownClient) + .order_by(UnifiKnownClient.last_seen.desc().nullslast()) + ) + known_clients = result.scalars().all() + + result = await db.execute( + select(UnifiNetwork).order_by(UnifiNetwork.name) + ) + networks = result.scalars().all() + + result = await db.execute( + select(UnifiPortForward).order_by(UnifiPortForward.name) + ) + port_forwards = result.scalars().all() + + result = await db.execute( + select(UnifiFwRule) + .order_by(UnifiFwRule.ruleset, UnifiFwRule.rule_index) + ) + fw_rules = result.scalars().all() + + return await render_template( + "unifi/inventory.html", + known_clients=known_clients, + networks=networks, + port_forwards=port_forwards, + fw_rules=fw_rules, + ) + + +@unifi_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """HTMX dashboard widget fragment.""" + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiWanStat).order_by(UnifiWanStat.scraped_at.desc()).limit(1) + ) + latest_wan = result.scalar_one_or_none() + + result = await db.execute( + select(UnifiClientSnapshot).order_by(UnifiClientSnapshot.scraped_at.desc()).limit(1) + ) + latest_clients = result.scalar_one_or_none() + + result = await db.execute(select(UnifiDevice)) + devices = result.scalars().all() + + result = await db.execute( + select(UnifiAlarm).where(UnifiAlarm.archived == False) # noqa: E712 + ) + active_alarms = result.scalars().all() + + offline_devices = [d for d in devices if d.state != 1] + top_clients = json.loads(latest_clients.top_clients_json)[:5] if latest_clients else [] + + return await render_template( + "unifi/widget.html", + latest_wan=latest_wan, + latest_clients=latest_clients, + offline_devices=offline_devices, + active_alarms=active_alarms, + top_clients=top_clients, + ) + + +@unifi_bp.get("/widget/clients") +@require_role(UserRole.viewer) +async def widget_clients(): + """HTMX dashboard widget: top active clients.""" + limit = max(1, min(20, int(request.args.get("limit", 5) or 5))) + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiClientSnapshot) + .order_by(UnifiClientSnapshot.scraped_at.desc()) + .limit(1) + ) + snapshot = result.scalar_one_or_none() + + clients = json.loads(snapshot.top_clients_json)[:limit] if snapshot else [] + return await render_template( + "unifi/widget_clients.html", + clients=clients, + snapshot=snapshot, + widget_id=widget_id, + ) + + +@unifi_bp.get("/widget/devices") +@require_role(UserRole.viewer) +async def widget_devices(): + """HTMX dashboard widget: infrastructure devices with optional type filter.""" + type_filter = request.args.get("type_filter", "all") + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name) + ) + all_devices = result.scalars().all() + + if type_filter == "wireless": + devices = [d for d in all_devices if d.device_type == "uap"] + elif type_filter == "wired": + devices = [d for d in all_devices if d.device_type != "uap"] + elif type_filter == "offline": + devices = [d for d in all_devices if d.state != 1] + else: + devices = list(all_devices) + + return await render_template( + "unifi/widget_devices.html", + devices=devices, + type_filter=type_filter, + widget_id=widget_id, + ) diff --git a/plugins/unifi/scheduler.py b/plugins/unifi/scheduler.py new file mode 100644 index 0000000..cfdeab6 --- /dev/null +++ b/plugins/unifi/scheduler.py @@ -0,0 +1,541 @@ +# plugins/unifi/scheduler.py +from __future__ import annotations +import json +import logging +from datetime import datetime, timezone +from typing import TYPE_CHECKING + +from steward.core.scheduler import ScheduledTask + +if TYPE_CHECKING: + from quart import Quart + +logger = logging.getLogger(__name__) + +_client = None # UnifiClient instance, initialised on first scrape + + +async def _drop_client() -> None: + """Close and discard the cached client. + + The client holds a long-lived ``httpx.AsyncClient`` (a connection pool over + real sockets). Nulling the reference without ``aclose()`` orphans that pool + until GC, so every failure tick that re-auths would leak file descriptors — + the same class of bug that took the app down via SNMP (Errno 24, "Too many + open files"). Close first, then drop. Best-effort: a failed close must not + break the poll loop. + """ + global _client + if _client is not None: + try: + await _client.close() + except Exception: + pass + _client = None + + +def make_poll_task(app: "Quart") -> ScheduledTask: + cfg = app.config["PLUGINS"]["unifi"] + interval = int(cfg.get("poll_interval_seconds", 60)) + + async def poll() -> None: + await _do_poll(app) + + return ScheduledTask( + name="unifi_poll", + coro_factory=poll, + interval_seconds=interval, + run_on_startup=True, + ) + + +async def _do_poll(app: "Quart") -> None: + global _client + + from .client import UnifiClient + from .models import UnifiClientSnapshot, UnifiDevice, UnifiWanStat + from steward.core.alerts import record_metric + + cfg = app.config["PLUGINS"]["unifi"] + + if _client is None: + _client = UnifiClient( + host=cfg["host"], + username=cfg["username"], + password=cfg["password"], + site=cfg.get("site", "default"), + verify_ssl=cfg.get("verify_ssl", False), + ) + try: + await _client.login() + except Exception as exc: + logger.warning("UniFi initial login failed: %s", exc) + await _drop_client() + return + + try: + health = await _client.get_health() + clients = await _client.get_active_clients() + devices = await _client.get_devices() + except Exception as exc: + logger.warning("UniFi poll failed — will retry next tick: %s", exc) + await _drop_client() # close + force re-auth on next tick + return + + scraped_at = datetime.now(timezone.utc) + top_n = int(cfg.get("top_clients", 10)) + + # ── WAN stat ────────────────────────────────────────────────────────────── + wan_data = next((h for h in health if h.get("subsystem") == "wan"), None) + wan_stat = UnifiWanStat(scraped_at=scraped_at) + if wan_data: + wan_stat.is_up = wan_data.get("status") == "ok" + wan_stat.wan_ip = wan_data.get("wan_ip") + wan_stat.latency_ms = wan_data.get("latency") + wan_stat.rx_bytes_rate = wan_data.get("rx_bytes-r") + wan_stat.tx_bytes_rate = wan_data.get("tx_bytes-r") + wan_stat.uptime_seconds = wan_data.get("uptime") + + # ── Client snapshot ─────────────────────────────────────────────────────── + wireless = [c for c in clients if not c.get("is_wired", True)] + wired = [c for c in clients if c.get("is_wired", True)] + guests = [c for c in clients if c.get("is_guest", False)] + + top_clients = sorted( + clients, + key=lambda c: (c.get("tx_bytes-r", 0) or 0) + (c.get("rx_bytes-r", 0) or 0), + reverse=True, + )[:top_n] + + top_clients_data = [ + { + "mac": c.get("mac", ""), + "hostname": c.get("hostname") or c.get("name") or c.get("mac", ""), + "ip": c.get("ip", ""), + "type": "wireless" if not c.get("is_wired", True) else "wired", + "tx_rate": c.get("tx_bytes-r", 0) or 0, + "rx_rate": c.get("rx_bytes-r", 0) or 0, + "signal": c.get("signal"), + "essid": c.get("essid"), + } + for c in top_clients + ] + + client_snapshot = UnifiClientSnapshot( + scraped_at=scraped_at, + total_clients=len(clients), + wireless_clients=len(wireless), + wired_clients=len(wired), + guest_clients=len(guests), + top_clients_json=json.dumps(top_clients_data), + ) + + async with app.db_sessionmaker() as session: + async with session.begin(): + session.add(wan_stat) + session.add(client_snapshot) + + # ── Devices (upsert by MAC) ─────────────────────────────────────── + for d in devices: + mac = d.get("mac", "") + if not mac: + continue + last_seen_ts = d.get("last_seen") + last_seen_dt = ( + datetime.fromtimestamp(last_seen_ts, tz=timezone.utc) + if last_seen_ts else None + ) + existing = await session.get(UnifiDevice, mac) + if existing: + existing.name = d.get("name") + existing.model = d.get("model") + existing.device_type = d.get("type") + existing.ip = d.get("ip") + existing.state = d.get("state", 0) + existing.uptime_seconds = d.get("uptime") + existing.last_seen = last_seen_dt + existing.version = d.get("version") + existing.scraped_at = scraped_at + else: + session.add(UnifiDevice( + mac=mac, + name=d.get("name"), + model=d.get("model"), + device_type=d.get("type"), + ip=d.get("ip"), + state=d.get("state", 0), + uptime_seconds=d.get("uptime"), + last_seen=last_seen_dt, + version=d.get("version"), + scraped_at=scraped_at, + )) + + # ── Alert metrics ───────────────────────────────────────────────── + if wan_stat.latency_ms is not None: + await record_metric( + session=session, + source_module="unifi", + resource_name="wan", + metric_name="latency_ms", + value=wan_stat.latency_ms, + ) + await record_metric( + session=session, + source_module="unifi", + resource_name="wan", + metric_name="is_up", + value=1.0 if wan_stat.is_up else 0.0, + ) + await record_metric( + session=session, + source_module="unifi", + resource_name="clients", + metric_name="total_clients", + value=float(len(clients)), + ) + + logger.debug( + "UniFi poll: wan=%s, clients=%d, devices=%d", + "up" if wan_stat.is_up else "down", + len(clients), + len(devices), + ) + + # ── Expanded data (best-effort, failures don't abort core poll) ─────────── + try: + await _do_expanded(app, scraped_at) + except Exception as exc: + logger.warning("UniFi expanded poll failed: %s", exc) + + +async def _do_expanded(app: "Quart", scraped_at: datetime) -> None: + """Fetch supplementary UniFi data: WLAN stats, events, alarms, DPI, + known clients, speedtest, networks, port forwards, firewall rules.""" + from .models import ( + UnifiAlarm, UnifiDpiSnapshot, UnifiEvent, UnifiFwRule, + UnifiKnownClient, UnifiNetwork, UnifiPortForward, UnifiSpeedtestResult, + UnifiWlanStat, + ) + + # ── Fetch from controller ───────────────────────────────────────────────── + try: + wlan_configs = await _client.get_wlan_configs() + except Exception: + wlan_configs = [] + + try: + events = await _client.get_events(limit=200) + except Exception: + events = [] + + try: + alarms = await _client.get_alarms(archived=False) + except Exception: + alarms = [] + + try: + dpi_cats = await _client.get_dpi_site_stats() + except Exception: + dpi_cats = [] + + try: + dpi_clients = await _client.get_dpi_stats() + except Exception: + dpi_clients = [] + + try: + known_clients = await _client.get_known_clients() + except Exception: + known_clients = [] + + try: + speedtest = await _client.get_speedtest_status() + except Exception: + speedtest = None + + try: + networks = await _client.get_networks() + except Exception: + networks = [] + + try: + port_forwards = await _client.get_port_forwards() + except Exception: + port_forwards = [] + + try: + fw_rules = await _client.get_firewall_rules() + except Exception: + fw_rules = [] + + # ── Derive WLAN stats from device VAP table ─────────────────────────────── + # WLAN usage stats live on the device objects' vap_table, not wlan configs. + # We get per-radio SSID data from the active clients grouped by essid. + # Store one row per SSID using wlan_configs for channel/satisfaction. + wlan_rows: list[UnifiWlanStat] = [] + for wc in wlan_configs: + ssid = wc.get("name") or wc.get("x_passphrase", "") + if not ssid: + continue + wlan_rows.append(UnifiWlanStat( + scraped_at=scraped_at, + ssid=ssid, + radio=None, + num_sta=wc.get("num_sta", 0) or 0, + tx_bytes=wc.get("tx_bytes"), + rx_bytes=wc.get("rx_bytes"), + channel=None, + satisfaction=wc.get("satisfaction"), + )) + + # ── Build DPI category summary ──────────────────────────────────────────── + cat_summary = [] + for entry in dpi_cats: + cat_id = entry.get("cat") or entry.get("cat_id") + cat_name = entry.get("cat_name") or str(cat_id) + rx = entry.get("rx_bytes", 0) or 0 + tx = entry.get("tx_bytes", 0) or 0 + if rx or tx: + cat_summary.append({"cat_id": cat_id, "cat_name": cat_name, "rx_bytes": rx, "tx_bytes": tx}) + cat_summary.sort(key=lambda x: x["rx_bytes"] + x["tx_bytes"], reverse=True) + + # Top 10 clients by DPI total + client_dpi: dict[str, dict] = {} + for entry in dpi_clients: + mac = entry.get("mac", "") + if not mac: + continue + if mac not in client_dpi: + client_dpi[mac] = {"mac": mac, "hostname": entry.get("hostname", mac), "bytes": 0, "top_cats": []} + client_dpi[mac]["bytes"] += (entry.get("rx_bytes", 0) or 0) + (entry.get("tx_bytes", 0) or 0) + top_dpi_clients = sorted(client_dpi.values(), key=lambda x: x["bytes"], reverse=True)[:10] + + # ── Write to DB ─────────────────────────────────────────────────────────── + async with app.db_sessionmaker() as session: + async with session.begin(): + # WLAN stats + for row in wlan_rows: + session.add(row) + + # DPI snapshot (only if we got data) + if cat_summary or top_dpi_clients: + session.add(UnifiDpiSnapshot( + scraped_at=scraped_at, + by_category_json=json.dumps(cat_summary), + by_client_json=json.dumps(top_dpi_clients), + )) + + # Events (upsert by _id) + _category_map = { + "wlan": "wlan", "wireless": "wlan", + "wired": "wired", + "ugw": "wan", "wan": "wan", + "ids": "ids", + "system": "system", + "ap": "ap", + } + for ev in events: + uid = ev.get("_id", "") + if not uid: + continue + ts_raw = ev.get("datetime") or ev.get("time") + try: + if isinstance(ts_raw, (int, float)): + event_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc) + else: + event_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) + except Exception: + event_time = scraped_at + subsystem = ev.get("subsystem", "system") + category = _category_map.get(subsystem, subsystem[:16]) + details = {k: v for k, v in ev.items() + if k not in ("_id", "datetime", "time", "key", "subsystem", "msg", "user", "src_ip")} + existing = await session.get(UnifiEvent, uid) + if not existing: + session.add(UnifiEvent( + unifi_id=uid, + event_time=event_time, + event_key=ev.get("key", ""), + category=category, + message=ev.get("msg", ""), + source_mac=ev.get("user") or ev.get("src_mac"), + source_ip=ev.get("src_ip"), + details_json=json.dumps(details), + )) + + # Alarms (upsert) + for al in alarms: + uid = al.get("_id", "") + if not uid: + continue + ts_raw = al.get("datetime") or al.get("time") + try: + if isinstance(ts_raw, (int, float)): + alarm_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc) + else: + alarm_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) + except Exception: + alarm_time = scraped_at + existing = await session.get(UnifiAlarm, uid) + if existing: + existing.archived = al.get("archived", False) + existing.scraped_at = scraped_at + else: + session.add(UnifiAlarm( + unifi_id=uid, + alarm_time=alarm_time, + alarm_key=al.get("key", ""), + message=al.get("msg", ""), + archived=al.get("archived", False), + scraped_at=scraped_at, + )) + + # Known clients (upsert by MAC) + for kc in known_clients: + mac = kc.get("mac", "") + if not mac: + continue + def _ts(val): + if not val: + return None + try: + if isinstance(val, (int, float)): + return datetime.fromtimestamp(val, tz=timezone.utc) + return datetime.fromisoformat(str(val).replace("Z", "+00:00")) + except Exception: + return None + existing = await session.get(UnifiKnownClient, mac) + if existing: + existing.hostname = kc.get("hostname") + existing.name = kc.get("name") + existing.ip = kc.get("ip") + existing.mac_vendor = kc.get("oui") + existing.note = kc.get("note") + existing.is_blocked = kc.get("blocked", False) + existing.first_seen = _ts(kc.get("first_seen")) + existing.last_seen = _ts(kc.get("last_seen")) + existing.scraped_at = scraped_at + else: + session.add(UnifiKnownClient( + mac=mac, + hostname=kc.get("hostname"), + name=kc.get("name"), + ip=kc.get("ip"), + mac_vendor=kc.get("oui"), + note=kc.get("note"), + is_blocked=kc.get("blocked", False), + first_seen=_ts(kc.get("first_seen")), + last_seen=_ts(kc.get("last_seen")), + scraped_at=scraped_at, + )) + + # Speedtest (upsert by run_at) + if speedtest: + ts_raw = speedtest.get("rundate") or speedtest.get("time") + try: + if isinstance(ts_raw, (int, float)): + run_at = datetime.fromtimestamp(ts_raw, tz=timezone.utc) + else: + run_at = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) + except Exception: + run_at = scraped_at + existing = await session.get(UnifiSpeedtestResult, run_at) + if not existing: + session.add(UnifiSpeedtestResult( + run_at=run_at, + download_mbps=speedtest.get("xput_download"), + upload_mbps=speedtest.get("xput_upload"), + latency_ms=speedtest.get("latency"), + server_name=speedtest.get("server_name"), + )) + + # Networks (upsert by UniFi ID) + for net in networks: + uid = net.get("_id", "") + if not uid: + continue + existing = await session.get(UnifiNetwork, uid) + if existing: + existing.name = net.get("name", "") + existing.purpose = net.get("purpose") + existing.vlan = net.get("vlan") + existing.ip_subnet = net.get("ip_subnet") + existing.dhcp_enabled = net.get("dhcpd_enabled", False) + existing.is_guest = net.get("is_guest", False) + existing.scraped_at = scraped_at + else: + session.add(UnifiNetwork( + unifi_id=uid, + name=net.get("name", ""), + purpose=net.get("purpose"), + vlan=net.get("vlan"), + ip_subnet=net.get("ip_subnet"), + dhcp_enabled=net.get("dhcpd_enabled", False), + is_guest=net.get("is_guest", False), + scraped_at=scraped_at, + )) + + # Port forwards (upsert by UniFi ID) + for pf in port_forwards: + uid = pf.get("_id", "") + if not uid: + continue + existing = await session.get(UnifiPortForward, uid) + if existing: + existing.name = pf.get("name") + existing.proto = pf.get("proto", "tcp") + existing.dst_port = pf.get("dst_port", "") + existing.fwd_ip = pf.get("fwd", "") + existing.fwd_port = pf.get("fwd_port", "") + existing.src_filter = pf.get("src") + existing.enabled = pf.get("enabled", True) + existing.scraped_at = scraped_at + else: + session.add(UnifiPortForward( + unifi_id=uid, + name=pf.get("name"), + proto=pf.get("proto", "tcp"), + dst_port=pf.get("dst_port", ""), + fwd_ip=pf.get("fwd", ""), + fwd_port=pf.get("fwd_port", ""), + src_filter=pf.get("src"), + enabled=pf.get("enabled", True), + scraped_at=scraped_at, + )) + + # Firewall rules (upsert by UniFi ID) + for rule in fw_rules: + uid = rule.get("_id", "") + if not uid: + continue + existing = await session.get(UnifiFwRule, uid) + if existing: + existing.name = rule.get("name", "") + existing.ruleset = rule.get("ruleset", "") + existing.rule_index = rule.get("rule_index") + existing.action = rule.get("action", "") + existing.protocol = rule.get("protocol") + existing.enabled = rule.get("enabled", True) + existing.src_address = rule.get("src_address") + existing.dst_address = rule.get("dst_address") + existing.scraped_at = scraped_at + else: + session.add(UnifiFwRule( + unifi_id=uid, + name=rule.get("name", ""), + ruleset=rule.get("ruleset", ""), + rule_index=rule.get("rule_index"), + action=rule.get("action", ""), + protocol=rule.get("protocol"), + enabled=rule.get("enabled", True), + src_address=rule.get("src_address"), + dst_address=rule.get("dst_address"), + scraped_at=scraped_at, + )) + + logger.debug( + "UniFi expanded: wlans=%d, events=%d, alarms=%d, known_clients=%d, " + "networks=%d, port_fwds=%d, fw_rules=%d", + len(wlan_rows), len(events), len(alarms), len(known_clients), + len(networks), len(port_forwards), len(fw_rules), + ) diff --git a/plugins/unifi/templates/unifi/dpi.html b/plugins/unifi/templates/unifi/dpi.html new file mode 100644 index 0000000..6e085b7 --- /dev/null +++ b/plugins/unifi/templates/unifi/dpi.html @@ -0,0 +1,65 @@ +{# plugins/unifi/templates/unifi/dpi.html #} +{% if not snapshot %} +
+

No DPI data yet. DPI must be enabled on the UniFi controller.

+
+{% else %} +
+ + {# Category breakdown #} + {% if categories %} +
+
+ Traffic by Category + {{ snapshot.scraped_at.strftime("%H:%M") }} +
+ {% set total_bytes = categories | sum(attribute='rx_bytes') + categories | sum(attribute='tx_bytes') %} + + {% for cat in categories[:20] %} + {% set bytes = cat.rx_bytes + cat.tx_bytes %} + {% set pct = (bytes / total_bytes * 100) if total_bytes else 0 %} + + + + + + {% endfor %} +
{{ cat.cat_name or cat.cat_id }} + {% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB + {% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB + {% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB + {% else %}{{ bytes }} B{% endif %} + +
+
+
+
+
+ {% endif %} + + {# Top clients by DPI #} + {% if top_clients %} +
+
Top Clients by Traffic
+ + {% for c in top_clients %} + + + + + {% endfor %} +
+ {{ c.hostname or c.mac }} + {{ c.mac }} + + {% set bytes = c.bytes %} + {% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB + {% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB + {% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB + {% else %}{{ bytes }} B{% endif %} +
+
+ {% endif %} + +
+{% endif %} diff --git a/plugins/unifi/templates/unifi/events.html b/plugins/unifi/templates/unifi/events.html new file mode 100644 index 0000000..79c4a21 --- /dev/null +++ b/plugins/unifi/templates/unifi/events.html @@ -0,0 +1,52 @@ +{# plugins/unifi/templates/unifi/events.html #} +{% if not events %} +
+

No events in the last {{ range_key }}.

+
+{% else %} +
+
+ Site Events + {{ events|length }} in {{ range_key }} +
+ + + + + + + + + + + + + + {% for ev in events %} + + + + + + + {% endfor %} +
TimeCategoryKeyMessage
+ {{ ev.event_time.strftime("%Y-%m-%d %H:%M:%S") }} + + + {{ ev.category }} + + + {{ ev.event_key.replace("EVT_", "") if ev.event_key else "" }} + + {{ ev.message }} + {% if ev.source_mac %} + {{ ev.source_mac }} + {% endif %} +
+
+{% endif %} diff --git a/plugins/unifi/templates/unifi/index.html b/plugins/unifi/templates/unifi/index.html new file mode 100644 index 0000000..517c030 --- /dev/null +++ b/plugins/unifi/templates/unifi/index.html @@ -0,0 +1,63 @@ +{# plugins/unifi/templates/unifi/index.html #} +{% extends "base.html" %} +{% block title %}UniFi — Steward{% endblock %} +{% block content %} +
+
+

UniFi Network

+ polling every {{ poll_interval }}s +
+ {% include "_time_range.html" %} +
+ +{# Tab nav #} +
+ {% set tabs = [ + ("overview", "Overview", "/plugins/unifi/rows"), + ("events", "Events", "/plugins/unifi/events"), + ("dpi", "Traffic", "/plugins/unifi/dpi"), + ("security", "Security", "/plugins/unifi/security"), + ("inventory", "Inventory", "/plugins/unifi/inventory"), + ] %} + {% for tab_id, label, url in tabs %} + + {% endfor %} +
+ +
+
+ + +{% endblock %} diff --git a/plugins/unifi/templates/unifi/inventory.html b/plugins/unifi/templates/unifi/inventory.html new file mode 100644 index 0000000..62e99f7 --- /dev/null +++ b/plugins/unifi/templates/unifi/inventory.html @@ -0,0 +1,146 @@ +{# plugins/unifi/templates/unifi/inventory.html #} +
+ + {# Networks / VLANs #} + {% if networks %} +
+
Networks / VLANs
+ + + + + + + {% for net in networks %} + + + + + + {% endfor %} +
NameVLANSubnet
+ {{ net.name }} + {% if net.is_guest %} guest{% endif %} + + {{ net.vlan or "—" }} + + {{ net.ip_subnet or "—" }} +
+
+ {% endif %} + + {# Port forwards #} + {% if port_forwards %} +
+
Port Forwards
+ + {% for pf in port_forwards %} + + + + + {% endfor %} +
+ + {{ pf.name or "—" }} + + {% if not pf.enabled %} disabled{% endif %} + + {{ pf.proto }}:{{ pf.dst_port }} → {{ pf.fwd_ip }}:{{ pf.fwd_port }} +
+
+ {% endif %} + + {# Firewall rules, grouped by ruleset #} + {% if fw_rules %} + {% set ns = namespace(current_rs=None) %} +
+
Firewall Rules
+ + {% for rule in fw_rules %} + {% if rule.ruleset != ns.current_rs %} + {% set ns.current_rs = rule.ruleset %} + + + + {% endif %} + + + + + + {% endfor %} +
+ {{ rule.ruleset }} +
+ + {{ rule.name }} + + + + {{ rule.action }} + + + {% if rule.protocol %}{{ rule.protocol }}{% endif %} + {% if rule.src_address %} src:{{ rule.src_address }}{% endif %} + {% if rule.dst_address %} dst:{{ rule.dst_address }}{% endif %} +
+
+ {% endif %} + + {# Known clients #} + {% if known_clients %} +
+
+ Known Clients + {{ known_clients|length }} total +
+ + + + + + + {% for kc in known_clients[:50] %} + + + + + + {% endfor %} + {% if known_clients|length > 50 %} + + + + {% endif %} +
Name / MACIPLast Seen
+
+ {{ kc.name or kc.hostname or kc.mac }} + {% if kc.is_blocked %} blocked{% endif %} +
+ {% if kc.name or kc.hostname %} +
{{ kc.mac }}
+ {% endif %} + {% if kc.mac_vendor %} +
{{ kc.mac_vendor }}
+ {% endif %} + 		+ {{ kc.ip or "—" }} + + {% if kc.last_seen %}{{ kc.last_seen.strftime("%m-%d %H:%M") }}{% else %}—{% endif %} +
+ +{{ known_clients|length - 50 }} more +
+
+ {% endif %} + + {% if not networks and not port_forwards and not fw_rules and not known_clients %} +
+

No inventory data yet. This populates on the next poll cycle.

+
+ {% endif %} + +
diff --git a/plugins/unifi/templates/unifi/rows.html b/plugins/unifi/templates/unifi/rows.html new file mode 100644 index 0000000..9de46ce --- /dev/null +++ b/plugins/unifi/templates/unifi/rows.html @@ -0,0 +1,203 @@ +{# plugins/unifi/templates/unifi/rows.html #} + +{# ── WAN health bar ───────────────────────────────────────────────────────── #} +{% if latest_wan %} +
+ + WAN + + {% if latest_wan.is_up %}✓ Up{% else %}✗ Down{% endif %} + + {% if latest_wan.wan_ip %} + {{ latest_wan.wan_ip }} + {% endif %} + + + {% if latest_wan.latency_ms is not none %} + + Latency + + {{ "%.1f"|format(latest_wan.latency_ms) }}ms + + {{ sparkline_latency | safe }} + + {% endif %} + + {% if latest_wan.rx_bytes_rate is not none %} + + ↓ RX + + {% set bps = latest_wan.rx_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s + {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %} + + {{ sparkline_rx | safe }} + + + ↑ TX + + {% set bps = latest_wan.tx_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s + {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %} + + {{ sparkline_tx | safe }} + + {% endif %} + + {% if latest_wan.uptime_seconds is not none %} + + up {{ (latest_wan.uptime_seconds // 3600) }}h {{ ((latest_wan.uptime_seconds % 3600) // 60) }}m + + {% endif %} +
+{% endif %} + +{# ── Content columns ──────────────────────────────────────────────────────── #} +
+ + {# Client summary card #} + {% if latest_clients %} +
+
+ Clients + {{ sparkline_clients | safe }} +
+
+ + {{ latest_clients.total_clients }} + total + + + {{ latest_clients.wireless_clients }}↗ wireless   + {{ latest_clients.wired_clients }}↔ wired + {% if latest_clients.guest_clients %}  {{ latest_clients.guest_clients }} guest{% endif %} + +
+
+ {% endif %} + + {# Infrastructure devices card #} + {% if devices %} +
+
Infrastructure Devices
+ + {% for d in devices %} + + + + + + {% if d.state == 1 and d.uptime_seconds %} + + + + {% endif %} + {% endfor %} +
+ ● + {{ d.name or d.mac }} + {{ d.model or d.device_type or "" }}{{ d.ip or "" }}
+ up {{ (d.uptime_seconds // 86400) }}d {{ ((d.uptime_seconds % 86400) // 3600) }}h +
+
+ {% endif %} + + {# Top talkers card #} + {% if top_clients %} +
+
Top Talkers
+ + {% for c in top_clients %} + + + + + {% endfor %} +
+ {{ c.hostname }} + {% if c.type == "wireless" and c.signal %} + {{ c.signal }}dBm + {% endif %} + + {% set rx = c.rx_rate %}{% set tx = c.tx_rate %} + ↓ + {% if rx >= 1048576 %}{{ "%.1f"|format(rx/1048576) }}M + {% elif rx >= 1024 %}{{ "%.0f"|format(rx/1024) }}K + {% else %}{{ "%.0f"|format(rx) }}B{% endif %} + ↑ + {% if tx >= 1048576 %}{{ "%.1f"|format(tx/1048576) }}M + {% elif tx >= 1024 %}{{ "%.0f"|format(tx/1024) }}K + {% else %}{{ "%.0f"|format(tx) }}B{% endif %} +
+
+ {% endif %} + + {# WLAN SSIDs card #} + {% if wlan_latest %} +
+
Wireless Networks
+ + + + + + + {% for w in wlan_latest %} + + + + + + {% endfor %} +
SSIDClientsScore
{{ w.ssid }}{{ w.num_sta }} + {% if w.satisfaction is not none %}{{ w.satisfaction }}%{% else %}—{% endif %} +
+
+ {% endif %} + + {# Speedtest card #} + {% if speedtest %} +
+
Last Speed Test
+
+ {% if speedtest.download_mbps is not none %} + + ↓ DL + {{ "%.1f"|format(speedtest.download_mbps) }} + Mbps + + {% endif %} + {% if speedtest.upload_mbps is not none %} + + ↑ UL + {{ "%.1f"|format(speedtest.upload_mbps) }} + Mbps + + {% endif %} + {% if speedtest.latency_ms is not none %} + + Latency + {{ "%.0f"|format(speedtest.latency_ms) }}ms + + {% endif %} +
+ {% if speedtest.server_name %} +
via {{ speedtest.server_name }}
+ {% endif %} +
{{ speedtest.run_at.strftime("%Y-%m-%d %H:%M") }} UTC
+
+ {% endif %} + +
+ +{% if not latest_wan and not devices %} +
+

No UniFi data yet. Check your host, username, and password in the plugin config.

+
+{% endif %} diff --git a/plugins/unifi/templates/unifi/security.html b/plugins/unifi/templates/unifi/security.html new file mode 100644 index 0000000..f28d4c0 --- /dev/null +++ b/plugins/unifi/templates/unifi/security.html @@ -0,0 +1,56 @@ +{# plugins/unifi/templates/unifi/security.html #} +
+ + {# Active alarms #} +
+
+ Active Alarms + {% if active_alarms %} + {{ active_alarms|length }} + {% endif %} +
+ {% if not active_alarms %} +

✓ No active alarms

+ {% else %} + + {% for al in active_alarms %} + + + + {% endfor %} +
+
+ ⚠ {{ al.alarm_key }} +
+
{{ al.message }}
+
+ {{ al.alarm_time.strftime("%Y-%m-%d %H:%M") }} UTC +
+
+ {% endif %} +
+ + {# Recent archived alarms #} + {% if archived_alarms %} +
+
+ Recent Archived Alarms + last {{ archived_alarms|length }} +
+ + {% for al in archived_alarms %} + + + + + {% endfor %} +
+
{{ al.alarm_key }}
+
{{ al.message }}
+ 		+ {{ al.alarm_time.strftime("%m-%d %H:%M") }} +
+
+ {% endif %} + +
diff --git a/plugins/unifi/templates/unifi/widget.html b/plugins/unifi/templates/unifi/widget.html new file mode 100644 index 0000000..2993ea6 --- /dev/null +++ b/plugins/unifi/templates/unifi/widget.html @@ -0,0 +1,95 @@ +{# plugins/unifi/templates/unifi/widget.html #} + +{# Active alarms #} +{% for al in active_alarms %} +
+ ⚠ {{ al.alarm_key }} + alarm +
+{% endfor %} +{# Offline device alerts #} +{% for d in offline_devices %} +
+ ⚠ {{ d.name or d.mac }} + offline +
+{% endfor %} +{% if active_alarms or offline_devices %}
{% endif %} + +{% if latest_wan %} +
+
+ WAN +
+
+ + {% if latest_wan.is_up %}Up{% else %}Down{% endif %} + + {% if latest_wan.latency_ms is not none %} + + lat + + {{ "%.0f"|format(latest_wan.latency_ms) }}ms + + + {% endif %} + {% if latest_wan.rx_bytes_rate is not none %} + + ↓ + {% set bps = latest_wan.rx_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s + {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %} + + + ↑ + {% set bps = latest_wan.tx_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s + {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %} + + {% endif %} +
+
+{% endif %} + +{% if latest_clients %} +
+
+ Clients +
+
+ {{ latest_clients.total_clients }} + {{ latest_clients.wireless_clients }}w / {{ latest_clients.wired_clients }}e +
+
+{% endif %} + +{% if top_clients %} +
+ {% for c in top_clients %} +
+
+ {{ c.hostname }} +
+
+ {% set total_bps = c.tx_rate + c.rx_rate %} + ↓ + {% set rx = c.rx_rate %} + {% if rx >= 1048576 %}{{ "%.1f"|format(rx / 1048576) }}M + {% elif rx >= 1024 %}{{ "%.0f"|format(rx / 1024) }}K + {% else %}{{ "%.0f"|format(rx) }}B{% endif %} + ↑ + {% set tx = c.tx_rate %} + {% if tx >= 1048576 %}{{ "%.1f"|format(tx / 1048576) }}M + {% elif tx >= 1024 %}{{ "%.0f"|format(tx / 1024) }}K + {% else %}{{ "%.0f"|format(tx) }}B{% endif %} +
+
+ {% endfor %} +
+{% endif %} + +{% if not latest_wan and not latest_clients and not active_alarms and not offline_devices %} +

No UniFi data yet.

+{% endif %} diff --git a/plugins/unifi/templates/unifi/widget_clients.html b/plugins/unifi/templates/unifi/widget_clients.html new file mode 100644 index 0000000..56059a9 --- /dev/null +++ b/plugins/unifi/templates/unifi/widget_clients.html @@ -0,0 +1,42 @@ +{# unifi/widget_clients.html — dashboard widget: top active clients #} +{% if not snapshot %} +

No client data yet.

+{% else %} + +
+
+ {{ snapshot.total_clients }} + total +
+
+ {{ snapshot.wireless_clients }} + wireless +
+
+ {{ snapshot.wired_clients }} + wired +
+
+ +{% if clients %} +
+ {% for c in clients %} +
+ + + {{ c.hostname or c.ip or c.mac }} + + {% if c.tx_rate or c.rx_rate %} + + {% if c.tx_rate %}↑{{ "%.0f" | format(c.tx_rate / 1000) }}k{% endif %} + {% if c.rx_rate %} ↓{{ "%.0f" | format(c.rx_rate / 1000) }}k{% endif %} + + {% endif %} +
+ {% endfor %} +
+{% else %} +
No clients connected.
+{% endif %} + +{% endif %} diff --git a/plugins/unifi/templates/unifi/widget_devices.html b/plugins/unifi/templates/unifi/widget_devices.html new file mode 100644 index 0000000..90994ed --- /dev/null +++ b/plugins/unifi/templates/unifi/widget_devices.html @@ -0,0 +1,36 @@ +{# unifi/widget_devices.html — dashboard widget: infrastructure devices #} +{% if not devices %} +
+ {% if type_filter != "all" %}No {{ type_filter }} devices found.{% else %}No devices found.{% endif %} +
+{% else %} + +{% set online = devices | selectattr("state", "equalto", 1) | list %} +{% set offline = devices | rejectattr("state", "equalto", 1) | list %} + +
+
+ {{ online | length }} + online +
+ {% if offline %} +
+ {{ offline | length }} + offline +
+ {% endif %} +
+ +
+ {% for d in devices %} +
+ + + {{ d.name or d.mac }} + + {{ d.model or d.device_type or "" }} +
+ {% endfor %} +
+ +{% endif %} diff --git a/pyproject.toml b/pyproject.toml index 465001e..c10b66a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "packaging>=24.0", "click>=8.0", "httpx>=0.27", + "cryptography>=42", ] [project.optional-dependencies] @@ -26,6 +27,11 @@ ldap = [ snmp = [ "pysnmp-lextudio>=6.2", ] +# Batteries-included Ansible (bundles the common community collections) for the +# playbook runner. Installed into the image via Dockerfile `pip install .[ansible]`. +ansible = [ + "ansible>=10,<13", +] dev = [ "pytest>=8.0", "pytest-asyncio>=0.23", @@ -37,6 +43,9 @@ steward = "steward.cli:main" [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = ["tests"] +markers = [ + "integration: requires a live Postgres; runs only in the integration CI lane", +] [tool.hatch.build.targets.wheel] packages = ["steward"] diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py index 9c71119..621c58a 100644 --- a/steward/alerts/routes.py +++ b/steward/alerts/routes.py @@ -4,8 +4,8 @@ from quart import Blueprint, render_template, request, redirect, url_for, curren from steward.core.audit import log_audit from sqlalchemy import select from steward.auth.middleware import require_role +from steward.ansible import sources as src_module from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertOperator, MaintenanceWindow -from steward.models.hosts import Host from steward.models.metrics import PluginMetric from steward.models.users import UserRole @@ -13,15 +13,20 @@ alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts") _OPERATORS = [(op.value, op.value) for op in AlertOperator] -# Static catalog: source_module → available metric names +# Static catalog: source_module → available metric names. +# Unified monitors emit metrics under their TYPE (icmp/tcp/dns/http), each with +# is_up (1.0 up / 0.0 down) and response_ms (where the probe times it). METRIC_CATALOG: dict[str, list[str]] = { - "ping": ["up", "response_time_ms"], - "dns": ["resolved", "ip_changed"], + "icmp": ["is_up", "response_ms"], + "tcp": ["is_up", "response_ms"], + "dns": ["is_up", "response_ms"], + "http": ["is_up", "response_ms"], "traefik": ["request_rate", "error_rate", "latency_p50_ms", "latency_p95_ms", "latency_p99_ms", "response_bytes_rate", "cert_expiry_days"], "unifi": ["is_up", "latency_ms", "total_clients"], - "docker": ["cpu_pct", "mem_pct"], - "http": ["is_up", "response_ms"], + # restart_count = cumulative restarts (alert on crash-looping); is_healthy = + # 1.0 healthy / 0.0 unhealthy from the container HEALTHCHECK (alert on <1). + "docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy"], "host_agent": [ "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes", "disk_used_pct_worst", "load_1m", "load_5m", "load_15m", "uptime_secs", @@ -37,8 +42,8 @@ _SOURCE_MODULES = list(METRIC_CATALOG.keys()) async def _get_resources(db, source_module: str) -> list[str]: """Return sorted distinct resource names for a source module. - For ping/dns, supplements with host names so the list is populated - even before any metrics have been recorded. + For monitor types (icmp/tcp/dns/http), supplements with monitor names so + the list is populated even before any metrics have been recorded. """ result = await db.execute( select(PluginMetric.resource_name) @@ -48,14 +53,72 @@ async def _get_resources(db, source_module: str) -> list[str]: ) resources: set[str] = {r for (r,) in result} - if source_module in ("ping", "dns"): - host_result = await db.execute(select(Host.name).order_by(Host.name)) - for (name,) in host_result: + if source_module in ("icmp", "tcp", "dns", "http"): + from steward.models.monitors import Monitor + mon_result = await db.execute( + select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name) + ) + for (name,) in mon_result: resources.add(name) return sorted(resources) +def _is_admin() -> bool: + return session.get("user_role") == UserRole.admin.value + + +def _ansible_source_names() -> list[str]: + try: + return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))] + except Exception: + return [] + + +def _parse_ansible_action(form) -> tuple[dict | None, str | None]: + """Parse + validate the admin-only Ansible action from a rule form. + + Returns (action_or_None, error_or_None). action is None when the action + checkbox is unchecked (i.e. the action is being cleared). + """ + if "ansible_enabled" not in form: + return None, None + source_name = (form.get("ansible_source", "") or "").strip() + playbook = (form.get("ansible_playbook", "") or "").strip() + inventory = (form.get("ansible_inventory", "") or "").strip() + if not (source_name and playbook and inventory): + return None, "Ansible action requires source, playbook, and inventory" + try: + srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {})) + except Exception: + return None, "Ansible sources are misconfigured" + source = next((s for s in srcs if s["name"] == source_name), None) + if source is None: + return None, f"Ansible source '{source_name}' not found" + if playbook not in src_module.discover_playbooks(source["path"]): + return None, f"Playbook '{playbook}' not found in source '{source_name}'" + extra_vars: list[str] = [] + for line in (form.get("ansible_extra_vars", "") or "").splitlines(): + line = line.strip() + if not line: + continue + if "=" not in line: + return None, f"Invalid extra var (expected key=value): {line!r}" + extra_vars.append(line) + action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory} + if extra_vars: + action["extra_vars"] = extra_vars + limit = (form.get("ansible_limit", "") or "").strip() + tags = (form.get("ansible_tags", "") or "").strip() + if limit: + action["limit"] = limit + if tags: + action["tags"] = tags + if "ansible_check" in form: + action["check"] = True + return action, None + + @alerts_bp.get("/") @require_role(UserRole.viewer) async def list_alerts(): @@ -120,6 +183,7 @@ async def new_rule(): initial_source=first_source, resources=resources, metrics=METRIC_CATALOG.get(first_source, []), + ansible_sources=_ansible_source_names(), ) @@ -140,6 +204,7 @@ async def edit_rule(rule_id: str): initial_source=rule.source_module, resources=resources, metrics=METRIC_CATALOG.get(rule.source_module, []), + ansible_sources=_ansible_source_names(), ) @@ -149,6 +214,12 @@ async def create_rule(): form = await request.form user_id = session.get("user_id") now = datetime.now(timezone.utc) + # Ansible action is admin-only; non-admins simply can't set one. + ansible_action = None + if _is_admin(): + ansible_action, err = _parse_ansible_action(form) + if err: + return err, 400 rule = AlertRule( name=form["name"].strip(), source_module=form["source_module"].strip(), @@ -158,6 +229,7 @@ async def create_rule(): threshold=float(form["threshold"]), consecutive_failures_required=int(form.get("consecutive_failures_required") or 1), enabled=True, + ansible_action=ansible_action, created_by=user_id, ) state = AlertState( @@ -182,6 +254,13 @@ async def create_rule(): @require_role(UserRole.operator) async def update_rule(rule_id: str): form = await request.form + # Validate the admin-only action before opening the transaction. Non-admins + # leave the existing action untouched (they can still edit everything else). + apply_action = _is_admin() + if apply_action: + ansible_action, err = _parse_ansible_action(form) + if err: + return err, 400 async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id)) @@ -195,6 +274,8 @@ async def update_rule(rule_id: str): rule.operator = AlertOperator(form["operator"]) rule.threshold = float(form["threshold"]) rule.consecutive_failures_required = int(form.get("consecutive_failures_required") or 1) + if apply_action: + rule.ansible_action = ansible_action await log_audit(current_app, session.get("user_id"), session.get("username", ""), "alert_rule.updated", entity_type="alert_rule", entity_id=rule.name) return redirect(url_for("alerts.list_alerts")) diff --git a/steward/ansible/PLAYBOOK_CONVENTIONS.md b/steward/ansible/PLAYBOOK_CONVENTIONS.md new file mode 100644 index 0000000..a44c3e3 --- /dev/null +++ b/steward/ansible/PLAYBOOK_CONVENTIONS.md @@ -0,0 +1,39 @@ +# Steward playbook conventions (quick reference) + +What Steward reads from a playbook in this subsystem. Full guide: +`docs/reference/playbook-authoring.md`. Parser: +`steward/ansible/sources.py` — `discover_playbook_meta()` and +`discover_playbook_variables()`. + +## Magic comments (metadata) + +```yaml +--- +# description: One line on what this playbook does. # shown on selection +# steward:category: maintenance # grouping badge +# steward:confirm: true # require confirm to run +- name: … # used as the description if no `# description:` comment + hosts: all # Steward generates the inventory from the chosen target/group +``` + +- `# description:` — first match wins; case-insensitive; falls back to first + play `name:`. +- `# steward:: ` — namespaced metadata. Implemented keys: + - `category` — free-text grouping label (badge in browse + run form). + - `confirm` — `true`/`yes`/`1`/`on` → run form requires a confirmation tick. + - `description` — alias for `# description:` (unprefixed wins if both present). + - Unknown `# steward:*` keys are ignored (namespace reserved for future use). + +## Structural cues (no comments needed) + +- **`vars:` scalars** → editable run-time fields (default shown as placeholder; + blank = keep default; values sent as highest-precedence extra-vars). +- **`vars_prompt:`** → run-time fields (required when no default; `private: true` + → masked). +- **Secret naming** — a var whose name contains `password`/`passwd`/`secret`/ + `token`/`api_key`/`private_key`/`credential` is masked **and not persisted**. +- **`hosts: all`** — target via the run form's scope; don't hardcode hosts. +- **Credentials** — never embed them. Steward connects as the managed `steward` + account (passwordless sudo; use `become: true`) or one-time bootstrap creds. + +Be idempotent — Steward re-runs playbooks (updates, schedules, retries). diff --git a/steward/ansible/bundled/host_agent/install.yml b/steward/ansible/bundled/host_agent/install.yml new file mode 100644 index 0000000..ce50939 --- /dev/null +++ b/steward/ansible/bundled/host_agent/install.yml @@ -0,0 +1,113 @@ +--- +# description: Install or re-enroll the Steward host agent on an already-provisioned host. +# steward:category: host-agent +# Install (or update) the Steward host agent on a target. Mirrors the +# curl-based install.sh. Pass the per-host registration token as extra-vars: +# steward_url=https://steward.example steward_token= +# Re-running updates agent.py in place and restarts the service. +- name: Install Steward host agent + hosts: all + become: true + gather_facts: false + # steward_token is a per-host inventory var — not declared here, or a play var + # would outrank and shadow it. steward_url arrives as an extra-var. + vars: + steward_url: "" + agent_interval: 30 + tasks: + - name: Require steward_url and steward_token + ansible.builtin.assert: + that: + - steward_url | default('') | length > 0 + - steward_token | default('') | length > 0 + fail_msg: "Pass steward_url and steward_token (Steward injects these)." + + - name: Create steward-agent system user + ansible.builtin.user: + name: steward-agent + system: true + shell: /usr/sbin/nologin + create_home: false + + # Let the agent read the local Docker socket so it can report this host's + # containers (the per-host docker collection path). Best-effort: skipped on + # hosts without Docker. Restart picks up the new supplementary group. + - name: Detect docker group + ansible.builtin.command: getent group docker + register: _docker_grp + changed_when: false + failed_when: false + + - name: Add steward-agent to the docker group + ansible.builtin.user: + name: steward-agent + groups: docker + append: true + when: _docker_grp.rc == 0 + notify: restart steward-agent + + - name: Create agent directory + ansible.builtin.file: + path: /usr/local/lib/steward-agent + state: directory + mode: "0755" + + - name: Download agent.py + ansible.builtin.get_url: + url: "{{ steward_url }}/plugins/host_agent/agent.py" + dest: /usr/local/lib/steward-agent/agent.py + mode: "0755" + force: true + notify: restart steward-agent + + - name: Write agent config + ansible.builtin.copy: + dest: /etc/steward-agent.conf + owner: root + group: steward-agent + mode: "0640" + content: | + url = {{ steward_url }} + token = {{ steward_token }} + interval_seconds = {{ agent_interval }} + notify: restart steward-agent + + - name: Install systemd unit + ansible.builtin.copy: + dest: /etc/systemd/system/steward-agent.service + mode: "0644" + content: | + [Unit] + Description=Steward host agent + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + User=steward-agent + Environment=STEWARD_AGENT_CONFIG=/etc/steward-agent.conf + ExecStart=/usr/bin/env python3 /usr/local/lib/steward-agent/agent.py + Restart=on-failure + RestartSec=5 + NoNewPrivileges=true + ProtectSystem=strict + ProtectHome=true + PrivateTmp=true + + [Install] + WantedBy=multi-user.target + notify: restart steward-agent + + - name: Enable and start the agent + ansible.builtin.systemd: + name: steward-agent + enabled: true + state: started + daemon_reload: true + + handlers: + - name: restart steward-agent + ansible.builtin.systemd: + name: steward-agent + state: restarted + daemon_reload: true diff --git a/steward/ansible/bundled/host_agent/provision.yml b/steward/ansible/bundled/host_agent/provision.yml new file mode 100644 index 0000000..21b62ac --- /dev/null +++ b/steward/ansible/bundled/host_agent/provision.yml @@ -0,0 +1,169 @@ +--- +# description: Provision a fresh host — create the steward account + managed key + passwordless sudo, then install the agent. +# steward:category: host-agent +# First-contact provisioning: bring a fresh host under Steward management, then +# install the host agent — all in one idempotent run. +# +# Run this ONCE per host with bootstrap credentials (an existing login + sudo, +# typically password auth via the "Provision host" button). It: +# 1. creates a dedicated `steward` login account, +# 2. installs Steward's managed public key into its authorized_keys, +# 3. grants it passwordless sudo, +# 4. installs/updates the host agent. +# After this, every future run (agent updates, maintenance) connects as +# `steward` with the managed key — no operator credentials needed. +# +# Extra-vars (injected by the Provision card): +# steward_url, steward_token — agent → Steward auth (per-host) +# steward_pubkey — managed public key to authorize +# steward_user (default steward), agent_interval (default 30) +- name: Provision host for Steward + install agent + hosts: all + become: true + gather_facts: false + # steward_token is injected as a per-host inventory var (it differs per host), + # so it must NOT be declared here — a play var would outrank the inventory var + # and shadow it. The globals below arrive as extra-vars (highest precedence), + # which override these defaults. + vars: + steward_url: "" + steward_pubkey: "" + steward_user: steward + agent_interval: 30 + tasks: + - name: Require provisioning vars + ansible.builtin.assert: + that: + - steward_url | default('') | length > 0 + - steward_token | default('') | length > 0 + - steward_pubkey | default('') | length > 0 + fail_msg: "Pass steward_url, steward_token and steward_pubkey (Steward injects these)." + + # ── Managed login account ──────────────────────────────────────────────── + - name: Create the steward management account + ansible.builtin.user: + name: "{{ steward_user }}" + shell: /bin/sh + create_home: true + state: present + + - name: Ensure ~/.ssh exists + ansible.builtin.file: + path: "/home/{{ steward_user }}/.ssh" + state: directory + owner: "{{ steward_user }}" + group: "{{ steward_user }}" + mode: "0700" + + - name: Authorize the managed public key (replace any prior steward-managed key) + ansible.builtin.lineinfile: + path: "/home/{{ steward_user }}/.ssh/authorized_keys" + # Match on the ' steward-managed' comment so a rotated key REPLACES the + # old one in place rather than stacking a second authorized key. Any + # keys the operator added by hand (different comment) are left untouched. + regexp: ' steward-managed$' + line: "{{ steward_pubkey }}" + create: true + owner: "{{ steward_user }}" + group: "{{ steward_user }}" + mode: "0600" + + - name: Grant passwordless sudo to the steward account + ansible.builtin.copy: + dest: "/etc/sudoers.d/steward" + content: "{{ steward_user }} ALL=(ALL) NOPASSWD:ALL\n" + owner: root + group: root + mode: "0440" + validate: "visudo -cf %s" + + # ── Host agent (same as install.yml) ───────────────────────────────────── + - name: Create steward-agent system user + ansible.builtin.user: + name: steward-agent + system: true + shell: /usr/sbin/nologin + create_home: false + + # Let the agent read the local Docker socket so it can report this host's + # containers (the per-host docker collection path). Best-effort: skipped on + # hosts without Docker. Restart picks up the new supplementary group. + - name: Detect docker group + ansible.builtin.command: getent group docker + register: _docker_grp + changed_when: false + failed_when: false + + - name: Add steward-agent to the docker group + ansible.builtin.user: + name: steward-agent + groups: docker + append: true + when: _docker_grp.rc == 0 + notify: restart steward-agent + + - name: Create agent directory + ansible.builtin.file: + path: /usr/local/lib/steward-agent + state: directory + mode: "0755" + + - name: Download agent.py + ansible.builtin.get_url: + url: "{{ steward_url }}/plugins/host_agent/agent.py" + dest: /usr/local/lib/steward-agent/agent.py + mode: "0755" + force: true + notify: restart steward-agent + + - name: Write agent config + ansible.builtin.copy: + dest: /etc/steward-agent.conf + owner: root + group: steward-agent + mode: "0640" + content: | + url = {{ steward_url }} + token = {{ steward_token }} + interval_seconds = {{ agent_interval }} + notify: restart steward-agent + + - name: Install systemd unit + ansible.builtin.copy: + dest: /etc/systemd/system/steward-agent.service + mode: "0644" + content: | + [Unit] + Description=Steward host agent + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + User=steward-agent + Environment=STEWARD_AGENT_CONFIG=/etc/steward-agent.conf + ExecStart=/usr/bin/env python3 /usr/local/lib/steward-agent/agent.py + Restart=on-failure + RestartSec=5 + NoNewPrivileges=true + ProtectSystem=strict + ProtectHome=true + PrivateTmp=true + + [Install] + WantedBy=multi-user.target + notify: restart steward-agent + + - name: Enable and start the agent + ansible.builtin.systemd: + name: steward-agent + enabled: true + state: started + daemon_reload: true + + handlers: + - name: restart steward-agent + ansible.builtin.systemd: + name: steward-agent + state: restarted + daemon_reload: true diff --git a/steward/ansible/bundled/host_agent/update.yml b/steward/ansible/bundled/host_agent/update.yml new file mode 100644 index 0000000..82f60f9 --- /dev/null +++ b/steward/ansible/bundled/host_agent/update.yml @@ -0,0 +1,49 @@ +--- +# description: Update the Steward host agent (refresh agent.py + restart) on hosts already running it. +# steward:category: host-agent +# Update an already-installed Steward host agent: refresh agent.py and restart. +# +# Deliberately does NOT touch the registration token or /etc/steward-agent.conf +# — the host already has a working token, so an update leaves identity alone and +# only swaps the agent binary. Connects as the managed `steward` account (set +# globally as the SSH user); no bootstrap password or token injection needed. +# +# For a fresh host with no agent, use provision.yml (new host) or install.yml +# (already-provisioned host) instead — those mint and install the token. +- name: Update Steward host agent + hosts: all + become: true + gather_facts: false + vars: + steward_url: "" + tasks: + - name: Require steward_url + ansible.builtin.assert: + that: + - steward_url | length > 0 + fail_msg: "Pass steward_url as an extra-var (the Update action sets it)." + + - name: Confirm the agent is already installed + ansible.builtin.stat: + path: /usr/local/lib/steward-agent/agent.py + register: agent_file + + - name: Fail clearly if the agent is missing + ansible.builtin.fail: + msg: "No agent at /usr/local/lib/steward-agent — provision or install this host first." + when: not agent_file.stat.exists + + - name: Refresh agent.py + ansible.builtin.get_url: + url: "{{ steward_url }}/plugins/host_agent/agent.py" + dest: /usr/local/lib/steward-agent/agent.py + mode: "0755" + force: true + notify: restart steward-agent + + handlers: + - name: restart steward-agent + ansible.builtin.systemd: + name: steward-agent + state: restarted + daemon_reload: true diff --git a/steward/ansible/bundled/maintenance/docker_prune.yml b/steward/ansible/bundled/maintenance/docker_prune.yml new file mode 100644 index 0000000..51c5bcf --- /dev/null +++ b/steward/ansible/bundled/maintenance/docker_prune.yml @@ -0,0 +1,29 @@ +--- +# description: Reclaim disk on Docker / Swarm nodes by pruning unused images, containers, networks and build cache. +# steward:category: maintenance +# steward:confirm: true +# Reclaim disk on Docker / Docker Swarm nodes by removing unused data. +# Safe by default: prunes dangling images, stopped containers, unused networks +# and build cache. Set extra-vars to widen scope: +# prune_all_images=true also remove ALL unused images (not just dangling) +# prune_volumes=true also remove unused named volumes (data loss risk) +- name: Docker system prune + hosts: all + gather_facts: false + become: true + vars: + prune_all_images: false + prune_volumes: false + tasks: + - name: Run docker system prune + ansible.builtin.command: + argv: >- + {{ ['docker', 'system', 'prune', '-f'] + + (['-a'] if prune_all_images | bool else []) + + (['--volumes'] if prune_volumes | bool else []) }} + register: prune_result + changed_when: "'Total reclaimed space: 0B' not in prune_result.stdout" + + - name: Report reclaimed space + ansible.builtin.debug: + msg: "{{ prune_result.stdout_lines | select | list }}" diff --git a/steward/ansible/bundled/maintenance/system_update.yml b/steward/ansible/bundled/maintenance/system_update.yml new file mode 100644 index 0000000..f94bebe --- /dev/null +++ b/steward/ansible/bundled/maintenance/system_update.yml @@ -0,0 +1,82 @@ +--- +# description: Update all OS packages; optionally restart affected services and reboot if one is pending. +# steward:category: maintenance +# steward:confirm: true +# +# Cross-distro (apt / dnf-yum). Flags (set in the run form): +# restart_services=true after the upgrade, restart every service that needs +# it (Debian: needrestart -r a; installed if missing). +# reboot_if_required=true reboot the host only if the update flagged a pending +# reboot (Debian: /var/run/reboot-required; +# RHEL: needs-restarting -r). Never reboots otherwise. +- name: System package update + hosts: all + become: true + gather_facts: true + vars: + restart_services: false + reboot_if_required: false + tasks: + # ── Debian / Ubuntu ────────────────────────────────────────────────────── + - name: Upgrade all packages (apt) + ansible.builtin.apt: + update_cache: true + upgrade: dist + autoremove: true + when: ansible_pkg_mgr == "apt" + + - name: Ensure needrestart is present (apt, for service restarts) + ansible.builtin.apt: + name: needrestart + state: present + when: + - restart_services | bool + - ansible_pkg_mgr == "apt" + + - name: Restart services that need it (apt / needrestart) + ansible.builtin.command: needrestart -r a + register: needrestart_run + changed_when: needrestart_run.rc == 0 + failed_when: false + when: + - restart_services | bool + - ansible_pkg_mgr == "apt" + + # ── RHEL / Fedora ───────────────────────────────────────────────────────── + - name: Upgrade all packages (dnf/yum) + ansible.builtin.dnf: + name: "*" + state: latest + when: ansible_pkg_mgr in ("dnf", "yum") + + - name: Restart services that need it (dnf / needs-restarting) + ansible.builtin.shell: needs-restarting -s | xargs -r -n1 systemctl try-restart + register: rhel_restart + changed_when: rhel_restart.rc == 0 + failed_when: false + when: + - restart_services | bool + - ansible_pkg_mgr in ("dnf", "yum") + + # ── Reboot only if one is pending ───────────────────────────────────────── + - name: Check pending reboot (Debian) + ansible.builtin.stat: + path: /var/run/reboot-required + register: deb_reboot + when: ansible_pkg_mgr == "apt" + + - name: Check pending reboot (RHEL) + ansible.builtin.command: needs-restarting -r + register: rhel_reboot + changed_when: false + failed_when: false + when: ansible_pkg_mgr in ("dnf", "yum") + + - name: Reboot if required + ansible.builtin.reboot: + msg: "Reboot triggered by Steward system_update" + reboot_timeout: 900 + when: + - reboot_if_required | bool + - (ansible_pkg_mgr == "apt" and deb_reboot.stat.exists) + or (ansible_pkg_mgr in ("dnf", "yum") and (rhel_reboot.rc | default(0)) != 0) diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py index 22e62c9..ac8667e 100644 --- a/steward/ansible/executor.py +++ b/steward/ansible/executor.py @@ -1,21 +1,38 @@ # steward/ansible/executor.py from __future__ import annotations import asyncio +import json import logging +import os +import re +import shutil +import tempfile import time from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING +from steward.core.crypto import is_encrypted as _crypto_is_encrypted + if TYPE_CHECKING: from quart import Quart logger = logging.getLogger(__name__) -_OUTPUT_CAP_BYTES = 1024 * 1024 # 1 MB DB cap +_OUTPUT_CAP_BYTES = 1024 * 1024 # 1 MB DB cap (full output also goes to a log artifact) _FLUSH_LINES = 50 _FLUSH_SECS = 5.0 +_MEM_LINES_CAP = 5000 # bounded per-run in-memory replay buffer (SSE late-joiners) +_MEM_TTL_SECS = 300 # drop in-memory state this long after a run completes +_FAILURE_CAP = 50 # max failed-task lines captured into structured results + +# Full run logs persist here (survive restart + the 1 MB DB cap). Override via env. +ARTIFACT_DIR = os.environ.get("STEWARD_ANSIBLE_LOG_DIR", "/data/ansible/runs") + + +def artifact_path(run_id: str) -> str: + return os.path.join(ARTIFACT_DIR, f"{run_id}.log") @dataclass @@ -23,11 +40,19 @@ class _Done: status: str -# Per-run broadcast state (lives until process restarts) -_run_lines: dict[str, list[str]] = {} # all output lines received so far +class _Aborted(Exception): + """Raised internally when a queued run is cancelled before it launches.""" + + +# Per-run broadcast state (lives until shortly after completion, then GC'd) +_run_lines: dict[str, list[str]] = {} # bounded tail of output lines _run_done: dict[str, _Done] = {} # set when run completes _run_listeners: dict[str, list[asyncio.Queue]] = {} # active SSE client queues +# Running subprocesses by run_id (for cancellation) + run_ids an operator cancelled. +_run_procs: dict[str, asyncio.subprocess.Process] = {} +_cancelled: set[str] = set() + def register_listener(run_id: str) -> asyncio.Queue: """Create a listener queue for a run. Replays existing lines immediately.""" @@ -48,42 +73,270 @@ def deregister_listener(run_id: str, q: asyncio.Queue) -> None: listeners.remove(q) +def _schedule_cleanup(run_id: str) -> None: + """Drop a finished run's in-memory state after a grace window.""" + async def _cleanup() -> None: + await asyncio.sleep(_MEM_TTL_SECS) + _run_lines.pop(run_id, None) + _run_done.pop(run_id, None) + try: + asyncio.create_task(_cleanup()) + except RuntimeError: + pass # no running loop (e.g. tests) — fine to skip + + def _broadcast(run_id: str, item: str | _Done) -> None: if isinstance(item, str): - _run_lines.setdefault(run_id, []).append(item) + buf = _run_lines.setdefault(run_id, []) + buf.append(item) + if len(buf) > _MEM_LINES_CAP: # keep only the most recent lines in memory + del buf[: len(buf) - _MEM_LINES_CAP] else: _run_done[run_id] = item for q in _run_listeners.get(run_id, []): q.put_nowait(item) if isinstance(item, _Done): _run_listeners.pop(run_id, None) + _schedule_cleanup(run_id) +def build_ansible_command( + playbook_path: str, + inventory_path: str, + params: dict | None = None, +) -> list[str]: + """Build the ansible-playbook argv from a run's optional params. + + Pure (no IO). Everything is passed as argv — never shell-interpolated — so + extra-var values and limits can contain arbitrary characters safely. + params keys: extra_vars (list of "key=value"), limit, tags, check (bool). + """ + cmd = ["ansible-playbook", playbook_path, "-i", inventory_path] + params = params or {} + for ev in params.get("extra_vars") or []: + cmd += ["-e", ev] + if params.get("limit"): + cmd += ["--limit", params["limit"]] + if params.get("tags"): + cmd += ["--tags", params["tags"]] + if params.get("check"): + cmd += ["--check", "--diff"] + return cmd + + +def build_credentials(creds: dict | None, tmpdir: str) -> tuple[list[str], list[tuple[str, str]]]: + """Given global Ansible creds + a temp dir, return (extra argv, files to write). + + Pure: decides flags + file contents; the caller writes the files (mode 0600). + Secrets are always passed via files — never on argv / the process list. + creds keys: ssh_private_key, vault_password, become_password. + """ + args: list[str] = [] + files: list[tuple[str, str]] = [] + creds = creds or {} + + from steward.core.crypto import is_encrypted + key = creds.get("ssh_private_key") + # A still-encrypted value means decryption failed (app secret key changed) — + # writing the ciphertext as a key file just yields a cryptic libcrypto error, + # so skip it; start_run surfaces a clear message instead. + if key and not is_encrypted(key): + path = os.path.join(tmpdir, "id_key") + files.append((path, key if key.endswith("\n") else key + "\n")) + args += ["--private-key", path] + + vault = creds.get("vault_password") + if vault: + path = os.path.join(tmpdir, "vault_pass") + files.append((path, vault + "\n")) + args += ["--vault-password-file", path] + + become = creds.get("become_password") + if become: + path = os.path.join(tmpdir, "become.yml") + # A JSON string is valid YAML; keeps the password out of argv. + files.append((path, "ansible_become_password: " + json.dumps(become) + "\n")) + args += ["-e", "@" + path] + + return args, files + + +def build_bootstrap(connection: dict | None, tmpdir: str) -> tuple[list[str], list[tuple[str, str]]]: + """Per-run connection override for first-contact provisioning. + + Pure: returns (extra argv, files to write). The bootstrap user/password are + written to a temp vars file (``-e @file``) — never argv, never the DB — so + they don't leak via the process list or persist on the AnsibleRun row. + A bare connection password doubles as the become password unless one is + given explicitly. connection keys: user, password, become_password. + """ + args: list[str] = [] + files: list[tuple[str, str]] = [] + connection = connection or {} + user = (connection.get("user") or "").strip() + password = connection.get("password") or "" + if not user and not password: + return args, files + + lines: dict[str, str] = {} + if user: + lines["ansible_user"] = user + if password: + lines["ansible_password"] = password + lines["ansible_become_password"] = connection.get("become_password") or password + # JSON values are valid YAML and keep arbitrary characters safe. + content = "".join(f"{k}: {json.dumps(v)}\n" for k, v in lines.items()) + path = os.path.join(tmpdir, "bootstrap.yml") + files.append((path, content)) + args += ["-e", "@" + path] + return args, files + + +def build_extra_vars_file(merged: dict | None, tmpdir: str) -> tuple[list[str], list[tuple[str, str]]]: + """Run-time variables → a JSON extra-vars file (``-e @file``). + + Pure: returns (argv, files). JSON is valid YAML to Ansible and, unlike + ``-e key=value`` strings (which Ansible shlex-splits on whitespace), keeps + values with spaces/quotes intact. Used for both operator-entered playbook + variables and the unpersisted secret subset (merged by the caller). + """ + if not merged: + return [], [] + path = os.path.join(tmpdir, "extravars.json") + return ["-e", "@" + path], [(path, json.dumps(merged))] + + +def ansible_env(creds: dict | None, base_env) -> dict: + """Return a copy of base_env with ANSIBLE_HOST_KEY_CHECKING set from creds.""" + env = dict(base_env) + env["ANSIBLE_HOST_KEY_CHECKING"] = "True" if (creds or {}).get("host_key_checking") else "False" + return env + + +# ── Structured results (parsed from the streamed output) ────────────────────── + +_RECAP_RE = re.compile( + r"^(?P\S+)\s*:\s*ok=(?P\d+)\s+changed=(?P\d+)\s+" + r"unreachable=(?P\d+)\s+failed=(?P\d+)" + r"(?:\s+skipped=(?P\d+))?" +) + + +def parse_recap(lines: list[str]) -> dict[str, dict[str, int]]: + """Parse Ansible's PLAY RECAP into {host: {ok,changed,unreachable,failed,skipped}}. + + Streaming-friendly: we keep the default stdout callback (live output) and + derive the per-host summary from the recap rather than swapping to the json + callback (which would buffer everything to the end and kill live streaming). + """ + hosts: dict[str, dict[str, int]] = {} + in_recap = False + for line in lines: + if "PLAY RECAP" in line: + in_recap = True + continue + if not in_recap: + continue + m = _RECAP_RE.match(line.strip()) + if m: + hosts[m.group("host")] = { + "ok": int(m.group("ok")), + "changed": int(m.group("changed")), + "unreachable": int(m.group("unreachable")), + "failed": int(m.group("failed")), + "skipped": int(m.group("skipped") or 0), + } + return hosts + + +# ── Cancellation ────────────────────────────────────────────────────────────── + +async def _set_status(app: "Quart", run_id: str, status) -> None: + from sqlalchemy import update + from steward.models.ansible import AnsibleRun + async with app.db_sessionmaker() as session: + async with session.begin(): + await session.execute( + update(AnsibleRun).where(AnsibleRun.id == run_id).values(status=status) + ) + + +async def _terminate_then_kill(proc: asyncio.subprocess.Process, grace: float = 10.0) -> None: + try: + proc.terminate() + except ProcessLookupError: + return + try: + await asyncio.wait_for(proc.wait(), timeout=grace) + except asyncio.TimeoutError: + try: + proc.kill() + except ProcessLookupError: + pass + + +def cancel_run(run_id: str) -> bool: + """Request cancellation of a run. Terminates the process if it's running; + a queued run is flagged so start_run aborts before launch. Returns False + only when the run isn't tracked in this process (already finished).""" + if run_id not in _run_procs and run_id not in _cancelled: + # Not obviously in-flight. Still flag it (covers a just-queued run whose + # proc hasn't been registered yet); caller should gate on run status. + _cancelled.add(run_id) + return run_id in _run_procs or True + _cancelled.add(run_id) + proc = _run_procs.get(run_id) + if proc is not None: + asyncio.create_task(_terminate_then_kill(proc)) + return True + + +# ── Run execution ───────────────────────────────────────────────────────────── + async def start_run( app: "Quart", run_id: str, playbook_path: str, inventory_path: str, source_path: str, + params: dict | None = None, + inventory_content: str | None = None, + connection: dict | None = None, + secret_vars: dict | None = None, ) -> None: - """Execute ansible-playbook as a subprocess and update the DB run row.""" - _run_lines[run_id] = [] + """Execute ansible-playbook as a subprocess and update the DB run row. + Respects a global concurrency cap (app._ansible_semaphore): if no slot is + free the run shows as 'queued' until one frees. Supports cancellation, a + persistent full-log artifact, and a parsed per-host result summary. + + connection is an optional per-run SSH override (user/password) for + first-contact provisioning. It is applied to the subprocess only — it is + never persisted on the AnsibleRun row, never placed on argv, and not logged. + secret_vars are run-time playbook variables flagged sensitive; like + connection they reach the subprocess (merged into the extra-vars file) but + are never persisted. + """ + from steward.models.ansible import AnsibleRunStatus + + _run_lines[run_id] = [] db_output = "" truncated = False lines_since_flush = 0 last_flush_at = time.monotonic() + failures: list[str] = [] - async def _flush(final_status: str | None = None) -> None: + async def _flush(final_status: str | None = None, results: dict | None = None) -> None: nonlocal lines_since_flush, last_flush_at from sqlalchemy import update - from steward.models.ansible import AnsibleRun, AnsibleRunStatus + from steward.models.ansible import AnsibleRun values: dict = {"output": db_output} + if results is not None: + values["results"] = results if final_status is not None: values["status"] = AnsibleRunStatus(final_status) values["finished_at"] = datetime.now(timezone.utc) - async with app.db_sessionmaker() as session: async with session.begin(): await session.execute( @@ -92,41 +345,151 @@ async def start_run( lines_since_flush = 0 last_flush_at = time.monotonic() - cmd = ["ansible-playbook", playbook_path, "-i", inventory_path] - cwd = source_path if Path(source_path).exists() else None + # ── Concurrency gate (semaphore lazily bound to the running loop so the + # cap is honoured in prod's single loop without breaking multi-loop tests). + loop = asyncio.get_running_loop() + sem = getattr(app, "_ansible_semaphore", None) + if sem is None or getattr(app, "_ansible_sem_loop", None) is not loop: + cap = int((app.config.get("ANSIBLE") or {}).get("max_concurrent_runs", 3) or 3) + sem = asyncio.Semaphore(max(1, cap)) + app._ansible_semaphore = sem + app._ansible_sem_loop = loop + queued = sem.locked() + if queued: + await _set_status(app, run_id, AnsibleRunStatus.queued) + _broadcast(run_id, "[queued — waiting for an Ansible run slot]") + await sem.acquire() + final_status = "failed" + logf = None try: - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.STDOUT, - cwd=cwd, - ) - assert proc.stdout is not None + if run_id in _cancelled: # cancelled while queued — abort before launching + raise _Aborted + if queued: + await _set_status(app, run_id, AnsibleRunStatus.running) - async for raw_line in proc.stdout: - line = raw_line.decode(errors="replace").rstrip("\n\r") - _broadcast(run_id, line) + cwd = source_path if Path(source_path).exists() else None + creds = app.config.get("ANSIBLE", {}) + try: + os.makedirs(ARTIFACT_DIR, exist_ok=True) + logf = open(artifact_path(run_id), "w", encoding="utf-8") + except OSError: + logf = None # artifact is best-effort; the DB still holds capped output - if not truncated: - candidate = (db_output + "\n" + line) if db_output else line - if len(candidate.encode()) > _OUTPUT_CAP_BYTES: - db_output += "\n[output truncated]" - truncated = True - else: - db_output = candidate + # Temp dir holds the ephemeral inventory (if any) + credential files. + tmpdir = tempfile.mkdtemp(prefix="steward-ansible-") + os.chmod(tmpdir, 0o700) + try: + if inventory_content: + # .yml so Ansible's yaml inventory plugin reliably claims it + # (the plugin's verify_file checks the extension). + inv_target = os.path.join(tmpdir, "inventory.yml") + with open(inv_target, "w", encoding="utf-8") as invf: + invf.write(inventory_content) + else: + inv_target = inventory_path + cmd = build_ansible_command(playbook_path, inv_target, params) - lines_since_flush += 1 - elapsed = time.monotonic() - last_flush_at - if lines_since_flush >= _FLUSH_LINES or elapsed >= _FLUSH_SECS: - await _flush() + # First-contact provisioning supplies a password and a bootstrap + # user; the managed key isn't on the host yet, so don't offer it + # (avoids a failed key attempt before the password fallback). + conn = connection or {} + effective_creds = dict(creds) + if conn.get("password"): + effective_creds.pop("ssh_private_key", None) + elif _crypto_is_encrypted(effective_creds.get("ssh_private_key") or ""): + _broadcast(run_id, + "[run error] The managed SSH key could not be decrypted " + "(the app secret key changed). Regenerate it in " + "Settings → Ansible and re-provision the host(s).") - await proc.wait() - final_status = "success" if proc.returncode == 0 else "failed" + cred_args, cred_files = build_credentials(effective_creds, tmpdir) + boot_args, boot_files = build_bootstrap(conn, tmpdir) + # Operator-entered run-time vars (persisted) + secret vars (not). + merged_vars = {**((params or {}).get("extra_vars_map") or {}), **(secret_vars or {})} + ev_args, ev_files = build_extra_vars_file(merged_vars, tmpdir) + for cred_path, content in cred_files + boot_files + ev_files: + with open(cred_path, "w", encoding="utf-8") as cf: + cf.write(content) + os.chmod(cred_path, 0o600) - except Exception: + # Steady-state floor: connect as the managed account unless a target + # var or the bootstrap override (extra-vars, higher precedence) wins. + user_args: list[str] = [] + if not conn.get("user"): + ssh_user = (creds.get("ssh_user") or "").strip() + if ssh_user: + user_args += ["--user", ssh_user] + + proc = await asyncio.create_subprocess_exec( + *cmd, *cred_args, *boot_args, *user_args, *ev_args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + cwd=cwd, + env=ansible_env(creds, os.environ), + ) + _run_procs[run_id] = proc + assert proc.stdout is not None + + async for raw_line in proc.stdout: + line = raw_line.decode(errors="replace").rstrip("\n\r") + _broadcast(run_id, line) + if logf is not None: + logf.write(line + "\n") + if (line.startswith("fatal:") or "FAILED!" in line) and len(failures) < _FAILURE_CAP: + failures.append(line[:500]) + + if not truncated: + candidate = (db_output + "\n" + line) if db_output else line + if len(candidate.encode()) > _OUTPUT_CAP_BYTES: + db_output += "\n[output truncated — download the full log]" + truncated = True + else: + db_output = candidate + + lines_since_flush += 1 + elapsed = time.monotonic() - last_flush_at + if lines_since_flush >= _FLUSH_LINES or elapsed >= _FLUSH_SECS: + await _flush() + + await proc.wait() + recap = parse_recap(_run_lines.get(run_id, [])) + if run_id in _cancelled: + final_status = "cancelled" + elif proc.returncode != 0: + final_status = "failed" + elif not recap: + # ansible-playbook exits 0 on "no hosts matched" / empty inventory + # — nothing actually ran, so don't report it as a success. + final_status = "failed" + if len(failures) < _FAILURE_CAP: + failures.append( + "No hosts matched — nothing executed " + "(check the host's Ansible target and inventory).") + else: + final_status = "success" + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + except _Aborted: + final_status = "cancelled" + except Exception as exc: logger.exception("Ansible run %s failed with exception", run_id) - final_status = "failed" + final_status = "cancelled" if run_id in _cancelled else "failed" + # Surface WHY in the run output instead of an empty "failed" (e.g. ENOSPC + # when the temp dir can't be created — the run never reaches Ansible). + msg = f"[run error] {type(exc).__name__}: {exc}" + _broadcast(run_id, msg) + if len(failures) < _FAILURE_CAP: + failures.append(msg[:500]) + db_output = (db_output + "\n" + msg) if db_output else msg + finally: + if logf is not None: + logf.close() + _run_procs.pop(run_id, None) + _cancelled.discard(run_id) + sem.release() - await _flush(final_status) + results = {"hosts": parse_recap(_run_lines.get(run_id, [])), "failures": failures} + await _flush(final_status, results=results) _broadcast(run_id, _Done(status=final_status)) diff --git a/steward/ansible/inventory_gen.py b/steward/ansible/inventory_gen.py new file mode 100644 index 0000000..2938625 --- /dev/null +++ b/steward/ansible/inventory_gen.py @@ -0,0 +1,108 @@ +"""Ansible --list inventory generation from DB records.""" +from __future__ import annotations + + +def generate_inventory(targets: list) -> dict: + """Build an Ansible --list JSON inventory from a list of AnsibleTarget objects. + + Accepts any objects with .name, .address, .ansible_vars (dict), .groups (list + of objects with .name and .ansible_vars). Uses duck typing so unit tests can + pass SimpleNamespace objects. + + Var precedence (lowest → highest, matching Ansible default): + group vars (sorted alphabetically by group name) → target ansible_vars → ansible_host + """ + inv: dict = {"all": {"hosts": []}, "_meta": {"hostvars": {}}} + + for target in targets: + inv["all"]["hosts"].append(target.name) + + # Merge vars: group vars alphabetically, then host vars, then force ansible_host. + merged: dict = {} + for group in sorted(target.groups, key=lambda g: g.name): + merged.update(group.ansible_vars or {}) + merged.update(target.ansible_vars or {}) + merged["ansible_host"] = target.address + + inv["_meta"]["hostvars"][target.name] = merged + + for group in target.groups: + if group.name not in inv: + inv[group.name] = { + "hosts": [], + "vars": dict(group.ansible_vars or {}), + } + inv[group.name]["hosts"].append(target.name) + + return inv + + +def inventory_to_yaml(inv: dict) -> str: + """Convert the ``--list`` dict from generate_inventory into a STATIC YAML + inventory string. + + The ``--list`` shape (``all.hosts`` is a *list*, vars under ``_meta``) is only + valid as the stdout of an executable dynamic-inventory script. When written to + a plain file and passed via ``-i``, Ansible's yaml plugin requires + ``all.hosts`` to be a *dict* keyed by hostname — otherwise it fails to parse + and silently falls back to implicit localhost. This emits that valid form. + """ + import yaml + hostvars = (inv.get("_meta") or {}).get("hostvars") or {} + all_hosts = (inv.get("all") or {}).get("hosts") or [] + root: dict = {"hosts": {h: (hostvars.get(h) or {}) for h in all_hosts}} + children: dict = {} + for key, val in inv.items(): + if key in ("all", "_meta") or not isinstance(val, dict): + continue + grp: dict = {"hosts": {h: {} for h in (val.get("hosts") or [])}} + if val.get("vars"): + grp["vars"] = val["vars"] + children[key] = grp + if children: + root["children"] = children + return yaml.safe_dump({"all": root}, default_flow_style=False, sort_keys=True) + + +async def fetch_scope_targets(db, scope: str) -> list: + """Query AnsibleTarget objects for a given inventory scope string. + + Scopes: + steward:all — every target + steward:group: — targets in that group + steward:target: — single target by ID + + Returns a list with groups pre-loaded (selectinload). Returns [] for + repo:* scopes (caller handles those via legacy path). + """ + from sqlalchemy import select + from sqlalchemy.orm import selectinload + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + + if scope == "steward:all": + stmt = ( + select(AnsibleTarget) + .options(selectinload(AnsibleTarget.groups)) + .order_by(AnsibleTarget.name) + ) + elif scope.startswith("steward:group:"): + group_id = scope[len("steward:group:"):] + stmt = ( + select(AnsibleTarget) + .join(AnsibleTarget.groups) + .where(AnsibleGroup.id == group_id) + .options(selectinload(AnsibleTarget.groups)) + .order_by(AnsibleTarget.name) + ) + elif scope.startswith("steward:target:"): + target_id = scope[len("steward:target:"):] + stmt = ( + select(AnsibleTarget) + .where(AnsibleTarget.id == target_id) + .options(selectinload(AnsibleTarget.groups)) + ) + else: + return [] + + result = await db.execute(stmt) + return list(result.scalars().all()) diff --git a/steward/ansible/inventory_routes.py b/steward/ansible/inventory_routes.py new file mode 100644 index 0000000..94d3a2c --- /dev/null +++ b/steward/ansible/inventory_routes.py @@ -0,0 +1,256 @@ +"""CRUD routes for Ansible inventory: groups and targets.""" +from __future__ import annotations +import uuid +import yaml + +from quart import Blueprint, current_app, redirect, render_template, request, url_for +from sqlalchemy import select +from sqlalchemy.orm import selectinload + +from steward.auth.middleware import require_role +from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget +from steward.models.users import UserRole + +inventory_bp = Blueprint("ansible_inventory", __name__, url_prefix="/ansible/inventory") + + +def _parse_ansible_vars(raw: str) -> dict: + """Parse a YAML/JSON string into a dict. Raises ValueError on invalid input.""" + raw = raw.strip() + if not raw: + return {} + try: + parsed = yaml.safe_load(raw) + except yaml.YAMLError as exc: + raise ValueError(f"Invalid YAML: {exc}") from exc + if parsed is None: + return {} + if not isinstance(parsed, dict): + raise ValueError("ansible_vars must be a YAML mapping (key: value pairs)") + return parsed + + +# --------------------------------------------------------------------------- +# Groups +# --------------------------------------------------------------------------- + +@inventory_bp.get("/groups") +@require_role(UserRole.viewer) +async def groups_list(): + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleGroup) + .options(selectinload(AnsibleGroup.targets)) + .order_by(AnsibleGroup.name) + ) + groups = result.scalars().all() + return await render_template("ansible/inventory/groups.html", groups=groups) + + +@inventory_bp.post("/groups") +@require_role(UserRole.operator) +async def groups_create(): + form = await request.form + name = form.get("name", "").strip() + if not name: + return "name is required", 400 + try: + ansible_vars = _parse_ansible_vars(form.get("ansible_vars", "")) + except ValueError as exc: + return str(exc), 400 + + group = AnsibleGroup(id=str(uuid.uuid4()), name=name, ansible_vars=ansible_vars) + async with current_app.db_sessionmaker() as db: + async with db.begin(): + db.add(group) + return redirect(url_for("ansible_inventory.groups_list")) + + +@inventory_bp.get("/groups/") +@require_role(UserRole.viewer) +async def group_detail(group_id: str): + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleGroup) + .where(AnsibleGroup.id == group_id) + .options(selectinload(AnsibleGroup.targets)) + ) + group = result.scalar_one_or_none() + if group is None: + return "Group not found", 404 + all_targets = ( + await db.execute(select(AnsibleTarget).order_by(AnsibleTarget.name)) + ).scalars().all() + member_ids = {t.id for t in group.targets} + ansible_vars_yaml = yaml.dump(group.ansible_vars) if group.ansible_vars else "" + return await render_template( + "ansible/inventory/group_detail.html", + group=group, + all_targets=all_targets, + member_ids=member_ids, + ansible_vars_yaml=ansible_vars_yaml, + ) + + +@inventory_bp.post("/groups/") +@require_role(UserRole.operator) +async def group_update(group_id: str): + form = await request.form + try: + ansible_vars = _parse_ansible_vars(form.get("ansible_vars", "")) + except ValueError as exc: + return str(exc), 400 + + async with current_app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute( + select(AnsibleGroup) + .where(AnsibleGroup.id == group_id) + .options(selectinload(AnsibleGroup.targets)) + ) + group = result.scalar_one_or_none() + if group is None: + return "Group not found", 404 + + name = form.get("name", group.name).strip() + if name: + group.name = name + group.ansible_vars = ansible_vars + + new_ids = set(form.getlist("target_ids")) + all_targets = ( + await db.execute(select(AnsibleTarget)) + ).scalars().all() + target_map = {t.id: t for t in all_targets} + group.targets = [target_map[tid] for tid in new_ids if tid in target_map] + + return redirect(url_for("ansible_inventory.group_detail", group_id=group_id)) + + +@inventory_bp.post("/groups//delete") +@require_role(UserRole.operator) +async def group_delete(group_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute( + select(AnsibleGroup).where(AnsibleGroup.id == group_id) + ) + group = result.scalar_one_or_none() + if group is not None: + await db.delete(group) + return redirect(url_for("ansible_inventory.groups_list")) + + +# --------------------------------------------------------------------------- +# Targets +# --------------------------------------------------------------------------- + +@inventory_bp.get("/targets") +@require_role(UserRole.viewer) +async def targets_list(): + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleTarget) + .options(selectinload(AnsibleTarget.groups)) + .order_by(AnsibleTarget.name) + ) + targets = result.scalars().all() + return await render_template("ansible/inventory/targets.html", targets=targets) + + +@inventory_bp.post("/targets") +@require_role(UserRole.operator) +async def targets_create(): + form = await request.form + name = form.get("name", "").strip() + address = form.get("address", "").strip() + if not name or not address: + return "name and address are required", 400 + try: + ansible_vars = _parse_ansible_vars(form.get("ansible_vars", "")) + except ValueError as exc: + return str(exc), 400 + + target = AnsibleTarget( + id=str(uuid.uuid4()), name=name, address=address, ansible_vars=ansible_vars + ) + async with current_app.db_sessionmaker() as db: + async with db.begin(): + db.add(target) + return redirect(url_for("ansible_inventory.targets_list")) + + +@inventory_bp.get("/targets/") +@require_role(UserRole.viewer) +async def target_detail(target_id: str): + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleTarget) + .where(AnsibleTarget.id == target_id) + .options(selectinload(AnsibleTarget.groups)) + ) + target = result.scalar_one_or_none() + if target is None: + return "Target not found", 404 + all_groups = ( + await db.execute(select(AnsibleGroup).order_by(AnsibleGroup.name)) + ).scalars().all() + member_group_ids = {g.id for g in target.groups} + ansible_vars_yaml = yaml.dump(target.ansible_vars) if target.ansible_vars else "" + return await render_template( + "ansible/inventory/target_detail.html", + target=target, + all_groups=all_groups, + member_group_ids=member_group_ids, + ansible_vars_yaml=ansible_vars_yaml, + ) + + +@inventory_bp.post("/targets/") +@require_role(UserRole.operator) +async def target_update(target_id: str): + form = await request.form + try: + ansible_vars = _parse_ansible_vars(form.get("ansible_vars", "")) + except ValueError as exc: + return str(exc), 400 + + async with current_app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute( + select(AnsibleTarget) + .where(AnsibleTarget.id == target_id) + .options(selectinload(AnsibleTarget.groups)) + ) + target = result.scalar_one_or_none() + if target is None: + return "Target not found", 404 + + name = form.get("name", target.name).strip() + address = form.get("address", target.address).strip() + if name: + target.name = name + if address: + target.address = address + target.ansible_vars = ansible_vars + + new_group_ids = set(form.getlist("group_ids")) + all_groups_result = await db.execute(select(AnsibleGroup)) + group_map = {g.id: g for g in all_groups_result.scalars().all()} + target.groups = [group_map[gid] for gid in new_group_ids if gid in group_map] + + return redirect(url_for("ansible_inventory.target_detail", target_id=target_id)) + + +@inventory_bp.post("/targets//delete") +@require_role(UserRole.operator) +async def target_delete(target_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute( + select(AnsibleTarget).where(AnsibleTarget.id == target_id) + ) + target = result.scalar_one_or_none() + if target is not None: + await db.delete(target) + return redirect(url_for("ansible_inventory.targets_list")) diff --git a/steward/ansible/routes.py b/steward/ansible/routes.py index d01f860..5217b84 100644 --- a/steward/ansible/routes.py +++ b/steward/ansible/routes.py @@ -1,20 +1,27 @@ # steward/ansible/routes.py from __future__ import annotations -import asyncio -import uuid -from datetime import datetime, timezone -from pathlib import Path +from datetime import datetime, timedelta, timezone from quart import ( Blueprint, Response, current_app, redirect, render_template, request, session, url_for, ) -from sqlalchemy import select, update +from sqlalchemy import select from steward.ansible import executor, sources as src_module from steward.auth.middleware import require_role from steward.models.ansible import AnsibleRun, AnsibleRunStatus -from steward.models.users import UserRole +from steward.models.ansible_schedule import AnsibleSchedule +from steward.models.users import User, UserRole + +INTERVAL_PRESETS = [ + (300, "Every 5 minutes"), + (3600, "Hourly"), + (21600, "Every 6 hours"), + (43200, "Every 12 hours"), + (86400, "Daily"), + (604800, "Weekly"), +] ansible_bp = Blueprint("ansible", __name__, url_prefix="/ansible") @@ -38,17 +45,43 @@ async def index(): @ansible_bp.get("/browse") @require_role(UserRole.viewer) async def browse(): + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + all_sources = _get_sources() source_data = [] for source in all_sources: - playbooks = src_module.discover_playbooks(source["path"]) + playbooks = [] + for path in src_module.discover_playbooks(source["path"]): + contents = src_module.read_playbook(source["path"], path) + meta = (src_module.discover_playbook_meta(contents) if contents is not None + else {"description": "", "category": ""}) + playbooks.append({ + "path": path, + "category": meta["category"], + "description": meta["description"], + }) inventories = src_module.discover_inventories(source["path"]) source_data.append({ "source": source, "playbooks": playbooks, "inventories": inventories, + "editable": src_module.is_editable_source(source), }) - return await render_template("ansible/browse.html", source_data=source_data) + + async with current_app.db_sessionmaker() as db: + targets = (await db.execute( + select(AnsibleTarget).order_by(AnsibleTarget.name) + )).scalars().all() + groups = (await db.execute( + select(AnsibleGroup).order_by(AnsibleGroup.name) + )).scalars().all() + + return await render_template( + "ansible/browse.html", + source_data=source_data, + targets=targets, + groups=groups, + ) @ansible_bp.get("/browse//") @@ -67,55 +100,156 @@ async def view_playbook(source_name: str, playbook_path: str): view_source=source_name, view_path=playbook_path, view_contents=contents, + view_editable=src_module.is_editable_source(source), + ) + + +def _parse_run_params(form) -> tuple[dict | None, dict | None, str | None]: + """Build executor params + the unpersisted secret-vars dict from form fields. + + Returns (params, secret_vars, error). Discovered variable fields are named + ``var__``; a sibling hidden ``secret__`` marks the sensitive + ones, which go into secret_vars (never persisted) instead of params. A + free-form ``extra_vars`` textarea (key=value per line) is also folded in. + All run-time values flow through a JSON extra-vars file downstream, so + values may contain spaces/quotes safely. + """ + extra_vars_map: dict[str, str] = {} + secret_vars: dict[str, str] = {} + + for key in set(form.keys()): + if not key.startswith("var__"): + continue + name = key[len("var__"):] + val = (form.get(key) or "").strip() + if not val: + continue # untouched → fall through to inventory/play default + if f"secret__{name}" in form: + secret_vars[name] = val + else: + extra_vars_map[name] = val + + for line in (form.get("extra_vars", "") or "").splitlines(): + line = line.strip() + if not line: + continue + if "=" not in line: + return None, None, f"Invalid extra var (expected key=value): {line!r}" + k, v = line.split("=", 1) + extra_vars_map[k.strip()] = v + + params: dict = {} + if extra_vars_map: + params["extra_vars_map"] = extra_vars_map + limit = (form.get("limit", "") or "").strip() + if limit: + params["limit"] = limit + tags = (form.get("tags", "") or "").strip() + if tags: + params["tags"] = tags + if "check" in form: + params["check"] = True + return (params or None), (secret_vars or None), None + + +@ansible_bp.get("/playbook-options") +@require_role(UserRole.operator) +async def playbook_options(): + """HTMX fragment: