Compare commits
240
Commits
+3
-2
@@ -26,8 +26,9 @@ config.yaml
|
||||
# Runtime data (mounted at runtime via volume)
|
||||
playbook_cache/
|
||||
|
||||
# Plugins (mounted at runtime via volume)
|
||||
plugins/
|
||||
# NOTE: first-party plugins under plugins/ now ship IN the image
|
||||
# (Dockerfile `COPY plugins/`), so plugins/ must NOT be ignored here.
|
||||
# Third-party plugins are mounted at runtime into /data/plugins instead.
|
||||
|
||||
# Deployment files not needed in image
|
||||
docker-compose.yml
|
||||
|
||||
+8
-3
@@ -1,6 +1,11 @@
|
||||
# Copy to .env for Docker / local development secrets
|
||||
# These override values in config.yaml
|
||||
|
||||
ROUNDTABLE_SECRET_KEY=change-me
|
||||
ROUNDTABLE_DATABASE__URL=postgresql+asyncpg://roundtable:password@localhost/roundtable
|
||||
ROUNDTABLE_SMTP__PASSWORD=
|
||||
STEWARD_DATABASE_URL=postgresql+asyncpg://steward:password@localhost/steward
|
||||
STEWARD_SMTP__PASSWORD=
|
||||
|
||||
# --- compose.deploy.yml (remote instance) only ---
|
||||
# Password for the bundled Postgres; the steward service builds its
|
||||
# STEWARD_DATABASE_URL from it. Leave STEWARD_SECRET_KEY above unset to let the
|
||||
# app generate + persist one on the /data volume.
|
||||
POSTGRES_PASSWORD=change-me
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
name: CI
|
||||
|
||||
# CI lanes per FabledRulebook forgejo.md "CI philosophy":
|
||||
# - lint: ruff only, no dep install — fast-fail for the common lint bounce.
|
||||
# - unit: pytest -m "not integration"; no service containers, no DB.
|
||||
# - integration: postgres service container; boots the real app + runs all
|
||||
# (core + bundled-plugin) migrations.
|
||||
# Single repo: first-party plugins are bundled in-tree under plugins/, so the
|
||||
# unit lane imports them directly — no cross-repo checkout.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
# pull_request intentionally absent — push on [dev, main] already fires CI for
|
||||
# every dev commit and dev→main PRs. Single-operator repo, no fork PRs.
|
||||
|
||||
# Serialize CI runs per ref so the publish lane never shares the runner's docker
|
||||
# daemon with another run. Two publishes racing on one daemon evict each other's
|
||||
# freshly-built image mid-push, breaking the post-build tag/push (issue #1093).
|
||||
# cancel-in-progress:false is deliberate — rule 46 requires EVERY dev/main push to
|
||||
# publish its own immutable :<sha> image, so a superseded run must still run to
|
||||
# completion (never cancelled) to emit its SHA. (Forgejo honors `concurrency:` —
|
||||
# CI-runner's build workflows use it.)
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
# Fast-fail lint lane. ruff is pre-installed in the ci-python image, so this
|
||||
# runs with NO dependency install and surfaces lint bounces in seconds.
|
||||
lint:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Ruff lint
|
||||
run: ruff check steward/ plugins/ tests/
|
||||
|
||||
unit:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install Steward + test deps
|
||||
# uv: 5-10x faster wheel resolve than pip on cold caches. Falls back to
|
||||
# pip on uv-missing runners. Steward declares its deps in pyproject; the
|
||||
# [dev] extra adds pytest + pytest-asyncio.
|
||||
run: |
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -e '.[dev]'
|
||||
else
|
||||
pip install -e '.[dev]'
|
||||
fi
|
||||
- name: Pytest (unit only — integration runs in its own lane)
|
||||
run: pytest tests/ -v -m "not integration"
|
||||
|
||||
integration:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
STEWARD_SECRET_KEY: ci_integration_placeholder
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_USER: steward
|
||||
POSTGRES_PASSWORD: steward
|
||||
POSTGRES_DB: steward
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U steward"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Boot + migrate against Postgres
|
||||
# act_runner (swarm-runner) puts service containers on the default bridge
|
||||
# with no embedded DNS, so the hostname `postgres` won't resolve — we
|
||||
# discover the service container's bridge IP via the docker socket. The
|
||||
# job container and its Postgres share a task-number name prefix.
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
echo "=== container landscape (diagnostic for filter scoping) ==="
|
||||
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||
echo "=== end landscape ==="
|
||||
# The runner's daemon is shared: another repo's integration job can be
|
||||
# running beside this one, and `name=integration` matches its Postgres
|
||||
# too (run 8358 took the wrong one and every test failed to connect).
|
||||
# Scope to this job's own task: act_runner names every container of a
|
||||
# task GITEA-ACTIONS-TASK-<n>-…, so read <n> from our own container
|
||||
# (its id is in the /etc/hostname bind mount) and require one match.
|
||||
SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
|
||||
SELF=${SELF:-$(hostname)}
|
||||
TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
|
||||
test -n "$TASK"
|
||||
PG=$(docker ps --filter "name=$TASK" --filter "ancestor=postgres:16-alpine" -q)
|
||||
test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
test -n "$PG_IP"
|
||||
export STEWARD_DATABASE_URL="postgresql+asyncpg://steward:steward@$PG_IP:5432/steward"
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -e '.[dev,ansible]'
|
||||
else
|
||||
pip install -e '.[dev,ansible]'
|
||||
fi
|
||||
pytest tests/integration -v -m integration --durations=10
|
||||
|
||||
# Image-publish lane — FabledRulebook rule 46: every push to dev/main publishes
|
||||
# an immutable commit-SHA image (the rollback unit); dev moves :dev, main moves
|
||||
# :latest. Gated on the three test lanes via `needs:` so a red commit never
|
||||
# produces a :dev image. Mechanics mirror CI-runner's build-ci-python.yml.
|
||||
publish:
|
||||
needs: [lint, unit, integration]
|
||||
runs-on: python-ci
|
||||
container:
|
||||
# ci-builder carries docker + buildx + git. The docker socket is
|
||||
# auto-mounted by act_runner (valid_volumes) — do NOT add a
|
||||
# container.options "-v /var/run/docker.sock:..." mount; that duplicate
|
||||
# mount fails the job at "Set up job".
|
||||
image: git.fabledsword.com/bvandeusen/ci-builder:latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Registry login
|
||||
# The injected GITHUB_TOKEN lacks write:package scope, so docker push
|
||||
# 401s with it. REGISTRY_TOKEN is a repo Actions secret holding a
|
||||
# Forgejo token scoped read:package + write:package.
|
||||
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.fabledsword.com -u bvandeusen --password-stdin
|
||||
- name: Build (commit-SHA always; :dev on dev, :latest on main)
|
||||
run: |
|
||||
set -eux
|
||||
IMAGE=git.fabledsword.com/bvandeusen/steward
|
||||
# The moving tag for this ref: dev→:dev, main→:latest (rule 46). main IS
|
||||
# the production line, so :latest tracks main's tip; there is no :main.
|
||||
MOVING=""
|
||||
if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
|
||||
MOVING="dev"
|
||||
elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
|
||||
MOVING="latest"
|
||||
fi
|
||||
# Tag BOTH targets at build time (mirrors CI-runner's build workflows):
|
||||
# one `docker build -t :<sha> -t :<moving>` points both tags at the image
|
||||
# atomically, so we never run a post-push `docker tag` of an image a
|
||||
# concurrent run could have evicted from the shared daemon (issue #1093).
|
||||
if [ -n "$MOVING" ]; then
|
||||
docker build -t "$IMAGE:${{ github.sha }}" -t "$IMAGE:$MOVING" .
|
||||
else
|
||||
docker build -t "$IMAGE:${{ github.sha }}" .
|
||||
fi
|
||||
|
||||
# Rule 164's check, BETWEEN build and push (rule 177): an image that
|
||||
# can't boot, migrate and serve with no internet never reaches a tag.
|
||||
# An --internal network lets it reach its own Postgres and nothing else;
|
||||
# --network none can't work for an app that needs a database to start.
|
||||
# Embedded DNS works on a user-defined network, unlike the services:
|
||||
# containers above (rule 79), so the app finds Postgres by name.
|
||||
- name: The image boots with no internet
|
||||
env:
|
||||
SMOKE: steward-smoke-${{ github.run_id }}
|
||||
run: |
|
||||
set -eux
|
||||
IMAGE=git.fabledsword.com/bvandeusen/steward:${{ github.sha }}
|
||||
docker network create --internal "$SMOKE-net"
|
||||
docker run -d --name "$SMOKE-pg" --network "$SMOKE-net" \
|
||||
-e POSTGRES_USER=steward -e POSTGRES_PASSWORD=steward -e POSTGRES_DB=steward \
|
||||
postgres:16-alpine
|
||||
# Waits with deadlines (rule 156); pg_isready, not /dev/tcp, which is
|
||||
# a bash builtin and never succeeds under sh.
|
||||
ready=""
|
||||
for _ in $(seq 1 60); do
|
||||
if docker exec "$SMOKE-pg" pg_isready -U steward -d steward -q; then ready=1; break; fi
|
||||
sleep 2
|
||||
done
|
||||
test -n "$ready" || { echo "FATAL: postgres not ready within 120s"; exit 1; }
|
||||
|
||||
docker run -d --name "$SMOKE-app" --network "$SMOKE-net" \
|
||||
-e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/steward" \
|
||||
-e STEWARD_SECRET_KEY=smoke-test-placeholder \
|
||||
"$IMAGE"
|
||||
# The image has no curl; its own Python fetches. Prints the body.
|
||||
fetch() {
|
||||
docker exec "$SMOKE-app" python -c 'import sys, urllib.request
|
||||
r = urllib.request.urlopen("http://127.0.0.1:5000" + sys.argv[1], timeout=5)
|
||||
sys.stdout.write(r.read().decode())' "$1"
|
||||
}
|
||||
# Migrations run before the server starts, so /health answering
|
||||
# means the schema applied against a real database.
|
||||
up=""
|
||||
for _ in $(seq 1 60); do
|
||||
if fetch /health >/dev/null 2>&1; then up=1; break; fi
|
||||
sleep 2
|
||||
done
|
||||
if [ -z "$up" ]; then
|
||||
echo "FATAL: Steward never answered /health with no internet access"
|
||||
docker logs "$SMOKE-app" 2>&1 | tail -80
|
||||
exit 1
|
||||
fi
|
||||
fetch /health | grep -q '"ok"'
|
||||
|
||||
# tini is PID 1, so zombies are reaped and SIGTERM reaches Steward.
|
||||
test "$(docker exec "$SMOKE-app" cat /proc/1/comm)" = "tini"
|
||||
|
||||
# A fresh database lands on first-run setup: a real page, its assets
|
||||
# served by Steward itself, and no third-party host in it (rule 155).
|
||||
fetch / > /tmp/smoke-page.html
|
||||
grep -qi "<!doctype html" /tmp/smoke-page.html
|
||||
grep -q "/static/" /tmp/smoke-page.html
|
||||
if grep -Eqi 'https?://(fonts\.|cdn\.|unpkg\.|cdnjs\.|ajax\.)' /tmp/smoke-page.html; then
|
||||
echo "FATAL: the page references a third-party host"
|
||||
grep -Eoi 'https?://[^"'"'"' ]+' /tmp/smoke-page.html
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A stop is a clean shutdown: SIGTERM through tini to Hypercorn, which
|
||||
# finishes within its graceful timeout and exits 0.
|
||||
docker stop -t 30 "$SMOKE-app"
|
||||
code="$(docker inspect -f '{{.State.ExitCode}}' "$SMOKE-app")"
|
||||
test "$code" = "0" || { echo "FATAL: Steward exited $code on SIGTERM"; exit 1; }
|
||||
|
||||
- name: Push
|
||||
run: |
|
||||
set -eux
|
||||
IMAGE=git.fabledsword.com/bvandeusen/steward
|
||||
docker push "$IMAGE:${{ github.sha }}"
|
||||
if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
|
||||
docker push "$IMAGE:dev"
|
||||
elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
|
||||
docker push "$IMAGE:latest"
|
||||
fi
|
||||
|
||||
- name: Smoke logs
|
||||
if: always()
|
||||
run: docker logs "steward-smoke-${{ github.run_id }}-app" 2>&1 | tail -60 || true
|
||||
|
||||
# A failed smoke must not leave a Postgres and an app on the shared daemon.
|
||||
- name: Smoke clean-up
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f "steward-smoke-${{ github.run_id }}-app" "steward-smoke-${{ github.run_id }}-pg" >/dev/null 2>&1 || true
|
||||
docker network rm "steward-smoke-${{ github.run_id }}-net" >/dev/null 2>&1 || true
|
||||
- name: Prune dangling layers
|
||||
if: always()
|
||||
run: docker image prune -f
|
||||
+10
-3
@@ -1,9 +1,10 @@
|
||||
# Planning files
|
||||
docs/superpowers/
|
||||
|
||||
# Plugin directory — managed as a separate repo (bvandeusen/roundtable-plugins)
|
||||
# PLUGIN_DIR in config.yaml points here at runtime
|
||||
/plugins/
|
||||
# First-party plugins are tracked in-tree under plugins/ and ship in the image.
|
||||
# Third-party plugins are mounted at runtime into the external dir
|
||||
# (STEWARD_PLUGIN_DIR, default /data/plugins) and are not tracked here.
|
||||
/plugins/**/*.zip
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
@@ -50,3 +51,9 @@ Thumbs.db
|
||||
|
||||
# Playbook cache (runtime data)
|
||||
playbook_cache/
|
||||
|
||||
# Superpowers brainstorm scratch (visual companion mockups, state)
|
||||
.superpowers/
|
||||
|
||||
# Local dev Ansible playbooks (operator content, mounted into the container)
|
||||
/ansible-dev/
|
||||
|
||||
+26
-7
@@ -1,10 +1,21 @@
|
||||
FROM python:3.13-slim
|
||||
# The Python CI tests on (ci-python:3.14), so the image runs what was tested.
|
||||
FROM python:3.14-slim
|
||||
|
||||
RUN DEBIAN_FRONTEND=noninteractive apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
iputils-ping \
|
||||
# gosu — minimal privilege-drop tool used by entrypoint.sh
|
||||
gosu \
|
||||
# openssh-client — ansible-playbook reaches remote hosts over ssh
|
||||
openssh-client \
|
||||
# sshpass — Ansible shells out to it for first-contact password SSH
|
||||
# (provisioning a fresh host before the managed key is installed)
|
||||
sshpass \
|
||||
# git — clones and pulls Ansible git sources (Settings > Ansible)
|
||||
git \
|
||||
# tini — PID 1: reaps the zombies ssh and ansible leave behind and
|
||||
# forwards SIGTERM to Steward so a stop is a graceful shutdown
|
||||
tini \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Upgrade pip to suppress the version notice
|
||||
@@ -13,21 +24,29 @@ RUN pip install --upgrade pip
|
||||
WORKDIR /app
|
||||
|
||||
COPY pyproject.toml .
|
||||
COPY roundtable/ roundtable/
|
||||
RUN pip install --no-cache-dir .
|
||||
COPY steward/ steward/
|
||||
# Bundle the extras whose features ship in the image: [ansible] for the
|
||||
# playbook runner, [snmp] (pysnmp) for the SNMP poller, [ldap] (ldap3, pure
|
||||
# Python) for LDAP sign-in under Settings -> Auth. Without them those features
|
||||
# load but fail for want of their runtime dependency.
|
||||
RUN pip install --no-cache-dir '.[ansible,snmp,ldap]'
|
||||
|
||||
COPY alembic.ini .
|
||||
# First-party plugins ship inside the image (bundled root at /app/plugins).
|
||||
# Third-party plugins are mounted at runtime into the external root
|
||||
# (STEWARD_PLUGIN_DIR, default /data/plugins).
|
||||
COPY plugins/ plugins/
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
# Runtime directories — owned by app user. The container starts as root
|
||||
# so the entrypoint can fix up /data permissions if needed, then drops to
|
||||
# 'app' (uid 1000) via gosu before launching roundtable.
|
||||
# 'app' (uid 1000) via gosu before launching steward.
|
||||
RUN useradd -m -u 1000 app \
|
||||
&& mkdir -p /data/playbook_cache /data/plugins \
|
||||
&& mkdir -p /data/ansible/sources /data/ansible/runs /data/ansible/playbooks /data/plugins \
|
||||
&& chown -R app:app /data /app
|
||||
|
||||
EXPOSE 5000
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
CMD ["roundtable", "--host", "0.0.0.0", "--port", "5000"]
|
||||
ENTRYPOINT ["/usr/bin/tini", "--", "/entrypoint.sh"]
|
||||
CMD ["steward", "--host", "0.0.0.0", "--port", "5000"]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Roundtable
|
||||
# Steward
|
||||
|
||||
A self-hosted network monitoring and infrastructure management hub for home servers. Roundtable gives you a single pane of glass over your hosts, services, and automation — with live-updating dashboards, alerting, and Ansible integration.
|
||||
A self-hosted network monitoring and infrastructure management hub for home servers. Steward gives you a single pane of glass over your hosts, services, and automation — with live-updating dashboards, alerting, and Ansible integration.
|
||||
|
||||
---
|
||||
|
||||
@@ -21,13 +21,49 @@ No JavaScript framework. No build step. No external workers or message brokers.
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# Set ROUNDTABLE_DATABASE_URL in .env
|
||||
# Set STEWARD_DATABASE_URL in .env
|
||||
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
Open `http://localhost:5000`. On first run you'll be prompted to create the admin account.
|
||||
|
||||
This compose file builds the image locally and bind-mounts the source for live editing. To run a persistent instance off the published CI image instead, see **Remote dev instance** below.
|
||||
|
||||
---
|
||||
|
||||
## Remote dev instance
|
||||
|
||||
For a long-lived instance on a remote server that tracks the `dev` CI line, use `compose.deploy.yml`. It pulls the published image (`git.fabledsword.com/bvandeusen/steward:dev`) rather than building, runs no source bind-mounts, and persists data in named volumes.
|
||||
|
||||
**One-time prerequisite — registry push token (on the Git host).** CI publishes the image, so the FabledSteward repo needs a push credential. The injected `GITHUB_TOKEN` lacks `write:package`, so create a Forgejo token scoped **`read:package` + `write:package`** and add it as the repo Actions secret **`REGISTRY_TOKEN`**. Until this exists, the `publish` CI job 401s and no `:dev` image is produced.
|
||||
|
||||
**Standup (on the server):**
|
||||
|
||||
```bash
|
||||
# 1. Authenticate to the registry (read:package token is enough to pull)
|
||||
docker login git.fabledsword.com -u <user>
|
||||
|
||||
# 2. Configure secrets
|
||||
cp .env.example .env
|
||||
# Set POSTGRES_PASSWORD. Leave STEWARD_SECRET_KEY unset to let the app
|
||||
# generate + persist one on the /data volume.
|
||||
|
||||
# 3. Pull + boot
|
||||
docker compose -f compose.deploy.yml up -d
|
||||
```
|
||||
|
||||
Open `http://<server>:5000` and create the admin account on first run.
|
||||
|
||||
**Update to the latest dev build:**
|
||||
|
||||
```bash
|
||||
docker compose -f compose.deploy.yml pull
|
||||
docker compose -f compose.deploy.yml up -d
|
||||
```
|
||||
|
||||
Every push to `dev` that goes CI-green publishes a fresh `:dev` (and an immutable `:<commit-sha>` for rollback). To pin a specific commit, change the `image:` tag in `compose.deploy.yml` to `:<sha>`.
|
||||
|
||||
---
|
||||
|
||||
## Quick Start — Bare Metal
|
||||
@@ -41,7 +77,7 @@ pip install -e .
|
||||
cp config.example.yaml config.yaml
|
||||
# Edit config.yaml — set database.url at minimum
|
||||
|
||||
roundtable --host 0.0.0.0 --port 5000
|
||||
steward --host 0.0.0.0 --port 5000
|
||||
```
|
||||
|
||||
ICMP ping requires `CAP_NET_RAW` or the `setuid` ping binary. TCP mode (the default) needs no elevated privileges.
|
||||
@@ -54,7 +90,7 @@ Only two things must be set to run the app:
|
||||
|
||||
| What | How |
|
||||
|---|---|
|
||||
| Database URL | `ROUNDTABLE_DATABASE_URL` env var or `database.url` in `config.yaml` |
|
||||
| Database URL | `STEWARD_DATABASE_URL` env var or `database.url` in `config.yaml` |
|
||||
| Secret key | Auto-generated on first run and saved to `/data/secret.key` |
|
||||
|
||||
Everything else — SMTP, webhooks, monitor intervals, Ansible sources, plugin settings — is configured through the web UI at `/settings/`.
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
[alembic]
|
||||
script_location = roundtable/migrations
|
||||
script_location = steward/migrations
|
||||
prepend_sys_path = .
|
||||
version_path_separator = os
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
# CI Requirements — Steward
|
||||
|
||||
> Spec: https://git.fabledsword.com/bvandeusen/CI-runner/src/branch/main/docs/process.md
|
||||
|
||||
## Runtime image
|
||||
|
||||
git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
|
||||
## Image deps used
|
||||
|
||||
- python 3.14
|
||||
- ruff (analyzer for `steward/`, `plugins/`, `tests/`)
|
||||
- docker CLI (integration lane: bridge-IP discovery of the Postgres service container)
|
||||
|
||||
## Per-job tool installs
|
||||
|
||||
- `uv pip install --system -e '.[dev]'` (pip fallback) — in the `unit` job.
|
||||
- `uv pip install --system -e '.[dev,ansible]'` (pip fallback) — in the
|
||||
`integration` job, which boots the real app and exercises the Ansible runner,
|
||||
so it needs the `ansible` extra (`ansible>=10,<13`) at import time.
|
||||
|
||||
Steward declares its deps in `pyproject.toml`; the `[dev]` extra adds
|
||||
`pytest` + `pytest-asyncio`. No `requirements.txt` is tracked.
|
||||
|
||||
## Lanes
|
||||
|
||||
- **lint** — `ruff check steward/ plugins/ tests/`, no dep install.
|
||||
- **unit** — `pytest -m "not integration"`. The whole current suite runs here:
|
||||
it builds the app with `testing=True`, which mocks `db_sessionmaker`, so no DB
|
||||
is touched. First-party plugins are bundled in-tree under `plugins/`, so plugin
|
||||
unit tests import them directly — no cross-repo checkout.
|
||||
- **integration** — `pytest tests/integration -m integration` against a
|
||||
`postgres:16-alpine` service. The single test boots the real app
|
||||
(`create_app(testing=False)`), which runs core + every bundled plugin's
|
||||
migrations, then round-trips the DB. This is the guard that the folded-in
|
||||
plugin migration graph resolves.
|
||||
- **publish** — builds the runtime `Dockerfile` and pushes to the package
|
||||
registry. Runs in `ci-builder:latest` (docker + buildx, socket auto-mounted),
|
||||
gated on `needs: [lint, unit, integration]` so a red commit never ships an
|
||||
image. Publishes `git.fabledsword.com/bvandeusen/steward:<sha>` always, plus
|
||||
`:dev` on `dev` and `:latest` on `main` (FabledRulebook rule 46). Needs the
|
||||
`REGISTRY_TOKEN` repo Actions secret (`read:package` + `write:package`) —
|
||||
the injected `GITHUB_TOKEN` can't push packages.
|
||||
|
||||
## Notes
|
||||
|
||||
- Steward has **no frontend and no Redis/Celery** (no external workers), so there
|
||||
is no frontend-build lane and the only service container is Postgres.
|
||||
- Integration uses Fabled-Git Actions `services:` + a socket-discovered bridge IP
|
||||
because `act_runner` (swarm-runner v0.6+) puts services on the default bridge
|
||||
with no embedded DNS. FabledCurator's `ci.yml` is the canonical example of the
|
||||
pattern; Steward's is the same shape minus Redis and minus sharding.
|
||||
- The job name `integration` has no underscore — `act_runner` strips underscores
|
||||
from job names when building service-container labels, so the `docker ps`
|
||||
name filter uses the bare job name.
|
||||
- Migrations run via the app itself (`create_app` → `run_core_migrations`, async
|
||||
through `asyncpg`), so the integration lane needs no separate `alembic upgrade`
|
||||
step and no psycopg/sync driver.
|
||||
@@ -0,0 +1,69 @@
|
||||
# Remote deployment — pulls the published :dev image instead of building locally.
|
||||
#
|
||||
# This is NOT the local-dev compose. `docker-compose.yml` bind-mounts ./steward
|
||||
# + ./plugins and runs --debug for live editing; THIS file runs the image as the
|
||||
# source of truth (no bind-mounts, no reloader) for a persistent remote instance
|
||||
# that tracks the dev CI line.
|
||||
#
|
||||
# Standup + update runbook: see README → "Remote dev instance".
|
||||
|
||||
services:
|
||||
steward:
|
||||
image: git.fabledsword.com/bvandeusen/steward:dev
|
||||
container_name: steward
|
||||
# Re-pull :dev on every `up` so restarting the stack picks up the latest
|
||||
# green dev build without a manual `docker pull`.
|
||||
pull_policy: always
|
||||
ports:
|
||||
- "5000:5000"
|
||||
networks:
|
||||
- backend
|
||||
volumes:
|
||||
# /data holds the auto-generated secret key, third-party plugins, and the
|
||||
# Ansible playbook cache — all of which must survive image updates. No
|
||||
# source bind-mounts: core + first-party plugins live inside the image.
|
||||
#
|
||||
# The container runs as the unprivileged 'app' user (uid 1000), so /data
|
||||
# MUST be writable by uid 1000. A named docker volume (below) gets that
|
||||
# automatically. If you bind-mount a host/NFS path instead, chown it to
|
||||
# 1000:1000 (and mind NFS root_squash) — otherwise the app can't persist
|
||||
# its secret key, mints a fresh one each boot, and every encrypted secret
|
||||
# (managed SSH key, SMTP/OIDC/LDAP creds) becomes unrecoverable. The app
|
||||
# now refuses to start in that state rather than silently degrade.
|
||||
- app_data:/data
|
||||
environment:
|
||||
- STEWARD_DATABASE_URL=postgresql+asyncpg://steward:${POSTGRES_PASSWORD:-steward}@db/steward
|
||||
# STEWARD_SECRET_KEY is intentionally absent — the app generates a random
|
||||
# key on first boot and persists it to /data/secret.key, which the app_data
|
||||
# volume keeps across image updates. On multi-node Swarm with a shared bind
|
||||
# mount, pin STEWARD_SECRET_KEY (env or a docker secret) instead, so the key
|
||||
# never depends on filesystem ownership being correct on every node.
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
db:
|
||||
image: postgres:16-alpine
|
||||
networks:
|
||||
- backend
|
||||
environment:
|
||||
POSTGRES_USER: steward
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-steward}
|
||||
POSTGRES_DB: steward
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
# No host port published — db is reachable only on the backend network.
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U steward"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
backend:
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
app_data:
|
||||
+11
-3
@@ -1,9 +1,9 @@
|
||||
# Roundtable — Bootstrap Configuration Example
|
||||
# Steward — Bootstrap Configuration Example
|
||||
#
|
||||
# The only REQUIRED setting is the database URL.
|
||||
# Set it via env var (recommended for Docker):
|
||||
#
|
||||
# ROUNDTABLE_DATABASE_URL=postgresql+asyncpg://user:pass@host/db
|
||||
# STEWARD_DATABASE_URL=postgresql+asyncpg://user:pass@host/db
|
||||
#
|
||||
# All other settings (SMTP, webhook, ansible, plugins, retention)
|
||||
# are stored in the database and managed via the Settings UI at /settings/.
|
||||
@@ -11,7 +11,15 @@
|
||||
# config.yaml is optional. Use it only if you prefer file-based bootstrap.
|
||||
|
||||
database:
|
||||
url: "postgresql+asyncpg://roundtable:password@localhost/roundtable"
|
||||
url: "postgresql+asyncpg://steward:password@localhost/steward"
|
||||
|
||||
# Optional: seconds to wait at startup for the database to become reachable
|
||||
# before giving up (default: 60). Steward retries with backoff rather than
|
||||
# crashing when the DB is merely slow to arrive — replaying WAL after an
|
||||
# unclean shutdown, or container DNS not yet resolving after a host reboot.
|
||||
# Raise it if your database is routinely slower than this to accept
|
||||
# connections. Env var: STEWARD_DB_CONNECT_TIMEOUT
|
||||
# connect_timeout: 60
|
||||
|
||||
# Optional: override the auto-generated secret key.
|
||||
# If not set, a key is auto-generated on first run and saved to /data/secret.key.
|
||||
|
||||
+28
-11
@@ -1,17 +1,31 @@
|
||||
services:
|
||||
roundtable:
|
||||
steward:
|
||||
build: .
|
||||
container_name: roundtable
|
||||
image: roundtable:latest
|
||||
container_name: steward
|
||||
image: steward:latest
|
||||
# Dev: --debug turns on Quart's auto-reloader so edits to the mounted source
|
||||
# below take effect without a rebuild.
|
||||
command: ["steward", "--host", "0.0.0.0", "--port", "5000", "--debug"]
|
||||
ports:
|
||||
- "5000:5000"
|
||||
volumes:
|
||||
- app_data:/data
|
||||
- ./plugins:/app/plugins:ro
|
||||
- /mnt/Data/traefik/log:/var/log/traefik:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
# Live-edit core + first-party plugins without rebuilding. PYTHONPATH=/app
|
||||
# (below) makes this mounted source win over the image-installed package.
|
||||
- ./steward:/app/steward
|
||||
- ./plugins:/app/plugins
|
||||
# Dev playbooks: edit on the host under ./ansible-dev/, register a *local*
|
||||
# Ansible source pointing at /srv/ansible in Settings → Ansible.
|
||||
- ./ansible-dev:/srv/ansible
|
||||
# Optional host mounts — enable the matching plugin in Settings first, then
|
||||
# uncomment (the traefik path must exist on this host):
|
||||
# - /mnt/Data/traefik/log:/var/log/traefik:ro # traefik plugin
|
||||
# - /var/run/docker.sock:/var/run/docker.sock:ro # docker plugin
|
||||
environment:
|
||||
- ROUNDTABLE_DATABASE_URL=postgresql+asyncpg://fabledscryer:fabledscryer@db/fabledscryer
|
||||
- STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@db/steward
|
||||
# Dev-only fixed key so sessions survive restarts. Replace in production.
|
||||
- STEWARD_SECRET_KEY=dev-secret-key-change-me
|
||||
- PYTHONPATH=/app
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
@@ -20,13 +34,16 @@ services:
|
||||
db:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_USER: fabledscryer
|
||||
POSTGRES_PASSWORD: fabledscryer
|
||||
POSTGRES_DB: fabledscryer
|
||||
POSTGRES_USER: steward
|
||||
POSTGRES_PASSWORD: steward
|
||||
POSTGRES_DB: steward
|
||||
ports:
|
||||
# Exposed for dev convenience (psql from the host). Drop for prod.
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U fabledscryer"]
|
||||
test: ["CMD-SHELL", "pg_isready -U steward"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
+37
-13
@@ -1,12 +1,12 @@
|
||||
# Architecture
|
||||
|
||||
Roundtable is a single Quart (async Python) process. There are no separate worker processes, no message brokers, and no build pipeline for the frontend. All monitoring, scheduling, and request handling runs on a single asyncio event loop.
|
||||
Steward is a single Quart (async Python) process. There are no separate worker processes, no message brokers, and no build pipeline for the frontend. All monitoring, scheduling, and request handling runs on a single asyncio event loop.
|
||||
|
||||
---
|
||||
|
||||
## Startup Sequence
|
||||
|
||||
`create_app()` in `roundtable/app.py` runs these steps synchronously before the event loop starts:
|
||||
`create_app()` in `steward/app.py` runs these steps synchronously before the event loop starts:
|
||||
|
||||
| Step | What happens |
|
||||
|---|---|
|
||||
@@ -32,23 +32,47 @@ All routes are Quart Blueprints registered in `app.py`:
|
||||
|
||||
| Prefix | Blueprint | Module |
|
||||
|---|---|---|
|
||||
| `/auth/` | `auth_bp` | `roundtable/auth/routes.py` |
|
||||
| `/` | `dashboard_bp` | `roundtable/dashboard/routes.py` |
|
||||
| `/hosts/` | `hosts_bp` | `roundtable/hosts/routes.py` |
|
||||
| `/ping/` | `ping_bp` | `roundtable/ping/routes.py` |
|
||||
| `/dns/` | `dns_bp` | `roundtable/dns/routes.py` |
|
||||
| `/alerts/` | `alerts_bp` | `roundtable/alerts/routes.py` |
|
||||
| `/ansible/` | `ansible_bp` | `roundtable/ansible/routes.py` |
|
||||
| `/settings/` | `settings_bp` | `roundtable/settings/routes.py` |
|
||||
| `/auth/` | `auth_bp` | `steward/auth/routes.py` |
|
||||
| `/` | `dashboard_bp` | `steward/dashboard/routes.py` |
|
||||
| `/hosts/` | `hosts_bp` | `steward/hosts/routes.py` |
|
||||
| `/ping/` | `ping_bp` | `steward/ping/routes.py` |
|
||||
| `/dns/` | `dns_bp` | `steward/dns/routes.py` |
|
||||
| `/alerts/` | `alerts_bp` | `steward/alerts/routes.py` |
|
||||
| `/ansible/` | `ansible_bp` | `steward/ansible/routes.py` |
|
||||
| `/settings/` | `settings_bp` | `steward/settings/routes.py` |
|
||||
| `/plugins/<name>/` | plugin blueprint | `plugins/<name>/routes.py` |
|
||||
|
||||
Plugin blueprints are mounted automatically by `load_plugins()` using the plugin directory name as the URL prefix.
|
||||
|
||||
`/ai-access/` (`ai_access_bp`, `steward/ai_access/routes.py`) is where a signed-in user manages their API tokens.
|
||||
|
||||
### `/mcp`: outside the blueprint pipeline
|
||||
|
||||
`mount_mcp(app)` (`steward/mcp/server.py`) wraps `app.asgi_app`. A request to
|
||||
`/mcp` is handed to the MCP SDK's streamable-HTTP app before Quart sees it, so
|
||||
no blueprint, session cookie or `before_request` hook runs for it. Each
|
||||
request then goes through these steps:
|
||||
|
||||
1. **Database gate.** If the database is unreachable, the request gets a 503
|
||||
with a readable message.
|
||||
2. **Bearer resolution.** `api_tokens.resolve_token()` turns the token into a
|
||||
`Caller` (user, effective role, token id). A missing, revoked or expired
|
||||
token gets a 401.
|
||||
3. **Caller context.** The `Caller` and the app are set in ContextVars
|
||||
(`steward/mcp/_context.py`). Tools read them through `current_caller()` and
|
||||
`mcp_app()`, because Quart's `current_app` and `session` do not exist there.
|
||||
4. **Role gate.** `StewardMCPServer.refusal()` hides and refuses tools above
|
||||
the caller's role, and plugin tools whose plugin is disabled.
|
||||
|
||||
The SDK's session manager runs in Quart's `before_serving`/`after_serving`
|
||||
lifecycle. The server is stateless: no MCP session outlives a request. See
|
||||
[MCP server](core/mcp.md) for the security model and the tools.
|
||||
|
||||
---
|
||||
|
||||
## Scheduler
|
||||
|
||||
`roundtable/core/scheduler.py` exports two things:
|
||||
`steward/core/scheduler.py` exports two things:
|
||||
|
||||
- `ScheduledTask` dataclass — holds a `name`, `coro_factory` (zero-argument callable returning a coroutine), `interval_seconds`, and optional `run_on_startup` flag
|
||||
- `start_scheduler(tasks)` — async function that loops every second, calling `asyncio.create_task()` for each task whose interval has elapsed
|
||||
@@ -93,9 +117,9 @@ This means the only file you must touch to get the app running is the database U
|
||||
|
||||
No JavaScript framework, no build step. The frontend is:
|
||||
|
||||
- **Jinja2 templates** rendered server-side (`roundtable/templates/`)
|
||||
- **Jinja2 templates** rendered server-side (`steward/templates/`)
|
||||
- **HTMX** for live-updating fragments (dashboard widgets, ping pills, DNS status)
|
||||
- A single CSS design system in `roundtable/templates/base.html` using CSS custom properties
|
||||
- A single CSS design system in `steward/templates/base.html` using CSS custom properties
|
||||
|
||||
Live-updating widgets use HTMX polling:
|
||||
```html
|
||||
|
||||
@@ -8,13 +8,13 @@ Alert rules evaluate every metric that flows through `record_metric()`. There is
|
||||
|
||||
When any monitor or plugin calls `record_metric()`:
|
||||
|
||||
1. The metric value is written to the `plugin_metrics` table
|
||||
1. The metric value is written to the `plugin_metrics` table, and `plugin_metric_latest` moves to it if it is the newest sample of that metric
|
||||
2. All enabled `AlertRule` rows matching `(source_module, resource_name, metric_name)` are loaded
|
||||
3. Each matching rule is evaluated against the new value
|
||||
4. If a state transition occurs, an `AlertEvent` row is written
|
||||
5. Notification I/O is deferred outside the transaction via `asyncio.create_task()`
|
||||
|
||||
**Key function:** `roundtable/core/alerts.py` → `record_metric(session, source_module, resource_name, metric_name, value)`
|
||||
**Key function:** `steward/core/alerts.py` → `record_metric(session, source_module, resource_name, metric_name, value)`
|
||||
|
||||
`record_metric()` must always be called inside an active transaction:
|
||||
|
||||
@@ -130,9 +130,10 @@ Webhook is skipped if `webhook.url` is empty.
|
||||
|
||||
## Data Models
|
||||
|
||||
Defined in `roundtable/models/alerts.py`:
|
||||
Defined in `steward/models/alerts.py`:
|
||||
|
||||
- **`alert_rules`** — one row per configured rule
|
||||
- **`alert_states`** — one row per rule, tracks current state and consecutive failure count
|
||||
- **`alert_events`** — append-only log of all state transitions and notification outcomes
|
||||
- **`plugin_metrics`** — all metric values written by any monitor or plugin
|
||||
- **`plugin_metric_latest`** — the newest sample of each (source, resource, metric). Pages that show "what it is now" read this instead of searching history; retention drops a row once its sample leaves the raw window
|
||||
|
||||
+16
-9
@@ -1,6 +1,6 @@
|
||||
# Ansible Integration
|
||||
|
||||
Roundtable can browse, trigger, and stream output from Ansible playbooks directly from the web UI. Runs execute as asyncio tasks inside the same process — no Celery, no external workers.
|
||||
Steward can browse, trigger, and stream output from Ansible playbooks directly from the web UI. Runs execute as asyncio tasks inside the same process — no Celery, no external workers.
|
||||
|
||||
---
|
||||
|
||||
@@ -31,10 +31,9 @@ ansible.sources:
|
||||
url: "https://github.com/example/infra.git"
|
||||
branch: "main"
|
||||
pull_interval_seconds: 300
|
||||
cache_path: "/data/playbook_cache/infra-repo"
|
||||
```
|
||||
|
||||
Git sources register a `ScheduledTask` named `ansible_git_pull_<name>` that runs `git pull` on the configured interval.
|
||||
Each git source is checked out under `/data/ansible/sources/<name>` (override with `STEWARD_ANSIBLE_SOURCE_DIR`), on the persistent volume, so a redeploy keeps it. One scheduled task, `ansible_git_pulls`, ticks every 30 s and pulls each source once its `pull_interval_seconds` has passed; a source added in Settings is pulled within a tick, with no restart. A clone or pull that runs past 120 s is cut (its process group terminated), a clone that fails is removed so the next attempt starts clean, and git never prompts for credentials.
|
||||
|
||||
---
|
||||
|
||||
@@ -102,12 +101,20 @@ If a client connects after the run has already completed, the endpoint immediate
|
||||
|
||||
| Status | Description |
|
||||
|---|---|
|
||||
| `queued` | Waiting for a host another run holds, or for a run slot |
|
||||
| `running` | Execution in progress |
|
||||
| `success` | Playbook exited 0 |
|
||||
| `failed` | Playbook exited non-zero |
|
||||
| `interrupted` | App restarted while run was in progress |
|
||||
| `cancelled` | Stopped by an operator, or by Steward shutting down mid-run |
|
||||
|
||||
On startup, the app marks any runs still in `running` state as `interrupted` (see `_mark_interrupted_runs()` in `app.py`).
|
||||
On startup, the app marks any runs still `running` or `queued` as `interrupted` (see `_mark_interrupted_runs()` in `app.py`).
|
||||
|
||||
### Overlapping runs
|
||||
|
||||
- **One run per host.** Before it takes a run slot, a run claims every host it touches. Hosts come from the inventory Steward writes, keyed by address. A run on a source's own inventory file claims that file, because Steward doesn't read its hosts. While another run holds one of its hosts, the run shows `queued` and says which run it waits for. After an hour of waiting it fails and says so.
|
||||
- **One run at a time per schedule.** A schedule that falls due while its previous run is still `queued` or `running` skips that firing. The Schedules page shows "Skipped: the previous run was still going" beside that run. The next firing comes an interval later.
|
||||
- **One action run at a time per alert rule.** When a rule fires again while the action run from an earlier firing is still going, no new run starts. The firing's event says it was skipped and names the run.
|
||||
|
||||
Output stored in the DB is capped at 1 MB. If truncated, `[output truncated]` is appended to the DB column, but the live SSE stream continues unaffected.
|
||||
|
||||
@@ -115,7 +122,7 @@ Output stored in the DB is capped at 1 MB. If truncated, `[output truncated]` is
|
||||
|
||||
## Data Model
|
||||
|
||||
`ansible_runs` table (defined in `roundtable/models/ansible.py`):
|
||||
`ansible_runs` table (defined in `steward/models/ansible.py`):
|
||||
|
||||
| Column | Type | Description |
|
||||
|---|---|---|
|
||||
@@ -137,7 +144,7 @@ Runs older than `data.retention_days` (default 90) are pruned by the `data_clean
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `roundtable/ansible/sources.py` | Source discovery, git pull logic |
|
||||
| `roundtable/ansible/executor.py` | Subprocess execution and output streaming |
|
||||
| `roundtable/ansible/routes.py` | HTTP routes (browse, trigger, stream, history) |
|
||||
| `roundtable/models/ansible.py` | `AnsibleRun` model |
|
||||
| `steward/ansible/sources.py` | Source discovery, git pull logic |
|
||||
| `steward/ansible/executor.py` | Subprocess execution and output streaming |
|
||||
| `steward/ansible/routes.py` | HTTP routes (browse, trigger, stream, history) |
|
||||
| `steward/models/ansible.py` | `AnsibleRun` model |
|
||||
|
||||
+10
-12
@@ -1,6 +1,6 @@
|
||||
# Configuration
|
||||
|
||||
Roundtable uses a two-layer configuration system. Only the bare minimum needed to boot lives in files or environment variables. Everything else is stored in the database and managed through the Settings UI.
|
||||
Steward uses a two-layer configuration system. Only the bare minimum needed to boot lives in files or environment variables. Everything else is stored in the database and managed through the Settings UI.
|
||||
|
||||
---
|
||||
|
||||
@@ -10,31 +10,29 @@ These three values are read at startup from `config.yaml` and/or environment var
|
||||
|
||||
| Key | Env var | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `database.url` | `ROUNDTABLE_DATABASE_URL` | — | PostgreSQL async URL. **Required.** |
|
||||
| `secret_key` | `ROUNDTABLE_SECRET_KEY` | auto-generated | Flask/Quart session signing key. Auto-generated and saved to `/data/secret.key` if not set. |
|
||||
| `plugin_dir` | `ROUNDTABLE_PLUGIN_DIR` | `plugins` | Path to the plugins directory. |
|
||||
| `database.url` | `STEWARD_DATABASE_URL` | — | PostgreSQL async URL. **Required.** |
|
||||
| `secret_key` | `STEWARD_SECRET_KEY` | auto-generated | Flask/Quart session signing key. Auto-generated and saved to `/data/secret.key` if not set. |
|
||||
| `plugin_dir` | `STEWARD_PLUGIN_DIR` | `plugins` | Path to the plugins directory. |
|
||||
|
||||
**Resolution order for `database_url`:** env var `ROUNDTABLE_DATABASE_URL` → env var `ROUNDTABLE_DATABASE__URL` (legacy double-underscore) → `database.url` in `config.yaml`.
|
||||
**Resolution order for `database_url`:** env var `STEWARD_DATABASE_URL` → env var `STEWARD_DATABASE__URL` (legacy double-underscore) → `database.url` in `config.yaml`.
|
||||
|
||||
**Resolution order for `secret_key`:** env var `ROUNDTABLE_SECRET_KEY` → `secret_key` in `config.yaml` → `/data/secret.key` file → auto-generate and write to `/data/secret.key`.
|
||||
**Resolution order for `secret_key`:** env var `STEWARD_SECRET_KEY` → `secret_key` in `config.yaml` → `/data/secret.key` file → auto-generate and write to `/data/secret.key`.
|
||||
|
||||
### Minimal config.yaml
|
||||
|
||||
```yaml
|
||||
database:
|
||||
url: "postgresql+asyncpg://user:password@localhost/roundtable"
|
||||
url: "postgresql+asyncpg://user:password@localhost/steward"
|
||||
```
|
||||
|
||||
### Minimal env-only setup (Docker)
|
||||
|
||||
```bash
|
||||
ROUNDTABLE_DATABASE_URL=postgresql+asyncpg://user:password@db/roundtable
|
||||
STEWARD_DATABASE_URL=postgresql+asyncpg://user:password@db/steward
|
||||
```
|
||||
|
||||
A `.env` file is loaded automatically if present.
|
||||
|
||||
> **Legacy env vars:** `FABLEDSCRYER_*` env vars are still accepted as a fallback for existing deployments. They will be removed in a future release — migrate to `ROUNDTABLE_*` when convenient.
|
||||
|
||||
---
|
||||
|
||||
## App Settings (Database-backed)
|
||||
@@ -68,7 +66,7 @@ Default webhook template:
|
||||
### Reading and Writing Settings in Code
|
||||
|
||||
```python
|
||||
from roundtable.core.settings import get_setting, set_setting
|
||||
from steward.core.settings import get_setting, set_setting
|
||||
|
||||
# Read
|
||||
async with current_app.db_sessionmaker() as session:
|
||||
@@ -84,7 +82,7 @@ async with current_app.db_sessionmaker() as session:
|
||||
|
||||
### The DEFAULTS Dict
|
||||
|
||||
`roundtable/core/settings.py` contains the `DEFAULTS` dict — the canonical list of all recognised settings and their default values. Add new settings here to make them recognised by `get_all_settings()` and the settings UI.
|
||||
`steward/core/settings.py` contains the `DEFAULTS` dict — the canonical list of all recognised settings and their default values. Add new settings here to make them recognised by `get_all_settings()` and the settings UI.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
# MCP server
|
||||
|
||||
Steward serves the [Model Context Protocol](https://modelcontextprotocol.io) at
|
||||
`/mcp`. An AI client connected to it can look into everything Steward watches
|
||||
(hosts, monitors, alerts, metrics, containers, playbook runs) and, with a
|
||||
high enough role, change it. A token's role sets that ceiling.
|
||||
|
||||
**Source:** `steward/mcp/`. The tools live in `steward/mcp/tools/`, and a
|
||||
plugin's tools live in `plugins/<name>/mcp_tools.py`.
|
||||
**Transport:** streamable HTTP, stateless. Every request carries its own
|
||||
token, so a client survives a Steward restart without reconnecting.
|
||||
|
||||
---
|
||||
|
||||
## Connecting
|
||||
|
||||
1. In Steward, open **AI access** and create a token. Pick the lowest role
|
||||
that does the job.
|
||||
2. Copy the token (it is shown once) and use one of the recipes on that
|
||||
page. They carry the real endpoint URL and the new token. For Claude Code:
|
||||
|
||||
```sh
|
||||
claude mcp add --transport http steward https://steward.example/mcp \
|
||||
--header "Authorization: Bearer stw_…"
|
||||
```
|
||||
|
||||
3. Ask the client to call `whoami`. It answers with the user, the effective
|
||||
role and what that role allows.
|
||||
|
||||
Claude Desktop connects through `mcp-remote`. Open WebUI and other clients
|
||||
need an MCP server with the **Streamable HTTP** transport and bearer
|
||||
authentication. The AI access page shows the recipes for both.
|
||||
|
||||
### Behind a reverse proxy
|
||||
|
||||
- The URL on the AI access page is **Settings → General → Public base URL**
|
||||
plus `/mcp`. If that setting is empty, the URL is built from the request's
|
||||
host instead.
|
||||
- The proxy must pass the `Authorization` header through unchanged.
|
||||
- It must not buffer responses. Replies may arrive as server-sent events
|
||||
(`text/event-stream`). For nginx, set `proxy_buffering off;` on the
|
||||
location.
|
||||
- Steward does no Host-header (DNS-rebinding) check on `/mcp`, because the
|
||||
bearer token is the security boundary. Serve it over HTTPS.
|
||||
|
||||
---
|
||||
|
||||
## Security model
|
||||
|
||||
| Rule | How it is enforced |
|
||||
|---|---|
|
||||
| Bearer tokens only, no OAuth | `steward/core/api_tokens.py`. Only a hash is stored, and the token is shown once. |
|
||||
| A token never acts above its user | Effective role = min(token role, user's **current** role). Lowering a user lowers every token they hold; disabling the user stops them all. |
|
||||
| Default deny | A tool is reachable only if it was registered with a role (`register_tool(fn, role=..., read_only=...)`). `tools/list` shows a caller only the tools their role allows. A call above the role is refused, and the refusal names the role it needs. |
|
||||
| Plugin tools follow the plugin | A disabled plugin's tools are hidden and refused. |
|
||||
| Dry run first | `run_playbook`, `create_schedule` and `set_alert_action` default to Ansible check mode (`--check --diff`). Changes are applied only when the caller passes `check=false`. A playbook its author flagged `# steward:confirm: true` also needs `confirmed=true`. `prune_docker` previews unless `check=false`. |
|
||||
| Secrets stay put | Playbook variables Steward detects as secret reach the run but are never stored, audited or returned. Schedules and alert actions refuse them, since they would have to store them. Inventory variables are listed by name only, and HTTP monitor header values are redacted. |
|
||||
| Every change is audited | Each write records an audit event as `<user> via MCP`, with the token id in its detail. `list_audit_events(user="<user>")` includes those events. |
|
||||
| Unknown arguments are refused | A misspelt argument is rejected with a "did you mean", rather than silently dropped. |
|
||||
|
||||
Tokens never mint or revoke tokens; that happens only on the AI access
|
||||
page, in a browser session.
|
||||
|
||||
---
|
||||
|
||||
## Tools by role
|
||||
|
||||
Each role includes the tools of the roles above it in this list.
|
||||
|
||||
### Viewer: read everything
|
||||
|
||||
| Area | Tools |
|
||||
|---|---|
|
||||
| Start here | `whoami`, `get_overview` (what is wrong now, ranked, each finding with the call that shows its evidence) |
|
||||
| Hosts and metrics | `list_hosts`, `get_host`, `get_host_metrics`, `query_metrics` |
|
||||
| Monitors | `list_monitors`, `get_monitor` |
|
||||
| Alerts | `list_alerts`, `get_alert_history`, `list_alert_rules`, `get_alert_rule_options`, `list_maintenance_windows` |
|
||||
| Automation | `list_playbooks`, `get_playbook`, `list_inventory`, `list_ansible_runs`, `get_ansible_run`, `wait_for_run`, `list_schedules` |
|
||||
| Audit | `list_audit_events` |
|
||||
| Docker plugin | `list_containers`, `get_container`, `list_swarm_services`, `search_container_logs`, `get_docker_disk_usage` |
|
||||
| Host agent plugin | `list_agents` |
|
||||
|
||||
### Operator: run playbooks, change monitoring
|
||||
|
||||
| Area | Tools |
|
||||
|---|---|
|
||||
| Automation | `run_playbook`, `cancel_ansible_run`, `create_schedule`, `update_schedule`, `delete_schedule` |
|
||||
| Monitors | `create_monitor`, `update_monitor`, `set_monitor_enabled` |
|
||||
| Alerts | `create_alert_rule`, `update_alert_rule`, `acknowledge_alert`, `create_maintenance_window`, `end_maintenance_window` |
|
||||
|
||||
### Admin: destructive and remediation
|
||||
|
||||
`set_alert_action` (the playbook a rule runs when it fires), `delete_monitor`,
|
||||
`delete_alert_rule`, `prune_docker`.
|
||||
|
||||
The client sees each tool's full contract: arguments, what an empty value
|
||||
means, and what comes back. That contract is the tool's docstring, so read the
|
||||
docstrings in `steward/mcp/tools/` for the detail.
|
||||
|
||||
---
|
||||
|
||||
## Response conventions
|
||||
|
||||
Every tool follows these conventions (`steward/mcp/tools/_shape.py`):
|
||||
|
||||
- **Times** are ISO-8601 UTC with a `Z`. Anything with a "when" also carries
|
||||
`age_s`, so the client never does clock arithmetic.
|
||||
- **Lists are bounded.** A list response carries `total` and `truncated`. When
|
||||
it was cut, `more` names the argument that fetches the rest. Nothing is cut
|
||||
silently.
|
||||
- **Time series** come back as a summary (min/avg/max/p95/last) plus a series
|
||||
of at most 60 points, never raw rows.
|
||||
- **Lookup by name or id.** A miss suggests close matches and names the tool
|
||||
that lists the valid choices.
|
||||
- **Refusals are readable.** Bad input becomes a tool error that says what was
|
||||
wrong, why, and what to do instead, and nothing is changed. The tools share
|
||||
their validation with the web forms (`monitors/service.py`,
|
||||
`alerts/service.py`), so a value the UI refuses is refused here for the same
|
||||
reason.
|
||||
- **Database outages** return a plain "Steward cannot reach its database"
|
||||
rather than a stack trace.
|
||||
|
||||
---
|
||||
|
||||
## Adding tools
|
||||
|
||||
Core tools: add an `async def` to a module in `steward/mcp/tools/` and list it
|
||||
in that module's `register()` with its minimum role and `read_only`. A plugin
|
||||
exposes `register_mcp_tools(register)`, and optionally `mcp_overview(db)` to
|
||||
contribute findings to `get_overview`. See
|
||||
[Writing a plugin](../plugins/writing-a-plugin.md). A tool that changes
|
||||
anything calls `audit_mcp(...)` from `steward/mcp/tools/_audit.py`.
|
||||
@@ -1,13 +1,13 @@
|
||||
# Core Monitors
|
||||
|
||||
Roundtable ships two built-in monitors: Ping and DNS. Both run as asyncio scheduled tasks on the same event loop as the web server, with no separate processes.
|
||||
Steward ships two built-in monitors: Ping and DNS. Both run as asyncio scheduled tasks on the same event loop as the web server, with no separate processes.
|
||||
|
||||
---
|
||||
|
||||
## Ping Monitor
|
||||
|
||||
**Source:** `roundtable/monitors/ping.py`
|
||||
**Scheduler task:** `ping_monitor` in `roundtable/app.py`
|
||||
**Source:** `steward/monitors/ping.py`
|
||||
**Scheduler task:** `ping_monitor` in `steward/app.py`
|
||||
**Interval:** `monitors.poll_interval_seconds` (default 60s), runs on startup
|
||||
|
||||
### How It Works
|
||||
@@ -29,7 +29,7 @@ Each probe writes a `PingResult` row and calls `record_metric()`:
|
||||
|
||||
### Data Model
|
||||
|
||||
`ping_results` table (defined in `roundtable/models/monitors.py`):
|
||||
`ping_results` table (defined in `steward/models/monitors.py`):
|
||||
|
||||
| Column | Type | Description |
|
||||
|---|---|---|
|
||||
@@ -51,8 +51,8 @@ Old rows are pruned by the `data_cleanup` task (default: 90 days).
|
||||
|
||||
## DNS Monitor
|
||||
|
||||
**Source:** `roundtable/monitors/dns.py`
|
||||
**Scheduler task:** `dns_monitor` in `roundtable/app.py`
|
||||
**Source:** `steward/monitors/dns.py`
|
||||
**Scheduler task:** `dns_monitor` in `steward/app.py`
|
||||
**Interval:** `monitors.poll_interval_seconds` (default 60s), runs on startup
|
||||
|
||||
### How It Works
|
||||
@@ -70,7 +70,7 @@ Each check writes a `DnsResult` row and calls `record_metric()`:
|
||||
|
||||
### Data Model
|
||||
|
||||
`dns_results` table (defined in `roundtable/models/monitors.py`):
|
||||
`dns_results` table (defined in `steward/models/monitors.py`):
|
||||
|
||||
| Column | Type | Description |
|
||||
|---|---|---|
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
|
||||
**Date:** 2026-04-14
|
||||
**Status:** Approved, ready for implementation planning
|
||||
**Scope:** Roundtable plugin for remote host resource monitoring (CPU, memory, storage, load, uptime) via a lightweight Python push agent.
|
||||
**Scope:** Steward plugin for remote host resource monitoring (CPU, memory, storage, load, uptime) via a lightweight Python push agent.
|
||||
|
||||
---
|
||||
|
||||
## Goal
|
||||
|
||||
Give Roundtable a fleet-glance view of remote Linux hosts — current resource usage, history, and alert integration — without requiring agentless SSH/Ansible round-trips, a new toolchain, or SNMP gymnastics on every target.
|
||||
Give Steward a fleet-glance view of remote Linux hosts — current resource usage, history, and alert integration — without requiring agentless SSH/Ansible round-trips, a new toolchain, or SNMP gymnastics on every target.
|
||||
|
||||
## Non-goals
|
||||
|
||||
@@ -24,7 +24,7 @@ Give Roundtable a fleet-glance view of remote Linux hosts — current resource u
|
||||
|
||||
```
|
||||
┌──────────────┐ POST /plugins/host_agent/ingest ┌────────────────────────┐
|
||||
│ Agent │ ──────────────────────────────────────────→ │ Roundtable │
|
||||
│ Agent │ ──────────────────────────────────────────→ │ Steward │
|
||||
│ (Python) │ Authorization: Bearer <per-host-token> │ host_agent plugin │
|
||||
│ on target │ JSON body: metrics snapshot │ │
|
||||
│ host │ │ routes.py (ingest + │
|
||||
@@ -48,8 +48,8 @@ Give Roundtable a fleet-glance view of remote Linux hosts — current resource u
|
||||
- **Agent** is a single Python file. No external dependencies beyond Python 3.8+ stdlib.
|
||||
- **Plugin** is self-contained under `plugins/host_agent/`. Writes to the core `PluginMetric` time-series bus (the designed cross-plugin integration point) and its own private `host_agent_registrations` table. Writes to the core `Host` model for identity, but adds no new columns to it.
|
||||
- **Auth** is per-host bearer tokens, minted on "Add host" in the plugin's settings page.
|
||||
- **Install** is a one-line `curl | sh` command rendered per-host by Roundtable with the token already baked in.
|
||||
- **Failure** is handled at the agent (in-memory ring buffer + exponential backoff) so Roundtable outages don't lose brief-window data.
|
||||
- **Install** is a one-line `curl | sh` command rendered per-host by Steward with the token already baked in.
|
||||
- **Failure** is handled at the agent (in-memory ring buffer + exponential backoff) so Steward outages don't lose brief-window data.
|
||||
|
||||
---
|
||||
|
||||
@@ -58,10 +58,10 @@ Give Roundtable a fleet-glance view of remote Linux hosts — current resource u
|
||||
### File layout on the target host
|
||||
|
||||
```
|
||||
/usr/local/lib/roundtable-agent/agent.py # the script, target ~300 lines
|
||||
/etc/roundtable-agent.conf # key=value config, 0640 root:roundtable-agent
|
||||
/etc/systemd/system/roundtable-agent.service # unit file
|
||||
# dedicated system user: roundtable-agent
|
||||
/usr/local/lib/steward-agent/agent.py # the script, target ~300 lines
|
||||
/etc/steward-agent.conf # key=value config, 0640 root:steward-agent
|
||||
/etc/systemd/system/steward-agent.service # unit file
|
||||
# dedicated system user: steward-agent
|
||||
```
|
||||
|
||||
### Config file format
|
||||
@@ -69,13 +69,20 @@ Give Roundtable a fleet-glance view of remote Linux hosts — current resource u
|
||||
Flat `key = value`, parsed by a ~20-line homegrown parser (no TOML/YAML dependency):
|
||||
|
||||
```
|
||||
url = https://roundtable.home.lan
|
||||
url = https://steward.home.lan
|
||||
token = a1b2c3d4...
|
||||
interval_seconds = 30
|
||||
hostname = myhost # optional; defaults to uname -n
|
||||
mounts = /, /mnt/data # optional; defaults to all real mounts (excluding tmpfs/devtmpfs/etc.)
|
||||
docker_logs_enabled = true # optional; on by default — per-host container-log kill-switch
|
||||
docker_log_exclude = watchtower, noisy-svc # optional; container names whose logs this host skips collecting
|
||||
```
|
||||
|
||||
Container-log collection is on by default (no config needed). The two keys above
|
||||
are the per-host opt-outs; the operator-facing global toggle + exclude live in
|
||||
Settings and are enforced server-side (the push model has no channel to command
|
||||
an agent), so they don't require touching a host's conf.
|
||||
|
||||
### Agent internals (function list, not classes)
|
||||
|
||||
- `read_config(path)` — parses the conf file into a dict.
|
||||
@@ -85,10 +92,17 @@ mounts = /, /mnt/data # optional; defaults to all real mounts (excludin
|
||||
- `collect_load()` — reads `/proc/loadavg`, returns `[1m, 5m, 15m]`.
|
||||
- `collect_uptime()` — reads `/proc/uptime`, returns seconds since boot (int).
|
||||
- `collect_metadata()` — `os.uname()` for kernel + arch, `/etc/os-release` for distro. Called once at startup and cached.
|
||||
- `collect_docker_logs(socket, containers, state, exclude)` — (m79) per running
|
||||
container, fetches new log lines over the Docker socket with an incremental
|
||||
`since` cursor kept per container in `state` (a container's first interval
|
||||
seeds from a short tail). Demuxes the multiplexed stream, parses each line's
|
||||
RFC3339 timestamp, caps the whole batch at a byte limit (a marker line records
|
||||
a truncation; the deferred lines come next interval). Folded into the sample as
|
||||
`docker_logs`; omitted when empty.
|
||||
- `build_payload()` — assembles a snapshot from all collectors into one dict.
|
||||
- `post_payload(url, token, payloads)` — POSTs a list of samples, returns success/failure.
|
||||
- `RingBuffer(maxlen=20)` — tiny FIFO wrapper, drops oldest when full.
|
||||
- `main_loop()` — the 30s loop: collect → try POST → on failure push to buffer + backoff → on success flush buffer.
|
||||
- `main_loop()` — the 30s loop: collect → try POST → on failure push to buffer + backoff → on success flush buffer. Container logs are stripped from a sample before it's buffered (metrics survive an outage; stale logs are dropped).
|
||||
|
||||
**Target: ~300 lines total including docstrings.** More than that is a smell that the agent is over-scoping.
|
||||
|
||||
@@ -111,7 +125,7 @@ To stderr only (systemd captures to journal). No file logging, no log rotation.
|
||||
|
||||
### Identity
|
||||
|
||||
The agent's self-reported hostname in the payload is advisory. The real identity is the bearer token — Roundtable looks up the `Host` row via `HostAgentRegistration.token_hash`, not via hostname. Changing a host's hostname does not break its identity, and two hosts accidentally sharing a hostname can't collide.
|
||||
The agent's self-reported hostname in the payload is advisory. The real identity is the bearer token — Steward looks up the `Host` row via `HostAgentRegistration.token_hash`, not via hostname. Changing a host's hostname does not break its identity, and two hosts accidentally sharing a hostname can't collide.
|
||||
|
||||
### Failure behavior
|
||||
|
||||
@@ -194,7 +208,7 @@ One `ScheduledTask` running every 60 seconds that flags `HostAgentRegistration`
|
||||
|
||||
### `METRIC_CATALOG` registration
|
||||
|
||||
Add to `roundtable/alerts/routes.py`:
|
||||
Add to `steward/alerts/routes.py`:
|
||||
|
||||
```python
|
||||
"host_agent": ["cpu_pct", "mem_used_pct", "mem_available_bytes",
|
||||
@@ -202,7 +216,7 @@ Add to `roundtable/alerts/routes.py`:
|
||||
"load_5m", "load_15m", "uptime_secs"],
|
||||
```
|
||||
|
||||
This is the one edit outside the plugin directory, matching the pattern every other plugin already follows. Not ideal from a pure-plugin-independence standpoint but unavoidable until Roundtable core grows a plugin-registered catalog API — deferred as future core work.
|
||||
This is the one edit outside the plugin directory, matching the pattern every other plugin already follows. Not ideal from a pure-plugin-independence standpoint but unavoidable until Steward core grows a plugin-registered catalog API — deferred as future core work.
|
||||
|
||||
---
|
||||
|
||||
@@ -212,7 +226,7 @@ This is the one edit outside the plugin directory, matching the pattern every ot
|
||||
|
||||
```http
|
||||
POST /plugins/host_agent/ingest HTTP/1.1
|
||||
Host: roundtable.home.lan
|
||||
Host: steward.home.lan
|
||||
Authorization: Bearer a1b2c3d4e5f6...
|
||||
Content-Type: application/json
|
||||
```
|
||||
@@ -255,6 +269,7 @@ Content-Type: application/json
|
||||
- **`metadata` is sent on every POST**, not just on change. Server-side diff detects actual changes and only writes on change. Cost per POST is one dict — negligible. Benefit: server can cleanly detect agent restarts.
|
||||
- **Raw bytes, not percentages, for memory and storage.** Percentages are derived server-side. Changing the "what counts as used" math doesn't require re-releasing the agent.
|
||||
- **CPU is the one exception** — reported as a percentage because it's inherently a derivative (delta over time), not a snapshot. The agent must sample twice to compute it.
|
||||
- **Container logs (m79) ride in the same push** as `docker_logs`: a list of `{container, stream, ts, line}` records — incremental since the previous interval. This pushes the *same direction* as metrics, so batched log history needs no inbound channel (only sub-second live-follow would). The server ingests them into `docker_logs`, enforces the global toggle + exclude list on ingest, and bounds storage with a per-container size+age ring. `docker_logs` is omitted when there's nothing new.
|
||||
|
||||
### Server expansion into `PluginMetric` rows
|
||||
|
||||
@@ -311,7 +326,7 @@ Agent treats anything non-2xx as failure → ring buffer. Agent treats 401 speci
|
||||
### The one-liner (what the UI shows)
|
||||
|
||||
```
|
||||
curl -sSL 'https://roundtable.home.lan/plugins/host_agent/install.sh?token=a1b2c3...' | sudo sh
|
||||
curl -sSL 'https://steward.home.lan/plugins/host_agent/install.sh?token=a1b2c3...' | sudo sh
|
||||
```
|
||||
|
||||
The UI also offers a "review script before running" link that opens a modal showing the two-step form:
|
||||
@@ -326,19 +341,19 @@ sudo sh install.sh
|
||||
|
||||
```sh
|
||||
#!/bin/sh
|
||||
# Roundtable host agent installer
|
||||
# Steward host agent installer
|
||||
# Generated for: {{ host_name }} ({{ host_address }})
|
||||
# Roundtable URL: {{ url }}
|
||||
# Steward URL: {{ url }}
|
||||
set -e
|
||||
|
||||
ROUNDTABLE_URL="{{ url }}"
|
||||
STEWARD_URL="{{ url }}"
|
||||
AGENT_TOKEN="{{ token }}"
|
||||
AGENT_VERSION="{{ agent_version }}"
|
||||
|
||||
AGENT_USER="roundtable-agent"
|
||||
AGENT_DIR="/usr/local/lib/roundtable-agent"
|
||||
CONF_FILE="/etc/roundtable-agent.conf"
|
||||
UNIT_FILE="/etc/systemd/system/roundtable-agent.service"
|
||||
AGENT_USER="steward-agent"
|
||||
AGENT_DIR="/usr/local/lib/steward-agent"
|
||||
CONF_FILE="/etc/steward-agent.conf"
|
||||
UNIT_FILE="/etc/systemd/system/steward-agent.service"
|
||||
|
||||
# ── preflight ────────────────────────────────────────────────────────────────
|
||||
[ "$(id -u)" = "0" ] || { echo "must run as root (use sudo)"; exit 1; }
|
||||
@@ -347,7 +362,7 @@ command -v python3 >/dev/null 2>&1 || { echo "python3 not found — install py
|
||||
|
||||
# Handle --uninstall
|
||||
if [ "${1:-}" = "--uninstall" ]; then
|
||||
systemctl disable --now roundtable-agent.service 2>/dev/null || true
|
||||
systemctl disable --now steward-agent.service 2>/dev/null || true
|
||||
rm -f "$UNIT_FILE" "$CONF_FILE"
|
||||
rm -rf "$AGENT_DIR"
|
||||
systemctl daemon-reload
|
||||
@@ -363,13 +378,13 @@ fi
|
||||
|
||||
# ── drop the agent file ──────────────────────────────────────────────────────
|
||||
mkdir -p "$AGENT_DIR"
|
||||
curl -sSL "${ROUNDTABLE_URL}/plugins/host_agent/agent.py" -o "$AGENT_DIR/agent.py"
|
||||
curl -sSL "${STEWARD_URL}/plugins/host_agent/agent.py" -o "$AGENT_DIR/agent.py"
|
||||
chmod 0755 "$AGENT_DIR/agent.py"
|
||||
chown root:root "$AGENT_DIR/agent.py"
|
||||
|
||||
# ── write config ─────────────────────────────────────────────────────────────
|
||||
cat > "$CONF_FILE" <<EOF
|
||||
url = $ROUNDTABLE_URL
|
||||
url = $STEWARD_URL
|
||||
token = $AGENT_TOKEN
|
||||
interval_seconds = 30
|
||||
EOF
|
||||
@@ -379,7 +394,7 @@ chmod 0640 "$CONF_FILE"
|
||||
# ── write systemd unit ───────────────────────────────────────────────────────
|
||||
cat > "$UNIT_FILE" <<EOF
|
||||
[Unit]
|
||||
Description=Roundtable host agent
|
||||
Description=Steward host agent
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
@@ -400,17 +415,17 @@ WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now roundtable-agent.service
|
||||
systemctl enable --now steward-agent.service
|
||||
|
||||
echo
|
||||
echo "Roundtable host agent $AGENT_VERSION installed and running."
|
||||
echo "Check status: systemctl status roundtable-agent"
|
||||
echo "Logs: journalctl -u roundtable-agent -f"
|
||||
echo "Steward host agent $AGENT_VERSION installed and running."
|
||||
echo "Check status: systemctl status steward-agent"
|
||||
echo "Logs: journalctl -u steward-agent -f"
|
||||
```
|
||||
|
||||
### Design points
|
||||
|
||||
- **Agent binary served by Roundtable itself.** `GET /plugins/host_agent/agent.py` serves the script bundled with the plugin, so version drift between install-script and agent is impossible — re-running the installer picks up whatever agent the currently-running Roundtable ships.
|
||||
- **Agent binary served by Steward itself.** `GET /plugins/host_agent/agent.py` serves the script bundled with the plugin, so version drift between install-script and agent is impossible — re-running the installer picks up whatever agent the currently-running Steward ships.
|
||||
- **Systemd hardening is cheap and correct.** `NoNewPrivileges`, `ProtectSystem=strict`, `ProtectHome`, `PrivateTmp`, `ReadOnlyPaths=/proc /sys`. The agent only needs read access to `/proc`, `/sys`, and mount points; denying everything else narrows blast radius.
|
||||
- **Uninstall is a first-class flag**, not a separate script. Same one-liner with `--uninstall` appended.
|
||||
- **Fail-fast on preflight.** Missing systemd or python3 → clear error, exit. No half-installed agent.
|
||||
@@ -421,7 +436,7 @@ echo "Logs: journalctl -u roundtable-agent -f"
|
||||
|
||||
## UI surfaces
|
||||
|
||||
### Dashboard widgets (`roundtable/core/widgets.py`)
|
||||
### Dashboard widgets (`steward/core/widgets.py`)
|
||||
|
||||
- **`host_resources`** — table widget. One row per monitored host: name, CPU %, mem %, disk % (worst mount), load 1m, "last seen Xs ago" with red/yellow/green coloring. Fleet glance.
|
||||
- **`host_resource_history`** — chart widget for one host. CPU / mem / disk over a selectable time range (1h, 6h, 24h, 7d). Same pattern as every other history widget — Chart.js.
|
||||
@@ -447,9 +462,9 @@ List of registered hosts with their enable flags, an "Add host" button that open
|
||||
|
||||
| Failure | Who handles it | Response |
|
||||
|---|---|---|
|
||||
| Agent can't reach Roundtable | Agent | Push to ring buffer, exponential backoff (30→60→120→300s cap), log WARN. Resets on success. |
|
||||
| Roundtable rejects with 401 | Agent | Log ERROR with remediation hint, continue retry loop. Admin fixes via UI + conf edit; no restart needed. |
|
||||
| Roundtable rejects with 400 | Agent | Log ERROR, drop the sample (don't re-buffer), continue. Indicates agent/server version skew. |
|
||||
| Agent can't reach Steward | Agent | Push to ring buffer, exponential backoff (30→60→120→300s cap), log WARN. Resets on success. |
|
||||
| Steward rejects with 401 | Agent | Log ERROR with remediation hint, continue retry loop. Admin fixes via UI + conf edit; no restart needed. |
|
||||
| Steward rejects with 400 | Agent | Log ERROR, drop the sample (don't re-buffer), continue. Indicates agent/server version skew. |
|
||||
| Ring buffer full | Agent | Drop oldest, keep newest. DEBUG log (expected during outages). |
|
||||
| Config file missing or malformed | Agent | Log ERROR, exit non-zero. Systemd restarts after 10s. Repeated restarts are visible in journal. |
|
||||
| `/proc/stat` read fails between samples | Agent | Skip CPU for this cycle, still POST the rest. Partial samples allowed. |
|
||||
@@ -528,7 +543,7 @@ These are not blockers; they are feature boundaries inherent to the design.
|
||||
6. Settings page: list, add-host flow, rotate-token, delete.
|
||||
7. Dashboard widgets: table + history chart. Register in `core/widgets.py`.
|
||||
8. Per-host detail page.
|
||||
9. `METRIC_CATALOG` entry in `roundtable/alerts/routes.py`.
|
||||
9. `METRIC_CATALOG` entry in `steward/alerts/routes.py`.
|
||||
10. Scheduler: stale-agent marker.
|
||||
11. Tests at every stage; final integration test to tie it together.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Ship a `host_agent` Roundtable plugin plus a stdlib-only Python push agent that reports CPU/memory/storage/load/uptime from remote Linux hosts to Roundtable.
|
||||
**Goal:** Ship a `host_agent` Steward plugin plus a stdlib-only Python push agent that reports CPU/memory/storage/load/uptime from remote Linux hosts to Steward.
|
||||
|
||||
**Architecture:** Plugin lives under `plugins/host_agent/` (mirrors `plugins/http/`). It owns a private `host_agent_registrations` table, writes time-series data into the core `PluginMetric` bus, serves its own agent source at `GET /plugins/host_agent/agent.py`, and renders a per-host install script at `GET /plugins/host_agent/install.sh`. The agent is a single ~300-line Python script with a 30s collect→POST loop, in-memory ring buffer, exponential backoff, and systemd unit installed by a curl one-liner.
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
## Execution note — path 1 (no DB-backed tests)
|
||||
|
||||
Mid-execution discovery: the Roundtable test harness runs `create_app(testing=True)`, which mocks `db_sessionmaker` and skips all migrations. There is no existing DB-backed test fixture in this codebase, and no existing plugin ships with tests. The first execution attempt tried to invent a SQLite-backed override fixture, which cascaded into modifying a production migration (`0007_dashboard_ownership.py`) to remove an inline FK — unacceptable. That commit was reverted.
|
||||
Mid-execution discovery: the Steward test harness runs `create_app(testing=True)`, which mocks `db_sessionmaker` and skips all migrations. There is no existing DB-backed test fixture in this codebase, and no existing plugin ships with tests. The first execution attempt tried to invent a SQLite-backed override fixture, which cascaded into modifying a production migration (`0007_dashboard_ownership.py`) to remove an inline FK — unacceptable. That commit was reverted.
|
||||
|
||||
**Revised test strategy:** test the agent exhaustively (everything that lives in `plugins/host_agent/agent.py` — pure Python, stdlib only, trivially unit-testable). Test the server-side metric-expansion function as a pure function that takes a sample dict + host name and returns a list of `(metric_name, resource_name, value)` tuples. Everything else — ingest route, install route, settings routes, widgets, detail page, scheduler — is verified manually against the dev server.
|
||||
|
||||
@@ -43,7 +43,7 @@ Where code blocks in tasks below reference DB-backed tests, treat them as guidan
|
||||
- `plugins/host_agent/scheduler.py` — stale-agent marker task.
|
||||
- `plugins/host_agent/agent.py` — the Python agent script, served to targets.
|
||||
- `plugins/host_agent/migrations/__init__.py`
|
||||
- `plugins/host_agent/migrations/env.py` — alembic env (copy of http plugin's, but imports `roundtable.models.base`).
|
||||
- `plugins/host_agent/migrations/env.py` — alembic env (copy of http plugin's, but imports `steward.models.base`).
|
||||
- `plugins/host_agent/migrations/versions/__init__.py`
|
||||
- `plugins/host_agent/migrations/versions/host_agent_001_initial.py` — creates `host_agent_registrations`.
|
||||
- `plugins/host_agent/templates/install.sh.j2` — Jinja install script template.
|
||||
@@ -68,8 +68,8 @@ Where code blocks in tasks below reference DB-backed tests, treat them as guidan
|
||||
|
||||
**Modified files (core, small edits):**
|
||||
|
||||
- `roundtable/alerts/routes.py` — add `"host_agent"` entry to `METRIC_CATALOG`.
|
||||
- `roundtable/core/widgets.py` — add `host_resources` and `host_resource_history` entries.
|
||||
- `steward/alerts/routes.py` — add `"host_agent"` entry to `METRIC_CATALOG`.
|
||||
- `steward/core/widgets.py` — add `host_resources` and `host_resource_history` entries.
|
||||
- `docs/plugins/index.yaml.example` — add catalog entry.
|
||||
|
||||
---
|
||||
@@ -102,7 +102,7 @@ pytestmark = pytest.mark.asyncio
|
||||
|
||||
async def test_host_agent_migration_creates_table(app):
|
||||
# The app fixture runs all plugin migrations on startup.
|
||||
from roundtable.core.db import get_engine
|
||||
from steward.core.db import get_engine
|
||||
engine = get_engine()
|
||||
async with engine.connect() as conn:
|
||||
def _check(sync_conn):
|
||||
@@ -127,11 +127,11 @@ Expected: FAIL (plugin not loaded, table missing).
|
||||
name: host_agent
|
||||
version: "1.0.0"
|
||||
description: "Remote Linux host resource monitoring via a lightweight Python push agent (CPU, memory, storage, load, uptime)"
|
||||
author: "Roundtable"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/src/branch/main/host_agent"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/host_agent"
|
||||
tags:
|
||||
- host
|
||||
- monitoring
|
||||
@@ -180,7 +180,7 @@ from __future__ import annotations
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy import Column, String, DateTime, ForeignKey
|
||||
from roundtable.models.base import Base
|
||||
from steward.models.base import Base
|
||||
|
||||
|
||||
def _uuid() -> str:
|
||||
@@ -269,8 +269,8 @@ from alembic import context
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
|
||||
|
||||
from roundtable.models.base import Base
|
||||
import roundtable.models # noqa: F401
|
||||
from steward.models.base import Base
|
||||
import steward.models # noqa: F401
|
||||
from plugins.host_agent.models import HostAgentRegistration # noqa: F401
|
||||
|
||||
config = context.config
|
||||
@@ -282,14 +282,14 @@ target_metadata = Base.metadata
|
||||
|
||||
def _get_url() -> str:
|
||||
import yaml
|
||||
cfg_path = os.environ.get("ROUNDTABLE_CONFIG", "config.yaml")
|
||||
cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml")
|
||||
try:
|
||||
with open(cfg_path) as f:
|
||||
cfg = yaml.safe_load(f) or {}
|
||||
url = cfg.get("database", {}).get("url", "")
|
||||
except FileNotFoundError:
|
||||
url = ""
|
||||
return os.environ.get("ROUNDTABLE_DATABASE__URL", url)
|
||||
return os.environ.get("STEWARD_DATABASE__URL", url)
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
@@ -413,12 +413,12 @@ from plugins.host_agent.agent import read_config, ConfigError
|
||||
def test_parses_flat_key_value(tmp_path):
|
||||
p = tmp_path / "agent.conf"
|
||||
p.write_text(
|
||||
"url = https://roundtable.example\n"
|
||||
"url = https://steward.example\n"
|
||||
"token = abc123\n"
|
||||
"interval_seconds = 45\n"
|
||||
)
|
||||
cfg = read_config(str(p))
|
||||
assert cfg["url"] == "https://roundtable.example"
|
||||
assert cfg["url"] == "https://steward.example"
|
||||
assert cfg["token"] == "abc123"
|
||||
assert cfg["interval_seconds"] == 45
|
||||
|
||||
@@ -466,7 +466,7 @@ Expected: FAIL (`read_config` does not exist).
|
||||
|
||||
```python
|
||||
# plugins/host_agent/agent.py
|
||||
"""Roundtable host agent — pushes resource metrics to a Roundtable instance.
|
||||
"""Steward host agent — pushes resource metrics to a Steward instance.
|
||||
|
||||
Python 3.8+ stdlib only. Target ~300 lines. Served to targets at
|
||||
GET /plugins/host_agent/agent.py.
|
||||
@@ -1060,7 +1060,7 @@ def post_payload(url: str, token: str, payload: dict) -> tuple[bool, int | None]
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {token}",
|
||||
"User-Agent": f"roundtable-host-agent/{AGENT_VERSION}",
|
||||
"User-Agent": f"steward-host-agent/{AGENT_VERSION}",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
@@ -1112,7 +1112,7 @@ def main_loop(conf_path: str) -> int:
|
||||
buffer = RingBuffer(maxlen=20)
|
||||
backoff = 0
|
||||
|
||||
_log("INFO", f"roundtable-host-agent {AGENT_VERSION} starting "
|
||||
_log("INFO", f"steward-host-agent {AGENT_VERSION} starting "
|
||||
f"(url={cfg['url']}, interval={cfg['interval_seconds']}s)")
|
||||
|
||||
while not _shutdown_requested:
|
||||
@@ -1171,7 +1171,7 @@ def main_loop(conf_path: str) -> int:
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
conf = os.environ.get("ROUNDTABLE_AGENT_CONFIG", "/etc/roundtable-agent.conf")
|
||||
conf = os.environ.get("STEWARD_AGENT_CONFIG", "/etc/steward-agent.conf")
|
||||
sys.exit(main_loop(conf))
|
||||
```
|
||||
|
||||
@@ -1209,8 +1209,8 @@ git commit -m "feat(host_agent): agent POST, backoff, and main loop"
|
||||
import hashlib
|
||||
import pytest_asyncio
|
||||
|
||||
from roundtable.models.hosts import Host
|
||||
from roundtable.core.db import get_session
|
||||
from steward.models.hosts import Host
|
||||
from steward.core.db import get_session
|
||||
from plugins.host_agent.models import HostAgentRegistration
|
||||
|
||||
|
||||
@@ -1233,7 +1233,7 @@ async def registered_host(app):
|
||||
return {"host": host, "registration": reg, "token": raw_token}
|
||||
```
|
||||
|
||||
> **Note:** Verify `Host.__init__` accepts `name=` and `address=` kwargs before relying on this. If the real Host model requires more fields (e.g., `probe_type`), pass them here. Read `roundtable/models/hosts.py` first to confirm.
|
||||
> **Note:** Verify `Host.__init__` accepts `name=` and `address=` kwargs before relying on this. If the real Host model requires more fields (e.g., `probe_type`), pass them here. Read `steward/models/hosts.py` first to confirm.
|
||||
|
||||
- [ ] **Step 2: Write failing ingest test**
|
||||
|
||||
@@ -1244,8 +1244,8 @@ from datetime import datetime, timezone
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from roundtable.models.metrics import PluginMetric
|
||||
from roundtable.core.db import get_session
|
||||
from steward.models.metrics import PluginMetric
|
||||
from steward.core.db import get_session
|
||||
from plugins.host_agent.models import HostAgentRegistration
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
@@ -1352,9 +1352,9 @@ from typing import Any
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from roundtable.core.db import get_session
|
||||
from roundtable.models.hosts import Host
|
||||
from roundtable.models.metrics import PluginMetric
|
||||
from steward.core.db import get_session
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.metrics import PluginMetric
|
||||
from .models import HostAgentRegistration
|
||||
|
||||
host_agent_bp = Blueprint("host_agent", __name__, template_folder="templates")
|
||||
@@ -1628,7 +1628,7 @@ async def test_install_sh_renders_with_token(client, registered_host):
|
||||
assert resp.status_code == 200
|
||||
assert resp.content_type.startswith("text/plain")
|
||||
text = (await resp.get_data()).decode()
|
||||
assert "roundtable-agent" in text
|
||||
assert "steward-agent" in text
|
||||
assert registered_host["token"] in text
|
||||
assert "systemctl enable --now" in text
|
||||
assert "NoNewPrivileges=yes" in text
|
||||
@@ -1758,11 +1758,11 @@ git commit -m "feat(host_agent): install.sh and agent.py serving routes"
|
||||
- Create: `plugins/host_agent/templates/settings_list.html`
|
||||
- Create: `tests/plugins/host_agent/test_settings_routes.py`
|
||||
|
||||
Auth note: the existing Roundtable admin decorator pattern needs to be used here. Read `roundtable/settings/routes.py` (or similar) to find the decorator name — likely something like `@require_admin` or a session check. Use whatever the rest of the codebase uses. **Do not invent a new auth mechanism.**
|
||||
Auth note: the existing Steward admin decorator pattern needs to be used here. Read `steward/settings/routes.py` (or similar) to find the decorator name — likely something like `@require_admin` or a session check. Use whatever the rest of the codebase uses. **Do not invent a new auth mechanism.**
|
||||
|
||||
- [ ] **Step 1: Find the admin auth decorator**
|
||||
|
||||
Run: `grep -rn "require_admin\|@admin_required\|def admin" roundtable/settings/ roundtable/core/auth.py 2>/dev/null | head -20`
|
||||
Run: `grep -rn "require_admin\|@admin_required\|def admin" steward/settings/ steward/core/auth.py 2>/dev/null | head -20`
|
||||
Note the decorator name and import path for use in Step 3.
|
||||
|
||||
- [ ] **Step 2: Write failing test**
|
||||
@@ -1772,8 +1772,8 @@ Note the decorator name and import path for use in Step 3.
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from roundtable.core.db import get_session
|
||||
from roundtable.models.hosts import Host
|
||||
from steward.core.db import get_session
|
||||
from steward.models.hosts import Host
|
||||
from plugins.host_agent.models import HostAgentRegistration
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
@@ -1839,7 +1839,7 @@ from quart import redirect, url_for
|
||||
|
||||
# TODO: replace _require_admin with the project-wide decorator found in Step 1.
|
||||
# Placeholder below mirrors the shape; swap for real admin auth.
|
||||
from roundtable.core.auth import require_admin # adjust import to actual path
|
||||
from steward.core.auth import require_admin # adjust import to actual path
|
||||
|
||||
|
||||
def _new_token_pair() -> tuple[str, str]:
|
||||
@@ -1938,7 +1938,7 @@ async def settings_list():
|
||||
|
||||
```html
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Host Agent — Roundtable{% endblock %}
|
||||
{% block title %}Host Agent — Steward{% endblock %}
|
||||
{% block content %}
|
||||
<h1 class="page-title">Host Agent — Registered Hosts</h1>
|
||||
|
||||
@@ -2003,7 +2003,7 @@ async def settings_list():
|
||||
- [ ] **Step 6: Run tests — iterate on auth decorator if needed**
|
||||
|
||||
Run: `pytest tests/plugins/host_agent/test_settings_routes.py -v`
|
||||
Expected: PASS. If admin auth fails, the actual import path from Step 1 is wrong — fix the `from roundtable.core.auth import require_admin` line.
|
||||
Expected: PASS. If admin auth fails, the actual import path from Step 1 is wrong — fix the `from steward.core.auth import require_admin` line.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
@@ -2020,8 +2020,8 @@ git commit -m "feat(host_agent): plugin settings page — add, rotate, delete"
|
||||
- Modify: `plugins/host_agent/routes.py` — add `/widget` and `/widget/history` partials.
|
||||
- Create: `plugins/host_agent/templates/widget_table.html`
|
||||
- Create: `plugins/host_agent/templates/widget_history.html`
|
||||
- Modify: `roundtable/core/widgets.py`
|
||||
- Modify: `roundtable/alerts/routes.py`
|
||||
- Modify: `steward/core/widgets.py`
|
||||
- Modify: `steward/alerts/routes.py`
|
||||
|
||||
- [ ] **Step 1: Add widget partial routes to `plugins/host_agent/routes.py`**
|
||||
|
||||
@@ -2165,7 +2165,7 @@ async def widget_history():
|
||||
</div>
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Register widgets in `roundtable/core/widgets.py`**
|
||||
- [ ] **Step 4: Register widgets in `steward/core/widgets.py`**
|
||||
|
||||
Add to `WIDGET_REGISTRY` (alphabetically by existing convention, or at the end — check the file first):
|
||||
|
||||
@@ -2206,7 +2206,7 @@ Add to `WIDGET_REGISTRY` (alphabetically by existing convention, or at the end
|
||||
|
||||
- [ ] **Step 5: Register `host_agent` in `METRIC_CATALOG`**
|
||||
|
||||
Read `roundtable/alerts/routes.py`. Locate the `METRIC_CATALOG` dict. Add:
|
||||
Read `steward/alerts/routes.py`. Locate the `METRIC_CATALOG` dict. Add:
|
||||
|
||||
```python
|
||||
"host_agent": [
|
||||
@@ -2244,7 +2244,7 @@ Expected: PASS.
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add plugins/host_agent/routes.py plugins/host_agent/templates/widget_table.html plugins/host_agent/templates/widget_history.html roundtable/core/widgets.py roundtable/alerts/routes.py tests/plugins/host_agent/test_ingest_route.py
|
||||
git add plugins/host_agent/routes.py plugins/host_agent/templates/widget_table.html plugins/host_agent/templates/widget_history.html steward/core/widgets.py steward/alerts/routes.py tests/plugins/host_agent/test_ingest_route.py
|
||||
git commit -m "feat(host_agent): dashboard widgets and alert metric catalog entry"
|
||||
```
|
||||
|
||||
@@ -2409,8 +2409,8 @@ from datetime import datetime, timedelta, timezone
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from roundtable.core.db import get_session
|
||||
from roundtable.models.hosts import Host
|
||||
from steward.core.db import get_session
|
||||
from steward.models.hosts import Host
|
||||
from plugins.host_agent.models import HostAgentRegistration
|
||||
from plugins.host_agent.scheduler import find_stale_registrations
|
||||
|
||||
@@ -2459,8 +2459,8 @@ from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from roundtable.core.db import get_session
|
||||
from roundtable.models.hosts import Host
|
||||
from steward.core.db import get_session
|
||||
from steward.models.hosts import Host
|
||||
from .models import HostAgentRegistration
|
||||
|
||||
|
||||
@@ -2531,8 +2531,8 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from roundtable.core.db import get_session
|
||||
from roundtable.models.metrics import PluginMetric
|
||||
from steward.core.db import get_session
|
||||
from steward.models.metrics import PluginMetric
|
||||
from plugins.host_agent import agent as a
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
@@ -2609,12 +2609,12 @@ Append to the `plugins:` list in `docs/plugins/index.yaml.example`:
|
||||
- name: host_agent
|
||||
version: "1.0.0"
|
||||
description: "Remote Linux host resource monitoring via a lightweight Python push agent"
|
||||
author: "Roundtable"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/src/branch/main/host_agent"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/releases/download/host_agent-v1.0.0/host_agent.zip"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/host_agent"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/host_agent-v1.0.0/host_agent.zip"
|
||||
checksum_sha256: ""
|
||||
tags:
|
||||
- host
|
||||
@@ -2643,7 +2643,7 @@ Use `fable_update_task` to set status=done on task 252 ("Implement host_agent pl
|
||||
|
||||
- [ ] **Step 2: Add a Fable note summarizing what shipped**
|
||||
|
||||
One-paragraph `fable_create_note` attached to Roundtable project (id 6) with:
|
||||
One-paragraph `fable_create_note` attached to Steward project (id 6) with:
|
||||
- Link to spec: `docs/plugins/host-agent-design.md`
|
||||
- Link to plan: `docs/plugins/host-agent-plan.md`
|
||||
- Note any deviations from the spec discovered during implementation (e.g., auth decorator name, fixture adjustments).
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# roundtable-plugins / index.yaml
|
||||
# steward-plugins / index.yaml
|
||||
#
|
||||
# This file is the catalog index for the roundtable plugin repository.
|
||||
# This file is the catalog index for the steward plugin repository.
|
||||
# It is fetched by the app's Settings → Plugins → Plugin Catalog UI.
|
||||
#
|
||||
# Roundtable reads this file from:
|
||||
# https://git.fabledsword.com/bvandeusen/Roundtable-plugins/raw/branch/main/index.yaml
|
||||
# Steward reads this file from:
|
||||
# https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml
|
||||
#
|
||||
# After adding or updating a plugin entry, commit and push — the change is
|
||||
# live immediately for anyone whose app fetches the catalog (cache TTL: 5 min).
|
||||
@@ -25,7 +25,7 @@
|
||||
#
|
||||
# Download URL conventions:
|
||||
# Gitea release assets (recommended):
|
||||
# https://git.fabledsword.com/bvandeusen/Roundtable-plugins/releases/download/traefik-v1.0.0/traefik.zip
|
||||
# https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip
|
||||
# Upload the zip as a release attachment in Gitea; paste the URL here.
|
||||
# Source archive tarballs work too but release assets are preferred (smaller, plugin-only).
|
||||
|
||||
@@ -37,12 +37,12 @@ plugins:
|
||||
- name: http
|
||||
version: "1.0.0"
|
||||
description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry"
|
||||
author: "Roundtable"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/src/branch/main/http"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/releases/download/http-v1.0.0/http.zip"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/http"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/http-v1.0.0/http.zip"
|
||||
checksum_sha256: ""
|
||||
tags:
|
||||
- monitoring
|
||||
@@ -53,12 +53,12 @@ plugins:
|
||||
- name: docker
|
||||
version: "1.0.0"
|
||||
description: "Docker container status, resource usage, and restart tracking via Docker socket"
|
||||
author: "Roundtable"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/src/branch/main/docker"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/releases/download/docker-v1.0.0/docker.zip"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v1.0.0/docker.zip"
|
||||
checksum_sha256: ""
|
||||
tags:
|
||||
- containers
|
||||
@@ -68,12 +68,12 @@ plugins:
|
||||
- name: traefik
|
||||
version: "1.0.0"
|
||||
description: "Traefik reverse proxy metrics and access log integration"
|
||||
author: "Roundtable"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/src/branch/main/traefik"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/releases/download/traefik-v1.0.0/traefik.zip"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip"
|
||||
checksum_sha256: "" # fill in after running: sha256sum traefik.zip
|
||||
tags:
|
||||
- proxy
|
||||
@@ -83,12 +83,12 @@ plugins:
|
||||
- name: unifi
|
||||
version: "1.0.0"
|
||||
description: "UniFi Network controller integration — WAN health, devices, clients, DPI"
|
||||
author: "Roundtable"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/src/branch/main/unifi"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/releases/download/unifi-v1.0.0/unifi.zip"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/unifi-v1.0.0/unifi.zip"
|
||||
checksum_sha256: ""
|
||||
tags:
|
||||
- network
|
||||
@@ -98,12 +98,12 @@ plugins:
|
||||
- name: host_agent
|
||||
version: "1.0.0"
|
||||
description: "Remote Linux host resource monitoring via a lightweight Python push agent"
|
||||
author: "Roundtable"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/src/branch/main/host_agent"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Roundtable-plugins/releases/download/host_agent-v1.0.0/host_agent.zip"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/host_agent"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/host_agent-v1.0.0/host_agent.zip"
|
||||
checksum_sha256: ""
|
||||
tags:
|
||||
- host
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# Plugin System Overview
|
||||
|
||||
Plugins extend Roundtable with new data sources, UI pages, scheduled tasks, and dashboard widgets. Only enabled plugins are imported — disabled or unlisted plugins have zero runtime overhead.
|
||||
Plugins extend Steward with new data sources, UI pages, scheduled tasks, and dashboard widgets. Only enabled plugins are imported — disabled or unlisted plugins have zero runtime overhead.
|
||||
|
||||
---
|
||||
|
||||
## How Plugins Are Loaded
|
||||
|
||||
Plugin loading happens in step 9 of `create_app()`, after all core blueprints and tasks are registered. The entrypoint is `load_plugins(app)` in `roundtable/core/plugin_manager.py`.
|
||||
Plugin loading happens in step 9 of `create_app()`, after all core blueprints and tasks are registered. The entrypoint is `load_plugins(app)` in `steward/core/plugin_manager.py`.
|
||||
|
||||
For each plugin listed as `enabled: true` in the `PLUGINS` config:
|
||||
|
||||
@@ -57,7 +57,7 @@ description: "Does a thing"
|
||||
|
||||
# Optional
|
||||
author: "Your Name"
|
||||
min_app_version: "0.1.0" # Minimum Roundtable version required
|
||||
min_app_version: "0.1.0" # Minimum Steward version required
|
||||
|
||||
# Default config — merged with user overrides at runtime
|
||||
# Access at runtime via: app.config["PLUGINS"]["myplugin"]["my_setting"]
|
||||
@@ -90,7 +90,7 @@ def setup(app):
|
||||
Returns a list of `ScheduledTask` objects. Return `[]` if the plugin has no background tasks. Called after `setup()`, so any app references set in `setup()` are available.
|
||||
|
||||
```python
|
||||
from roundtable.core.scheduler import ScheduledTask
|
||||
from steward.core.scheduler import ScheduledTask
|
||||
|
||||
def get_scheduled_tasks():
|
||||
app = _app
|
||||
@@ -154,7 +154,7 @@ A plugin can contribute a dashboard widget by:
|
||||
1. Adding a `GET /widget` route to its blueprint that returns an HTMX HTML fragment
|
||||
2. The dashboard template polling that endpoint with HTMX
|
||||
|
||||
The dashboard (`roundtable/templates/dashboard/index.html`) currently includes the Traefik widget conditionally based on `traefik_enabled`. To add a new plugin widget, the dashboard route and template both need to be updated to detect and render the new widget. See the Traefik widget implementation as a reference.
|
||||
The dashboard (`steward/templates/dashboard/index.html`) currently includes the Traefik widget conditionally based on `traefik_enabled`. To add a new plugin widget, the dashboard route and template both need to be updated to detect and render the new widget. See the Traefik widget implementation as a reference.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ config:
|
||||
|
||||
## Step 3: Define Models (if needed)
|
||||
|
||||
If your plugin stores data, define SQLAlchemy models using the shared `Base` from `roundtable.models.base`.
|
||||
If your plugin stores data, define SQLAlchemy models using the shared `Base` from `steward.models.base`.
|
||||
|
||||
```python
|
||||
# plugins/myplugin/models.py
|
||||
@@ -55,7 +55,7 @@ import uuid
|
||||
from datetime import datetime
|
||||
from sqlalchemy import String, Float, DateTime
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from roundtable.models.base import Base
|
||||
from steward.models.base import Base
|
||||
|
||||
|
||||
class MyPluginMetric(Base):
|
||||
@@ -79,7 +79,7 @@ Generate the initial migration:
|
||||
# From the project root
|
||||
alembic --config alembic.ini revision \
|
||||
--autogenerate \
|
||||
--head=roundtable@head \
|
||||
--head=steward@head \
|
||||
--branch-label=myplugin \
|
||||
-m "myplugin initial"
|
||||
```
|
||||
@@ -106,8 +106,8 @@ Keep task logic in a separate file so `__init__.py` stays clean.
|
||||
# plugins/myplugin/scheduler.py
|
||||
from __future__ import annotations
|
||||
import logging
|
||||
from roundtable.core.scheduler import ScheduledTask
|
||||
from roundtable.core.alerts import record_metric
|
||||
from steward.core.scheduler import ScheduledTask
|
||||
from steward.core.alerts import record_metric
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -174,8 +174,8 @@ async def _fetch_value(url: str) -> float:
|
||||
```python
|
||||
# plugins/myplugin/routes.py
|
||||
from quart import Blueprint, current_app, render_template
|
||||
from roundtable.auth.middleware import require_role
|
||||
from roundtable.models.users import UserRole
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.models.users import UserRole
|
||||
from .models import MyPluginMetric
|
||||
|
||||
myplugin_bp = Blueprint("myplugin", __name__, template_folder="templates")
|
||||
@@ -215,7 +215,7 @@ Templates live in `plugins/myplugin/templates/myplugin/` (the extra nesting avoi
|
||||
```html
|
||||
{# plugins/myplugin/templates/myplugin/index.html #}
|
||||
{% extends "base.html" %}
|
||||
{% block title %}My Plugin — Roundtable{% endblock %}
|
||||
{% block title %}My Plugin — Steward{% endblock %}
|
||||
{% block content %}
|
||||
<div class="page-title">My Plugin</div>
|
||||
{% for row in rows %}
|
||||
@@ -283,7 +283,7 @@ On next startup, the plugin will be loaded, its migrations applied, and its blue
|
||||
`record_metric()` is how plugins feed data into the alert pipeline. Any metric you write here can be the target of an alert rule created in the UI.
|
||||
|
||||
```python
|
||||
from roundtable.core.alerts import record_metric
|
||||
from steward.core.alerts import record_metric
|
||||
|
||||
# Must be inside an active transaction
|
||||
async with session.begin():
|
||||
@@ -302,11 +302,11 @@ async with session.begin():
|
||||
|
||||
## Auth in Routes
|
||||
|
||||
Use the `@require_role` decorator from `roundtable.auth.middleware`:
|
||||
Use the `@require_role` decorator from `steward.auth.middleware`:
|
||||
|
||||
```python
|
||||
from roundtable.auth.middleware import require_role
|
||||
from roundtable.models.users import UserRole
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.models.users import UserRole
|
||||
|
||||
@myplugin_bp.get("/admin-only")
|
||||
@require_role(UserRole.admin)
|
||||
@@ -323,15 +323,70 @@ Role hierarchy: `admin > operator > viewer`. Requiring `viewer` grants access to
|
||||
|
||||
---
|
||||
|
||||
## Agent Tools (MCP)
|
||||
|
||||
Steward serves an MCP endpoint at `/mcp` that AI clients reach with a personal API token created on the AI access page. A plugin adds tools to it by exporting an optional `register_mcp_tools(register)` from `__init__.py`:
|
||||
|
||||
```python
|
||||
# plugins/myplugin/__init__.py
|
||||
def register_mcp_tools(register) -> None:
|
||||
from .mcp_tools import list_targets, rescan_target
|
||||
from steward.models.users import UserRole
|
||||
register(list_targets, role=UserRole.viewer, read_only=True)
|
||||
register(rescan_target, role=UserRole.operator, read_only=False)
|
||||
```
|
||||
|
||||
```python
|
||||
# plugins/myplugin/mcp_tools.py
|
||||
from steward.mcp._context import current_caller, mcp_app
|
||||
|
||||
|
||||
async def list_targets(limit: int = 50) -> dict:
|
||||
"""What this tool is for, when to use it over its siblings, what each
|
||||
argument's empty value means, and what comes back. This docstring is the
|
||||
tool's description: it is all the agent knows about it."""
|
||||
app = mcp_app() # Quart's current_app does not exist here
|
||||
async with app.db_sessionmaker() as db:
|
||||
...
|
||||
return {"targets": [...], "total": n, "truncated": n > limit}
|
||||
```
|
||||
|
||||
- `role` and `read_only` are required. A tool is listed and callable only for callers whose role reaches `role`, and only while the plugin is enabled.
|
||||
- Tools run outside Quart's request pipeline: use `mcp_app()` and `current_caller()` from `steward.mcp._context`, never `current_app` or `session`.
|
||||
- Refuse by raising `ValueError("what was wrong, and what to do instead")`. The agent reads that text. Any other exception reaches it only as a generic failure.
|
||||
- Tool names are global across core and every plugin; a name already taken fails registration and the plugin contributes no tools (logged).
|
||||
- Hot-reloading the plugin replaces its tools.
|
||||
- Follow the response conventions the core tools use (`steward/mcp/tools/_shape.py`): times via `iso()` plus `age_s()`, lists through `bounded()` so a cut is never silent, series through `summarize()`, and lookups by name or id through `pick()`.
|
||||
|
||||
### Contributing to get_overview
|
||||
|
||||
`get_overview` is the agent's first call: what is wrong right now, ranked. A plugin adds its own problems to it by exporting an optional `mcp_overview(db)` that returns findings built with `steward.mcp.tools.overview.finding`:
|
||||
|
||||
```python
|
||||
# plugins/myplugin/__init__.py
|
||||
async def mcp_overview(db) -> list[dict]:
|
||||
from steward.mcp.tools.overview import finding
|
||||
return [
|
||||
finding("warning", "target_unreachable", t.name, f"{t.name} has not answered for {t.misses} scans",
|
||||
since=t.last_ok_at, ids={"target_id": t.id}, next=f'get_target(target="{t.id}")')
|
||||
for t in await _unreachable_targets(db)
|
||||
]
|
||||
```
|
||||
|
||||
- `severity` is `"critical"` (something is down or about to be) or `"warning"` (degraded, or heading that way). Report problems only; a healthy plugin returns `[]`.
|
||||
- It is called only while the plugin is enabled, with its own session. If it raises, the overview still returns, and names the plugin under `unavailable`.
|
||||
|
||||
---
|
||||
|
||||
## Publishing to the Catalog
|
||||
|
||||
The official plugin catalog is hosted at `https://git.fabledsword.com/bvandeusen/Roundtable-plugins`. Anyone can submit a plugin by opening a pull request — first-party and third-party plugins are treated identically by the catalog system.
|
||||
The official plugin catalog is hosted at `https://git.fabledsword.com/bvandeusen/Steward-plugins`. Anyone can submit a plugin by opening a pull request — first-party and third-party plugins are treated identically by the catalog system.
|
||||
|
||||
### Repo layout
|
||||
|
||||
```
|
||||
roundtable-plugins/
|
||||
├── index.yaml ← catalog index — the only file Roundtable fetches
|
||||
steward-plugins/
|
||||
├── index.yaml ← catalog index — the only file Steward fetches
|
||||
├── myplugin/
|
||||
│ ├── plugin.yaml
|
||||
│ ├── __init__.py
|
||||
@@ -381,7 +436,7 @@ myplugin.zip
|
||||
Generate the zip and its checksum:
|
||||
|
||||
```bash
|
||||
cd roundtable-plugins
|
||||
cd steward-plugins
|
||||
zip -r myplugin.zip myplugin/
|
||||
sha256sum myplugin.zip # paste this into index.yaml checksum_sha256
|
||||
```
|
||||
@@ -413,6 +468,7 @@ Hot-reload works reliably for brand-new plugin installs. Updates to already-acti
|
||||
- [ ] Revision IDs prefixed with plugin name
|
||||
- [ ] `record_metric()` called inside `session.begin()`
|
||||
- [ ] Routes use `@require_role` decorator
|
||||
- [ ] MCP tools (if any) registered via `register_mcp_tools(register)` with a role and `read_only`
|
||||
- [ ] Templates namespaced under `templates/myplugin/`
|
||||
- [ ] Plugin enabled in app settings
|
||||
- [ ] `index.yaml` entry added with `download_url` and `checksum_sha256`
|
||||
|
||||
+82
-59
@@ -8,15 +8,15 @@ Quick reference for where key functions, models, and entry points live in the co
|
||||
|
||||
| What | File | Function / Class |
|
||||
|---|---|---|
|
||||
| App factory | `roundtable/app.py` | `create_app()` |
|
||||
| CLI entry point | `roundtable/cli.py` | `main()` |
|
||||
| Bootstrap config loading | `roundtable/config.py` | `load_bootstrap()` |
|
||||
| Secret key resolution | `roundtable/config.py` | `_resolve_secret_key()` |
|
||||
| DB engine init | `roundtable/database.py` | `init_db()` |
|
||||
| Core migrations | `roundtable/core/migration_runner.py` | `run_core_migrations()` |
|
||||
| Plugin migrations | `roundtable/core/migration_runner.py` | `run_plugin_migrations()` |
|
||||
| Core task registration | `roundtable/app.py` | `_register_core_tasks()` |
|
||||
| Scheduler loop | `roundtable/core/scheduler.py` | `start_scheduler()` |
|
||||
| App factory | `steward/app.py` | `create_app()` |
|
||||
| CLI entry point | `steward/cli.py` | `main()` |
|
||||
| Bootstrap config loading | `steward/config.py` | `load_bootstrap()` |
|
||||
| Secret key resolution | `steward/config.py` | `_resolve_secret_key()` |
|
||||
| DB engine init | `steward/database.py` | `init_db()` |
|
||||
| Core migrations | `steward/core/migration_runner.py` | `run_core_migrations()` |
|
||||
| Plugin migrations | `steward/core/migration_runner.py` | `run_plugin_migrations()` |
|
||||
| Core task registration | `steward/app.py` | `_register_core_tasks()` |
|
||||
| Scheduler loop | `steward/core/scheduler.py` | `start_scheduler()` |
|
||||
|
||||
---
|
||||
|
||||
@@ -24,15 +24,15 @@ Quick reference for where key functions, models, and entry points live in the co
|
||||
|
||||
| What | File | Function |
|
||||
|---|---|---|
|
||||
| All defaults | `roundtable/core/settings.py` | `DEFAULTS` dict |
|
||||
| Read a setting | `roundtable/core/settings.py` | `get_setting(session, key)` |
|
||||
| Write a setting | `roundtable/core/settings.py` | `set_setting(session, key, value)` |
|
||||
| Read all settings | `roundtable/core/settings.py` | `get_all_settings(session)` |
|
||||
| Sync load at startup | `roundtable/core/settings.py` | `load_settings_sync(db_url)` |
|
||||
| Extract SMTP dict | `roundtable/core/settings.py` | `to_smtp_cfg(settings)` |
|
||||
| Extract webhook dict | `roundtable/core/settings.py` | `to_webhook_cfg(settings)` |
|
||||
| Extract Ansible dict | `roundtable/core/settings.py` | `to_ansible_cfg(settings)` |
|
||||
| Extract plugins dict | `roundtable/core/settings.py` | `to_plugins_cfg(settings)` |
|
||||
| All defaults | `steward/core/settings.py` | `DEFAULTS` dict |
|
||||
| Read a setting | `steward/core/settings.py` | `get_setting(session, key)` |
|
||||
| Write a setting | `steward/core/settings.py` | `set_setting(session, key, value)` |
|
||||
| Read all settings | `steward/core/settings.py` | `get_all_settings(session)` |
|
||||
| Sync load at startup | `steward/core/settings.py` | `load_settings_sync(db_url)` |
|
||||
| Extract SMTP dict | `steward/core/settings.py` | `to_smtp_cfg(settings)` |
|
||||
| Extract webhook dict | `steward/core/settings.py` | `to_webhook_cfg(settings)` |
|
||||
| Extract Ansible dict | `steward/core/settings.py` | `to_ansible_cfg(settings)` |
|
||||
| Extract plugins dict | `steward/core/settings.py` | `to_plugins_cfg(settings)` |
|
||||
|
||||
---
|
||||
|
||||
@@ -40,9 +40,9 @@ Quick reference for where key functions, models, and entry points live in the co
|
||||
|
||||
| What | File | Function |
|
||||
|---|---|---|
|
||||
| Plugin loading | `roundtable/core/plugin_manager.py` | `load_plugins(app)` |
|
||||
| ScheduledTask dataclass | `roundtable/core/scheduler.py` | `ScheduledTask` |
|
||||
| Task runner | `roundtable/core/scheduler.py` | `start_scheduler(tasks)` |
|
||||
| Plugin loading | `steward/core/plugin_manager.py` | `load_plugins(app)` |
|
||||
| ScheduledTask dataclass | `steward/core/scheduler.py` | `ScheduledTask` |
|
||||
| Task runner | `steward/core/scheduler.py` | `start_scheduler(tasks)` |
|
||||
|
||||
---
|
||||
|
||||
@@ -50,11 +50,11 @@ Quick reference for where key functions, models, and entry points live in the co
|
||||
|
||||
| What | File | Function |
|
||||
|---|---|---|
|
||||
| Write metric + evaluate alerts | `roundtable/core/alerts.py` | `record_metric(session, source_module, resource_name, metric_name, value)` |
|
||||
| Init alert pipeline | `roundtable/core/alerts.py` | `init_alerts(app)` |
|
||||
| Rule evaluation | `roundtable/core/alerts.py` | `_evaluate_rule()` (internal) |
|
||||
| Notification dispatch | `roundtable/core/alerts.py` | `_dispatch_notification()` (internal) |
|
||||
| Email + webhook send | `roundtable/core/notifications.py` | `dispatch_notifications()` |
|
||||
| Write metric + evaluate alerts | `steward/core/alerts.py` | `record_metric(session, source_module, resource_name, metric_name, value)` |
|
||||
| Init alert pipeline | `steward/core/alerts.py` | `init_alerts(app)` |
|
||||
| Rule evaluation | `steward/core/alerts.py` | `_evaluate_rule()` (internal) |
|
||||
| Notification dispatch | `steward/core/alerts.py` | `_dispatch_notification()` (internal) |
|
||||
| Email + webhook send | `steward/core/notifications.py` | `dispatch_notifications()` |
|
||||
|
||||
---
|
||||
|
||||
@@ -62,9 +62,9 @@ Quick reference for where key functions, models, and entry points live in the co
|
||||
|
||||
| What | File | Function |
|
||||
|---|---|---|
|
||||
| Ping a host | `roundtable/monitors/ping.py` | `ping_check(host, session)` |
|
||||
| DNS check a host | `roundtable/monitors/dns.py` | `dns_check(host, session)` |
|
||||
| Data cleanup | `roundtable/core/cleanup.py` | `run_cleanup(app)` |
|
||||
| Ping a host | `steward/monitors/ping.py` | `ping_check(host, session)` |
|
||||
| DNS check a host | `steward/monitors/dns.py` | `dns_check(host, session)` |
|
||||
| Data cleanup | `steward/core/cleanup.py` | `run_cleanup(app)` |
|
||||
|
||||
---
|
||||
|
||||
@@ -72,9 +72,9 @@ Quick reference for where key functions, models, and entry points live in the co
|
||||
|
||||
| What | File | Function / Class |
|
||||
|---|---|---|
|
||||
| Role-based access decorator | `roundtable/auth/middleware.py` | `@require_role(UserRole.X)` |
|
||||
| Login / session handling | `roundtable/auth/middleware.py` | `login_user()`, `logout_user()` |
|
||||
| User count (for first-run) | `roundtable/auth/middleware.py` | `get_user_count(app)` |
|
||||
| Role-based access decorator | `steward/auth/middleware.py` | `@require_role(UserRole.X)` |
|
||||
| Login / session handling | `steward/auth/middleware.py` | `login_user()`, `logout_user()` |
|
||||
| User count (for first-run) | `steward/auth/middleware.py` | `get_user_count(app)` |
|
||||
|
||||
---
|
||||
|
||||
@@ -82,18 +82,19 @@ Quick reference for where key functions, models, and entry points live in the co
|
||||
|
||||
| Model | File | Table |
|
||||
|---|---|---|
|
||||
| `Host` | `roundtable/models/hosts.py` | `hosts` |
|
||||
| `PingResult` | `roundtable/models/monitors.py` | `ping_results` |
|
||||
| `DnsResult` | `roundtable/models/monitors.py` | `dns_results` |
|
||||
| `AlertRule` | `roundtable/models/alerts.py` | `alert_rules` |
|
||||
| `AlertState` | `roundtable/models/alerts.py` | `alert_states` |
|
||||
| `AlertEvent` | `roundtable/models/alerts.py` | `alert_events` |
|
||||
| `PluginMetric` | `roundtable/models/metrics.py` | `plugin_metrics` |
|
||||
| `AnsibleRun` | `roundtable/models/ansible.py` | `ansible_runs` |
|
||||
| `User` | `roundtable/models/users.py` | `users` |
|
||||
| `AppSetting` | `roundtable/models/settings.py` | `app_settings` |
|
||||
| `Host` | `steward/models/hosts.py` | `hosts` |
|
||||
| `PingResult` | `steward/models/monitors.py` | `ping_results` |
|
||||
| `DnsResult` | `steward/models/monitors.py` | `dns_results` |
|
||||
| `AlertRule` | `steward/models/alerts.py` | `alert_rules` |
|
||||
| `AlertState` | `steward/models/alerts.py` | `alert_states` |
|
||||
| `AlertEvent` | `steward/models/alerts.py` | `alert_events` |
|
||||
| `PluginMetric` | `steward/models/metrics.py` | `plugin_metrics` |
|
||||
| `PluginMetricLatest` | `steward/models/metrics.py` | `plugin_metric_latest` |
|
||||
| `AnsibleRun` | `steward/models/ansible.py` | `ansible_runs` |
|
||||
| `User` | `steward/models/users.py` | `users` |
|
||||
| `AppSetting` | `steward/models/settings.py` | `app_settings` |
|
||||
| `TraefikMetric` | `plugins/traefik/models.py` | `traefik_metrics` |
|
||||
| SQLAlchemy `Base` | `roundtable/models/base.py` | (shared declarative base) |
|
||||
| SQLAlchemy `Base` | `steward/models/base.py` | (shared declarative base) |
|
||||
|
||||
---
|
||||
|
||||
@@ -101,16 +102,16 @@ Quick reference for where key functions, models, and entry points live in the co
|
||||
|
||||
| URL pattern | Blueprint | File |
|
||||
|---|---|---|
|
||||
| `/` (dashboard) | `dashboard_bp` | `roundtable/dashboard/routes.py` |
|
||||
| `/auth/login`, `/auth/logout` | `auth_bp` | `roundtable/auth/routes.py` |
|
||||
| `/hosts/` | `hosts_bp` | `roundtable/hosts/routes.py` |
|
||||
| `/ping/`, `/ping/rows`, `/ping/settings` | `ping_bp` | `roundtable/ping/routes.py` |
|
||||
| `/dns/`, `/dns/rows` | `dns_bp` | `roundtable/dns/routes.py` |
|
||||
| `/alerts/` | `alerts_bp` | `roundtable/alerts/routes.py` |
|
||||
| `/ansible/` | `ansible_bp` | `roundtable/ansible/routes.py` |
|
||||
| `/settings/` | `settings_bp` | `roundtable/settings/routes.py` |
|
||||
| `/` (dashboard) | `dashboard_bp` | `steward/dashboard/routes.py` |
|
||||
| `/auth/login`, `/auth/logout` | `auth_bp` | `steward/auth/routes.py` |
|
||||
| `/hosts/` | `hosts_bp` | `steward/hosts/routes.py` |
|
||||
| `/ping/`, `/ping/rows`, `/ping/settings` | `ping_bp` | `steward/ping/routes.py` |
|
||||
| `/dns/`, `/dns/rows` | `dns_bp` | `steward/dns/routes.py` |
|
||||
| `/alerts/` | `alerts_bp` | `steward/alerts/routes.py` |
|
||||
| `/ansible/` | `ansible_bp` | `steward/ansible/routes.py` |
|
||||
| `/settings/` | `settings_bp` | `steward/settings/routes.py` |
|
||||
| `/plugins/traefik/`, `/plugins/traefik/widget` | `traefik_bp` | `plugins/traefik/routes.py` |
|
||||
| `/health` | (inline) | `roundtable/app.py` |
|
||||
| `/health` | (inline) | `steward/app.py` |
|
||||
|
||||
---
|
||||
|
||||
@@ -118,12 +119,12 @@ Quick reference for where key functions, models, and entry points live in the co
|
||||
|
||||
| Template | Purpose |
|
||||
|---|---|
|
||||
| `roundtable/templates/base.html` | Layout, navigation, full CSS design system |
|
||||
| `roundtable/templates/dashboard/index.html` | Dashboard with stat strip and widget grid |
|
||||
| `roundtable/templates/ping/rows.html` | HTMX fragment: ping pill rows (shared by dashboard and /ping/) |
|
||||
| `roundtable/templates/ping/index.html` | Full /ping/ page |
|
||||
| `roundtable/templates/dns/rows.html` | HTMX fragment: DNS status rows |
|
||||
| `roundtable/templates/dns/index.html` | Full /dns/ page |
|
||||
| `steward/templates/base.html` | Layout, navigation, full CSS design system |
|
||||
| `steward/templates/dashboard/index.html` | Dashboard with stat strip and widget grid |
|
||||
| `steward/templates/ping/rows.html` | HTMX fragment: ping pill rows (shared by dashboard and /ping/) |
|
||||
| `steward/templates/ping/index.html` | Full /ping/ page |
|
||||
| `steward/templates/dns/rows.html` | HTMX fragment: DNS status rows |
|
||||
| `steward/templates/dns/index.html` | Full /dns/ page |
|
||||
| `plugins/traefik/templates/traefik/widget.html` | HTMX fragment: Traefik dashboard widget |
|
||||
| `plugins/traefik/templates/traefik/index.html` | Full Traefik detail page |
|
||||
|
||||
@@ -133,6 +134,28 @@ Quick reference for where key functions, models, and entry points live in the co
|
||||
|
||||
| Location | Covers |
|
||||
|---|---|
|
||||
| `roundtable/migrations/versions/` | Core schema (hosts, users, monitors, alerts, app_settings) |
|
||||
| `steward/migrations/versions/` | Core schema (hosts, users, monitors, alerts, app_settings) |
|
||||
| `plugins/traefik/migrations/versions/` | `traefik_metrics` table |
|
||||
| `alembic.ini` | Alembic config; `version_locations` lists all migration directories |
|
||||
|
||||
---
|
||||
|
||||
## MCP server and API tokens
|
||||
|
||||
| What | File | Function / Class |
|
||||
|---|---|---|
|
||||
| Mount `/mcp` (ASGI dispatch, bearer auth, DB gate) | `steward/mcp/server.py` | `mount_mcp(app)` |
|
||||
| Server, role registry, refusals | `steward/mcp/server.py` | `StewardMCPServer`, `register_tool()`, `refusal()` |
|
||||
| Plugin tools and overview findings | `steward/mcp/server.py` | `register_plugin(name, module)` |
|
||||
| Caller and app inside a tool | `steward/mcp/_context.py` | `current_caller()`, `mcp_app()` |
|
||||
| Core tool modules | `steward/mcp/tools/` | `register_all()`; one module per area |
|
||||
| Response conventions | `steward/mcp/tools/_shape.py` | `iso`, `age_s`, `bounded`, `lookback`, `summarize`, `pick` |
|
||||
| Audit a write as "<user> via MCP" | `steward/mcp/tools/_audit.py` | `audit_mcp()` |
|
||||
| "What is wrong now" | `steward/mcp/tools/overview.py` | `get_overview()` |
|
||||
| Playbooks, runs, schedules, alert actions | `steward/mcp/tools/automation.py` | `run_playbook()`, `wait_for_run()`, … |
|
||||
| Monitor validation (forms and tools) | `steward/monitors/service.py` | `monitor_fields()`, `build_config()` |
|
||||
| Alert rule / window validation | `steward/alerts/service.py` | `rule_fields()`, `window_fields()`, `acknowledge()` |
|
||||
| Docker and host-agent tools | `plugins/docker/mcp_tools.py`, `plugins/host_agent/mcp_tools.py` | `register_tools()`, `mcp_overview()` |
|
||||
| Token model | `steward/models/api_tokens.py` | `ApiToken` |
|
||||
| Mint, resolve, revoke tokens | `steward/core/api_tokens.py` | `create_token()`, `resolve_token()`, `revoke_token()` |
|
||||
| Token management page and connect card | `steward/ai_access/routes.py`, `steward/templates/ai_access/` | `ai_access_bp` |
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
# Authoring Steward-friendly Ansible playbooks
|
||||
|
||||
This is the contract between a playbook and Steward's run UI. Follow it and a
|
||||
playbook drops into Steward with a description, fill-in variable fields, correct
|
||||
targeting, and credentials supplied automatically — no per-playbook wiring.
|
||||
|
||||
It applies to **any** playbook in a configured source (bundled, the writable
|
||||
local source, or a git source), including third-party ones.
|
||||
|
||||
---
|
||||
|
||||
## 1. Describe what it does — `# description:`
|
||||
|
||||
Steward shows a one-line description when a playbook is selected in the run form.
|
||||
|
||||
```yaml
|
||||
---
|
||||
# description: Reclaim disk on Docker/Swarm nodes by pruning unused images and build cache.
|
||||
- name: Docker system prune
|
||||
hosts: all
|
||||
...
|
||||
```
|
||||
|
||||
- Format: a comment line `# description: <text>` anywhere in the file. **First
|
||||
match wins.** Case-insensitive on the `description:` key.
|
||||
- Why a comment and not a key: Ansible rejects unknown *play* keys (you can't add
|
||||
`description:` to a play), so a comment is the portable place. It survives
|
||||
`ansible-playbook` untouched.
|
||||
- Fallback: if there's no `# description:` comment, Steward uses the **first
|
||||
play's `name:`**. So always give your play a meaningful `name:` even without
|
||||
the comment.
|
||||
- Keep it to one readable line. Longer "how to use" notes can go in additional
|
||||
normal comments — Steward only reads the `description:` line.
|
||||
|
||||
## 2. Declare tunables in `vars:` — they become fill-in fields
|
||||
|
||||
Every **scalar** entry in a play's `vars:` block becomes an editable field in the
|
||||
run form, with the default shown as the input's placeholder.
|
||||
|
||||
```yaml
|
||||
vars:
|
||||
prune_all_images: false # → checkbox-ish text field, placeholder "default: false"
|
||||
keep_last_days: 7 # → field, placeholder "default: 7"
|
||||
registry_url: "" # → field, placeholder "no default"
|
||||
```
|
||||
|
||||
- **Blank field = use the default.** Steward only sends fields the operator
|
||||
actually fills, so an untouched field falls through to the playbook/inventory
|
||||
default rather than overriding it.
|
||||
- Only scalars (string/int/float/bool) surface as fields. Lists/dicts are
|
||||
skipped — set those in the playbook or via inventory group/host vars.
|
||||
- Values are delivered as **extra-vars** (`-e`), which are the **highest**
|
||||
precedence in Ansible — they override the `vars:` defaults. (This is why the
|
||||
default can be empty and still be safely overridden at run time.)
|
||||
|
||||
### `vars_prompt:` also works
|
||||
|
||||
Steward reads `vars_prompt` too. Use it when you want an explicit prompt or a
|
||||
required value:
|
||||
|
||||
```yaml
|
||||
vars_prompt:
|
||||
- name: release_tag
|
||||
prompt: "Which release to deploy?" # shown as the field label
|
||||
# no default → Steward marks the field REQUIRED
|
||||
- name: admin_password
|
||||
prompt: "Admin password"
|
||||
private: true # → masked field, never stored
|
||||
```
|
||||
|
||||
## 3. Secrets — name them so they're masked and not persisted
|
||||
|
||||
A field is treated as **secret** (rendered masked, and its value is **never
|
||||
written to the DB / run history**) when either:
|
||||
|
||||
- the variable name contains `password`, `passwd`, `secret`, `token`,
|
||||
`api_key` / `apikey`, `private_key`, or `credential` (case-insensitive), **or**
|
||||
- it's a `vars_prompt` entry with `private: true` (Ansible's default for
|
||||
vars_prompt is private).
|
||||
|
||||
So name sensitive variables accordingly (`db_password`, `api_token`,
|
||||
`vault_secret`) and they're handled safely with no extra config. Non-secret
|
||||
run-time vars are persisted (so scheduled runs can reuse them); secret ones are
|
||||
passed to the run only.
|
||||
|
||||
## 4. Target with `hosts: all`
|
||||
|
||||
Steward builds the inventory itself from the **target / group** the operator
|
||||
picks in the run form (or the single host on a host page). Your play should:
|
||||
|
||||
```yaml
|
||||
hosts: all # run against whatever Steward scoped to
|
||||
```
|
||||
|
||||
- Don't hardcode hostnames or rely on a checked-in inventory for Steward runs
|
||||
(Steward generates a fresh inventory per run).
|
||||
- Per-host connection vars (`ansible_host`, plus anything you set on the target
|
||||
in **Ansible → Inventory**) arrive as inventory host vars.
|
||||
- The run form's **Limit** / **Tags** map to `--limit` / `--tags`.
|
||||
|
||||
## 5. Privileges & connection — don't put credentials in the playbook
|
||||
|
||||
Steward supplies SSH and become for you:
|
||||
|
||||
- Steady-state runs connect as the managed **`steward`** account using Steward's
|
||||
managed key; that account has **passwordless sudo**. So just use
|
||||
`become: true` where you need root.
|
||||
- First-contact provisioning uses a one-time bootstrap user/password the
|
||||
operator enters (never stored).
|
||||
- Never embed SSH keys, passwords, or `ansible_user`/`ansible_ssh_pass` in the
|
||||
playbook. Connection identity is global (Settings → Ansible) or per-target.
|
||||
|
||||
## 6. Be idempotent
|
||||
|
||||
Steward re-runs playbooks (updates, schedules, retries). Use modules that
|
||||
converge (state-based) rather than ad-hoc `command:`/`shell:` where possible, so
|
||||
re-runs are safe.
|
||||
|
||||
---
|
||||
|
||||
## What Steward reads from a playbook (summary)
|
||||
|
||||
| Source in the playbook | What Steward does with it |
|
||||
|---|---|
|
||||
| `# description: <text>` comment | Description shown on selection (first match) |
|
||||
| first play `name:` | Description fallback |
|
||||
| `# steward:category: <text>` | Grouping badge in the browse list + run form |
|
||||
| `# steward:confirm: true` | Requires a confirmation tick before the run launches |
|
||||
| `vars:` scalar entries | Run-time fill-in fields (placeholder = default) |
|
||||
| `vars_prompt:` entries | Run-time fields (required if no default) |
|
||||
| secret-looking var name / `private: true` | Field masked + value not persisted |
|
||||
| `hosts:` | Expected to be `all`; Steward provides the inventory |
|
||||
|
||||
Everything else (SSH user/key, become password, the inventory, `steward_token`
|
||||
etc. for the agent playbooks) is injected by Steward at run time.
|
||||
|
||||
## Metadata comments
|
||||
|
||||
Steward reads metadata from magic comments (Ansible rejects unknown play keys,
|
||||
so comments are the portable place). Two forms:
|
||||
|
||||
- `# description: <text>` — the description (see §1).
|
||||
- `# steward:<key>: <value>` — the namespaced metadata block. First match per
|
||||
key wins; keys are case-insensitive.
|
||||
|
||||
**Implemented `# steward:` keys:**
|
||||
|
||||
| Key | Example | Effect |
|
||||
|---|---|---|
|
||||
| `category` | `# steward:category: maintenance` | Free-text grouping label. Shown as a badge in the browse list and on the run form. Purely organizational. |
|
||||
| `confirm` | `# steward:confirm: true` | Marks the playbook as significant/destructive. The run form then requires an explicit confirmation tick before it can be launched. Use for data-loss-capable plays (prune with volumes, resets, etc.). Truthy values: `true`/`yes`/`1`/`on`. |
|
||||
| `description` | `# steward:description: ...` | Alias for `# description:` (the unprefixed form takes precedence if both exist). |
|
||||
|
||||
```yaml
|
||||
---
|
||||
# description: Reclaim disk on Docker/Swarm nodes by pruning unused data.
|
||||
# steward:category: maintenance
|
||||
# steward:confirm: true
|
||||
- name: Docker system prune
|
||||
hosts: all
|
||||
...
|
||||
```
|
||||
|
||||
Other `# steward:<key>:` keys are simply ignored today — the namespace is
|
||||
reserved, so it's safe to add ones Steward doesn't yet understand without
|
||||
breaking anything, but only `category`, `confirm`, and `description` do something.
|
||||
|
||||
## Minimal annotated example
|
||||
|
||||
```yaml
|
||||
---
|
||||
# description: Restart a systemd service and confirm it came back up.
|
||||
- name: Restart a service
|
||||
hosts: all
|
||||
become: true
|
||||
gather_facts: false
|
||||
vars:
|
||||
service_name: "" # required-ish: operator fills it in the run form
|
||||
tasks:
|
||||
- name: Validate input
|
||||
ansible.builtin.assert:
|
||||
that: service_name | default('') | length > 0
|
||||
fail_msg: "Set service_name."
|
||||
- name: Restart
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ service_name }}"
|
||||
state: restarted
|
||||
- name: Confirm active
|
||||
ansible.builtin.command: "systemctl is-active {{ service_name }}"
|
||||
changed_when: false
|
||||
```
|
||||
@@ -1,10 +1,10 @@
|
||||
# Roundtable Rebrand Implementation Plan
|
||||
# Steward Rebrand Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
>
|
||||
> **Testing:** Per project feedback, skip all test authoring and test runs during this work. Verify manually by running the app.
|
||||
|
||||
**Goal:** Rebrand FabledScryer to Roundtable: full visual reskin plus full rename of package, config, containers, and docs.
|
||||
**Goal:** Rebrand FabledScryer to Steward: full visual reskin plus full rename of package, config, containers, and docs.
|
||||
|
||||
**Architecture:** Staged as six sequential PRs. PR 1 is a pure visual/copy reskin that still ships under the FabledScryer name. PRs 2–4 perform the mechanical rename in order (package → config → container + user-visible strings) with a fallback shim in PR 3 so self-hosters don't break mid-upgrade. PR 5 updates docs. PR 6 removes the shim after one release cycle.
|
||||
|
||||
@@ -25,29 +25,29 @@
|
||||
- Any template with an empty-state message — themed line
|
||||
|
||||
**Renamed (PR 2 — package):**
|
||||
- `fabledscryer/` → `roundtable/` (directory + every `from fabledscryer` / `import fabledscryer` reference)
|
||||
- `fabledscryer/` → `steward/` (directory + every `from fabledscryer` / `import fabledscryer` reference)
|
||||
- `pyproject.toml` — name, entry point, hatch packages
|
||||
- `alembic.ini` — `script_location`
|
||||
- `tests/**` — imports (tests not executed, but must not break collection; update imports mechanically)
|
||||
|
||||
**Modified (PR 3 — config & env):**
|
||||
- `fabledscryer/config.py` → `roundtable/config.py` (already moved in PR 2) — read both `ROUNDTABLE_*` and `FABLEDSCRYER_*` with deprecation warning
|
||||
- `fabledscryer/config.py` → `steward/config.py` (already moved in PR 2) — read both `STEWARD_*` and `FABLEDSCRYER_*` with deprecation warning
|
||||
- `.env.example`, `config.example.yaml` — new names
|
||||
- First-boot migration of `~/.config/fabledscryer` → `~/.config/roundtable` (if the app uses it — verify during task)
|
||||
- First-boot migration of `~/.config/fabledscryer` → `~/.config/steward` (if the app uses it — verify during task)
|
||||
|
||||
**Modified (PR 4 — container + user strings):**
|
||||
- `Dockerfile` — COPY paths, CMD, image labels
|
||||
- `docker-compose.yml` — service name, image, container_name, volumes, env
|
||||
- `entrypoint.sh` — package path for `nut_setup.py` invocation
|
||||
- `roundtable/templates/base.html` — wordmark text "Fabled Scryer" → "Roundtable", `<title>`
|
||||
- `roundtable/templates/auth/login.html` — any residual "Fabled Scryer" text
|
||||
- `steward/templates/base.html` — wordmark text "Fabled Scryer" → "Steward", `<title>`
|
||||
- `steward/templates/auth/login.html` — any residual "Fabled Scryer" text
|
||||
- `README.md` — first-page references (full doc sweep is PR 5)
|
||||
|
||||
**Modified (PR 5 — docs):**
|
||||
- `README.md`, `docs/**/*.md`
|
||||
|
||||
**Modified (PR 6 — cleanup):**
|
||||
- `roundtable/config.py` — remove fallback shim
|
||||
- `steward/config.py` — remove fallback shim
|
||||
|
||||
---
|
||||
|
||||
@@ -197,7 +197,7 @@ git commit -m "style: candlelit glow background replaces star field"
|
||||
- [ ] **Step 1: Change the title block**
|
||||
|
||||
```html
|
||||
<title>{% block title %}Roundtable{% endblock %}</title>
|
||||
<title>{% block title %}Steward{% endblock %}</title>
|
||||
```
|
||||
|
||||
Note: the visible wordmark text still reads "Fabled Scryer" — that flips in PR 4. This only changes the browser tab title, which is acceptable to flip early since it's not visible inside the UI.
|
||||
@@ -263,7 +263,7 @@ Content for `fabledscryer/templates/errors/404.html`:
|
||||
|
||||
```html
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Not Found — Roundtable{% endblock %}
|
||||
{% block title %}Not Found — Steward{% endblock %}
|
||||
{% block content %}
|
||||
<div style="text-align:center; padding: 4rem 2rem; font-family: var(--font-serif);">
|
||||
<h1 style="color: var(--gold); font-size: 2.5rem; margin-bottom: 0.5rem;">404</h1>
|
||||
@@ -278,7 +278,7 @@ Content for `fabledscryer/templates/errors/500.html`:
|
||||
|
||||
```html
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Error — Roundtable{% endblock %}
|
||||
{% block title %}Error — Steward{% endblock %}
|
||||
{% block content %}
|
||||
<div style="text-align:center; padding: 4rem 2rem; font-family: var(--font-serif);">
|
||||
<h1 style="color: var(--red); font-size: 2.5rem; margin-bottom: 0.5rem;">500</h1>
|
||||
@@ -361,7 +361,7 @@ git add -A
|
||||
git commit -m "style: visual polish after manual review"
|
||||
```
|
||||
|
||||
**PR 1 done.** Open PR with title `feat: roundtable visual reskin (pewter & gold)`.
|
||||
**PR 1 done.** Open PR with title `feat: steward visual reskin (pewter & gold)`.
|
||||
|
||||
---
|
||||
|
||||
@@ -370,39 +370,39 @@ git commit -m "style: visual polish after manual review"
|
||||
### Task 10: Rename the package directory
|
||||
|
||||
**Files:**
|
||||
- Rename: `fabledscryer/` → `roundtable/`
|
||||
- Rename: `fabledscryer/` → `steward/`
|
||||
|
||||
- [ ] **Step 1: Rename the directory**
|
||||
|
||||
```bash
|
||||
git mv fabledscryer roundtable
|
||||
git mv fabledscryer steward
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Confirm the move**
|
||||
|
||||
```bash
|
||||
ls roundtable/ | head
|
||||
ls steward/ | head
|
||||
git status | head
|
||||
```
|
||||
|
||||
Expected: see `__init__.py`, `app.py`, `cli.py`, etc. inside `roundtable/`.
|
||||
Expected: see `__init__.py`, `app.py`, `cli.py`, etc. inside `steward/`.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git commit -m "refactor: rename package directory fabledscryer → roundtable"
|
||||
git commit -m "refactor: rename package directory fabledscryer → steward"
|
||||
```
|
||||
|
||||
### Task 11: Rewrite all `fabledscryer` imports to `roundtable`
|
||||
### Task 11: Rewrite all `fabledscryer` imports to `steward`
|
||||
|
||||
**Files:**
|
||||
- Modify: every file under `roundtable/`, `tests/`, `alembic.ini`, `entrypoint.sh`, `Dockerfile` that imports from the old package
|
||||
- Modify: every file under `steward/`, `tests/`, `alembic.ini`, `entrypoint.sh`, `Dockerfile` that imports from the old package
|
||||
|
||||
- [ ] **Step 1: Find every remaining `fabledscryer` reference in Python code**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
grep -rn "fabledscryer" roundtable/ tests/ alembic.ini entrypoint.sh Dockerfile pyproject.toml
|
||||
grep -rn "fabledscryer" steward/ tests/ alembic.ini entrypoint.sh Dockerfile pyproject.toml
|
||||
```
|
||||
|
||||
Keep the output visible as a checklist.
|
||||
@@ -410,24 +410,24 @@ Keep the output visible as a checklist.
|
||||
- [ ] **Step 2: Run a safe codemod across Python files**
|
||||
|
||||
```bash
|
||||
grep -rl "fabledscryer" roundtable/ tests/ | xargs sed -i 's/fabledscryer/roundtable/g'
|
||||
grep -rl "fabledscryer" steward/ tests/ | xargs sed -i 's/fabledscryer/steward/g'
|
||||
```
|
||||
|
||||
This touches only files under `roundtable/` and `tests/`. Do NOT run it across the whole repo yet — `docs/`, `.env.example`, `docker-compose.yml`, `config.example.yaml` and `README.md` are handled in later PRs.
|
||||
This touches only files under `steward/` and `tests/`. Do NOT run it across the whole repo yet — `docs/`, `.env.example`, `docker-compose.yml`, `config.example.yaml` and `README.md` are handled in later PRs.
|
||||
|
||||
- [ ] **Step 3: Spot-check a few critical files**
|
||||
|
||||
```bash
|
||||
grep -n "roundtable\|fabledscryer" roundtable/app.py roundtable/cli.py roundtable/config.py roundtable/migrations/env.py
|
||||
grep -n "steward\|fabledscryer" steward/app.py steward/cli.py steward/config.py steward/migrations/env.py
|
||||
```
|
||||
|
||||
Expected: only `roundtable` references remain; no stray `fabledscryer`.
|
||||
Expected: only `steward` references remain; no stray `fabledscryer`.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "refactor: update imports fabledscryer → roundtable"
|
||||
git commit -m "refactor: update imports fabledscryer → steward"
|
||||
```
|
||||
|
||||
### Task 12: Update `pyproject.toml`
|
||||
@@ -439,15 +439,15 @@ git commit -m "refactor: update imports fabledscryer → roundtable"
|
||||
|
||||
```toml
|
||||
[project]
|
||||
name = "roundtable"
|
||||
name = "steward"
|
||||
version = "0.1.0"
|
||||
# ... dependencies unchanged ...
|
||||
|
||||
[project.scripts]
|
||||
roundtable = "roundtable.cli:main"
|
||||
steward = "steward.cli:main"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["roundtable"]
|
||||
packages = ["steward"]
|
||||
```
|
||||
|
||||
Leave `[project.optional-dependencies]`, `[tool.pytest.ini_options]`, and all dependency pins untouched.
|
||||
@@ -456,7 +456,7 @@ Leave `[project.optional-dependencies]`, `[tool.pytest.ini_options]`, and all de
|
||||
|
||||
```bash
|
||||
git add pyproject.toml
|
||||
git commit -m "build: rename package to roundtable in pyproject"
|
||||
git commit -m "build: rename package to steward in pyproject"
|
||||
```
|
||||
|
||||
### Task 13: Update `alembic.ini`
|
||||
@@ -468,7 +468,7 @@ git commit -m "build: rename package to roundtable in pyproject"
|
||||
|
||||
```ini
|
||||
[alembic]
|
||||
script_location = roundtable/migrations
|
||||
script_location = steward/migrations
|
||||
prepend_sys_path = .
|
||||
version_path_separator = os
|
||||
```
|
||||
@@ -477,7 +477,7 @@ version_path_separator = os
|
||||
|
||||
```bash
|
||||
git add alembic.ini
|
||||
git commit -m "build: point alembic at roundtable/migrations"
|
||||
git commit -m "build: point alembic at steward/migrations"
|
||||
```
|
||||
|
||||
### Task 14: Verify the package installs and imports cleanly
|
||||
@@ -487,8 +487,8 @@ git commit -m "build: point alembic at roundtable/migrations"
|
||||
```bash
|
||||
python -m venv /tmp/rt-verify
|
||||
/tmp/rt-verify/bin/pip install -e .
|
||||
/tmp/rt-verify/bin/python -c "import roundtable; import roundtable.app; import roundtable.cli; print('ok')"
|
||||
/tmp/rt-verify/bin/roundtable --help
|
||||
/tmp/rt-verify/bin/python -c "import steward; import steward.app; import steward.cli; print('ok')"
|
||||
/tmp/rt-verify/bin/steward --help
|
||||
```
|
||||
|
||||
Expected: `ok` and CLI help output. No ImportError.
|
||||
@@ -501,20 +501,20 @@ rm -rf /tmp/rt-verify
|
||||
|
||||
- [ ] **Step 3: No commit needed (verification only).**
|
||||
|
||||
**PR 2 done.** Open PR with title `refactor: rename python package fabledscryer → roundtable`.
|
||||
**PR 2 done.** Open PR with title `refactor: rename python package fabledscryer → steward`.
|
||||
|
||||
---
|
||||
|
||||
## PR 3 — Config & Environment Variables
|
||||
|
||||
### Task 15: Add env-var fallback shim in `roundtable/config.py`
|
||||
### Task 15: Add env-var fallback shim in `steward/config.py`
|
||||
|
||||
**Files:**
|
||||
- Modify: `roundtable/config.py`
|
||||
- Modify: `steward/config.py`
|
||||
|
||||
- [ ] **Step 1: Replace the env var lookups with a fallback helper**
|
||||
|
||||
Open `roundtable/config.py` and replace the existing `load_bootstrap` body so it reads `ROUNDTABLE_*` first and falls back to `FABLEDSCRYER_*` with a deprecation warning.
|
||||
Open `steward/config.py` and replace the existing `load_bootstrap` body so it reads `STEWARD_*` first and falls back to `FABLEDSCRYER_*` with a deprecation warning.
|
||||
|
||||
```python
|
||||
def _env_with_fallback(new_name: str, old_name: str) -> str | None:
|
||||
@@ -536,19 +536,19 @@ Then inside `load_bootstrap`, replace the existing env lookups with:
|
||||
|
||||
```python
|
||||
database_url = (
|
||||
_env_with_fallback("ROUNDTABLE_DATABASE_URL", "FABLEDSCRYER_DATABASE_URL")
|
||||
or _env_with_fallback("ROUNDTABLE_DATABASE__URL", "FABLEDSCRYER_DATABASE__URL")
|
||||
_env_with_fallback("STEWARD_DATABASE_URL", "FABLEDSCRYER_DATABASE_URL")
|
||||
or _env_with_fallback("STEWARD_DATABASE__URL", "FABLEDSCRYER_DATABASE__URL")
|
||||
or raw.get("database", {}).get("url")
|
||||
)
|
||||
if not database_url:
|
||||
raise ValueError(
|
||||
"Database URL is required. Set ROUNDTABLE_DATABASE_URL env var "
|
||||
"Database URL is required. Set STEWARD_DATABASE_URL env var "
|
||||
"or add 'database.url' to config.yaml."
|
||||
)
|
||||
|
||||
# ...
|
||||
plugin_dir = (
|
||||
_env_with_fallback("ROUNDTABLE_PLUGIN_DIR", "FABLEDSCRYER_PLUGIN_DIR")
|
||||
_env_with_fallback("STEWARD_PLUGIN_DIR", "FABLEDSCRYER_PLUGIN_DIR")
|
||||
or raw.get("plugin_dir", "plugins")
|
||||
)
|
||||
```
|
||||
@@ -557,15 +557,15 @@ And in `_resolve_secret_key`:
|
||||
|
||||
```python
|
||||
from_env = (
|
||||
_env_with_fallback("ROUNDTABLE_SECRET_KEY", "FABLEDSCRYER_SECRET_KEY")
|
||||
_env_with_fallback("STEWARD_SECRET_KEY", "FABLEDSCRYER_SECRET_KEY")
|
||||
or raw.get("secret_key")
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Grep for any other `FABLEDSCRYER_` or `FABLEDNETMON_` env var reads across `roundtable/`**
|
||||
- [ ] **Step 2: Grep for any other `FABLEDSCRYER_` or `FABLEDNETMON_` env var reads across `steward/`**
|
||||
|
||||
```bash
|
||||
grep -rn "FABLEDSCRYER_\|FABLEDNETMON_" roundtable/
|
||||
grep -rn "FABLEDSCRYER_\|FABLEDNETMON_" steward/
|
||||
```
|
||||
|
||||
For each hit, apply the same pattern (new name first, old as fallback with warning). If `FABLEDNETMON_` is pure residue with no current consumer, delete the reference.
|
||||
@@ -573,8 +573,8 @@ For each hit, apply the same pattern (new name first, old as fallback with warni
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add roundtable/config.py
|
||||
git commit -m "feat: ROUNDTABLE_* env vars with FABLEDSCRYER_* fallback"
|
||||
git add steward/config.py
|
||||
git commit -m "feat: STEWARD_* env vars with FABLEDSCRYER_* fallback"
|
||||
```
|
||||
|
||||
### Task 16: Update `.env.example` and `config.example.yaml`
|
||||
@@ -588,11 +588,11 @@ git commit -m "feat: ROUNDTABLE_* env vars with FABLEDSCRYER_* fallback"
|
||||
grep -n "FABLEDSCRYER\|FABLEDNETMON" .env.example
|
||||
```
|
||||
|
||||
Replace every occurrence with `ROUNDTABLE_*`. Add a short comment at top:
|
||||
Replace every occurrence with `STEWARD_*`. Add a short comment at top:
|
||||
|
||||
```
|
||||
# Roundtable environment configuration.
|
||||
# Only ROUNDTABLE_DATABASE_URL is required. Other settings live in the DB.
|
||||
# Steward environment configuration.
|
||||
# Only STEWARD_DATABASE_URL is required. Other settings live in the DB.
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Rewrite `config.example.yaml`**
|
||||
@@ -600,25 +600,25 @@ Replace every occurrence with `ROUNDTABLE_*`. Add a short comment at top:
|
||||
Replace the header and example strings:
|
||||
|
||||
```yaml
|
||||
# Roundtable — Bootstrap Configuration Example
|
||||
# Steward — Bootstrap Configuration Example
|
||||
#
|
||||
# The only REQUIRED setting is the database URL.
|
||||
# Set it via env var (recommended for Docker):
|
||||
#
|
||||
# ROUNDTABLE_DATABASE_URL=postgresql+asyncpg://user:pass@host/db
|
||||
# STEWARD_DATABASE_URL=postgresql+asyncpg://user:pass@host/db
|
||||
#
|
||||
# All other settings are stored in the database and managed via
|
||||
# the Settings UI at /settings/.
|
||||
|
||||
database:
|
||||
url: "postgresql+asyncpg://roundtable:password@localhost/roundtable"
|
||||
url: "postgresql+asyncpg://steward:password@localhost/steward"
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add .env.example config.example.yaml
|
||||
git commit -m "docs: ROUNDTABLE_* env var examples"
|
||||
git commit -m "docs: STEWARD_* env var examples"
|
||||
```
|
||||
|
||||
### Task 17: Manual config verification
|
||||
@@ -626,8 +626,8 @@ git commit -m "docs: ROUNDTABLE_* env var examples"
|
||||
- [ ] **Step 1: Run the app with only the new env var**
|
||||
|
||||
```bash
|
||||
ROUNDTABLE_DATABASE_URL=postgresql+asyncpg://fabledscryer:fabledscryer@localhost/fabledscryer \
|
||||
python -m roundtable.cli --host 127.0.0.1 --port 5001
|
||||
STEWARD_DATABASE_URL=postgresql+asyncpg://fabledscryer:fabledscryer@localhost/fabledscryer \
|
||||
python -m steward.cli --host 127.0.0.1 --port 5001
|
||||
```
|
||||
|
||||
(Use whatever local DB you have; the old db name is fine — data isn't being renamed.)
|
||||
@@ -638,14 +638,14 @@ Expected: app starts, no deprecation warning.
|
||||
|
||||
```bash
|
||||
FABLEDSCRYER_DATABASE_URL=postgresql+asyncpg://fabledscryer:fabledscryer@localhost/fabledscryer \
|
||||
python -m roundtable.cli --host 127.0.0.1 --port 5001
|
||||
python -m steward.cli --host 127.0.0.1 --port 5001
|
||||
```
|
||||
|
||||
Expected: app starts, deprecation warning logged once.
|
||||
|
||||
- [ ] **Step 3: Kill processes.** No commit.
|
||||
|
||||
**PR 3 done.** Open PR with title `feat: ROUNDTABLE_* env vars (FABLEDSCRYER_* fallback)`.
|
||||
**PR 3 done.** Open PR with title `feat: STEWARD_* env vars (FABLEDSCRYER_* fallback)`.
|
||||
|
||||
---
|
||||
|
||||
@@ -664,12 +664,12 @@ COPY fabledscryer/ fabledscryer/
|
||||
```
|
||||
to
|
||||
```dockerfile
|
||||
COPY roundtable/ roundtable/
|
||||
COPY steward/ steward/
|
||||
```
|
||||
|
||||
Change the CMD:
|
||||
```dockerfile
|
||||
CMD ["roundtable", "--host", "0.0.0.0", "--port", "5000"]
|
||||
CMD ["steward", "--host", "0.0.0.0", "--port", "5000"]
|
||||
```
|
||||
|
||||
Leave everything else (apt packages, gosu, app user, EXPOSE 5000) untouched.
|
||||
@@ -678,7 +678,7 @@ Leave everything else (apt packages, gosu, app user, EXPOSE 5000) untouched.
|
||||
|
||||
```bash
|
||||
git add Dockerfile
|
||||
git commit -m "build: update Dockerfile for roundtable package"
|
||||
git commit -m "build: update Dockerfile for steward package"
|
||||
```
|
||||
|
||||
### Task 19: Update `entrypoint.sh`
|
||||
@@ -688,15 +688,15 @@ git commit -m "build: update Dockerfile for roundtable package"
|
||||
|
||||
- [ ] **Step 1: Replace the `nut_setup.py` invocation path**
|
||||
|
||||
Change every `/app/fabledscryer/nut_setup.py` to `/app/roundtable/nut_setup.py`.
|
||||
Change every `/app/fabledscryer/nut_setup.py` to `/app/steward/nut_setup.py`.
|
||||
|
||||
Update any comment that says "FabledScryer container entrypoint." to "Roundtable container entrypoint."
|
||||
Update any comment that says "FabledScryer container entrypoint." to "Steward container entrypoint."
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
|
||||
```bash
|
||||
git add entrypoint.sh
|
||||
git commit -m "build: entrypoint.sh uses roundtable package path"
|
||||
git commit -m "build: entrypoint.sh uses steward package path"
|
||||
```
|
||||
|
||||
### Task 20: Update `docker-compose.yml`
|
||||
@@ -708,10 +708,10 @@ git commit -m "build: entrypoint.sh uses roundtable package path"
|
||||
|
||||
```yaml
|
||||
services:
|
||||
roundtable:
|
||||
steward:
|
||||
build: .
|
||||
container_name: roundtable
|
||||
image: roundtable:latest
|
||||
container_name: steward
|
||||
image: steward:latest
|
||||
ports:
|
||||
- "5000:5000"
|
||||
volumes:
|
||||
@@ -720,7 +720,7 @@ services:
|
||||
- /mnt/Data/traefik/log:/var/log/traefik:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
environment:
|
||||
- ROUNDTABLE_DATABASE_URL=postgresql+asyncpg://roundtable:roundtable@db/roundtable
|
||||
- STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@db/steward
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
@@ -729,13 +729,13 @@ services:
|
||||
db:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_USER: roundtable
|
||||
POSTGRES_PASSWORD: roundtable
|
||||
POSTGRES_DB: roundtable
|
||||
POSTGRES_USER: steward
|
||||
POSTGRES_PASSWORD: steward
|
||||
POSTGRES_DB: steward
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U roundtable"]
|
||||
test: ["CMD-SHELL", "pg_isready -U steward"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -752,37 +752,37 @@ volumes:
|
||||
|
||||
```bash
|
||||
git add docker-compose.yml
|
||||
git commit -m "build: docker-compose service → roundtable"
|
||||
git commit -m "build: docker-compose service → steward"
|
||||
```
|
||||
|
||||
### Task 21: Flip user-visible wordmark and title
|
||||
|
||||
**Files:**
|
||||
- Modify: `roundtable/templates/base.html`
|
||||
- Modify: `steward/templates/base.html`
|
||||
|
||||
- [ ] **Step 1: Update the `<title>` block (around line 6)**
|
||||
|
||||
```html
|
||||
<title>{% block title %}Roundtable{% endblock %}</title>
|
||||
<title>{% block title %}Steward{% endblock %}</title>
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Update the nav wordmark (around line 214)**
|
||||
|
||||
Change `Fabled Scryer` inside `nav .brand` to `Roundtable`.
|
||||
Change `Fabled Scryer` inside `nav .brand` to `Steward`.
|
||||
|
||||
- [ ] **Step 3: Grep for any other user-visible "Fabled Scryer" strings in templates**
|
||||
|
||||
```bash
|
||||
grep -rn "Fabled Scryer\|FabledScryer" roundtable/templates/
|
||||
grep -rn "Fabled Scryer\|FabledScryer" steward/templates/
|
||||
```
|
||||
|
||||
Replace each with "Roundtable".
|
||||
Replace each with "Steward".
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add roundtable/templates/
|
||||
git commit -m "copy: flip visible wordmark → Roundtable"
|
||||
git add steward/templates/
|
||||
git commit -m "copy: flip visible wordmark → Steward"
|
||||
```
|
||||
|
||||
### Task 22: Manual container verification
|
||||
@@ -795,8 +795,8 @@ docker compose up --build
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Verify**
|
||||
- Container named `roundtable` runs
|
||||
- Browser shows "Roundtable" in tab title and nav
|
||||
- Container named `steward` runs
|
||||
- Browser shows "Steward" in tab title and nav
|
||||
- Login, dashboard, and error pages all render with the new palette and wordmark
|
||||
- Logs show no import errors and no unhandled deprecation warnings
|
||||
|
||||
@@ -806,7 +806,7 @@ docker compose up --build
|
||||
docker compose down
|
||||
```
|
||||
|
||||
**PR 4 done.** Open PR with title `feat: roundtable container & wordmark flip`.
|
||||
**PR 4 done.** Open PR with title `feat: steward container & wordmark flip`.
|
||||
|
||||
---
|
||||
|
||||
@@ -828,9 +828,9 @@ grep -rn "fabledscryer\|FabledScryer\|Fabled Scryer" README.md docs/
|
||||
```bash
|
||||
grep -rl "fabledscryer\|FabledScryer\|Fabled Scryer" README.md docs/ \
|
||||
| xargs sed -i \
|
||||
-e 's/FabledScryer/Roundtable/g' \
|
||||
-e 's/fabledscryer/roundtable/g' \
|
||||
-e 's/Fabled Scryer/Roundtable/g'
|
||||
-e 's/FabledScryer/Steward/g' \
|
||||
-e 's/fabledscryer/steward/g' \
|
||||
-e 's/Fabled Scryer/Steward/g'
|
||||
```
|
||||
|
||||
Then read each changed file top-to-bottom and fix any mangled sentences (e.g. capitalization at the start of sentences, code blocks that now reference a directory that still needs a `./` prefix, etc.). Pay extra attention to:
|
||||
@@ -843,12 +843,12 @@ Then read each changed file top-to-bottom and fix any mangled sentences (e.g. ca
|
||||
|
||||
```bash
|
||||
git add README.md docs/
|
||||
git commit -m "docs: roundtable rename sweep"
|
||||
git commit -m "docs: steward rename sweep"
|
||||
```
|
||||
|
||||
### Task 24: Rename the Forgejo repo
|
||||
|
||||
- [ ] **Step 1: Via Forgejo UI or API, rename the repo from `FabledScryer` to `Roundtable`.** The old URL will redirect for one grace period.
|
||||
- [ ] **Step 1: Via Forgejo UI or API, rename the repo from `FabledScryer` to `Steward`.** The old URL will redirect for one grace period.
|
||||
|
||||
- [ ] **Step 2: Update your local remote URL**
|
||||
|
||||
@@ -862,13 +862,13 @@ git remote -v
|
||||
```bash
|
||||
# from the parent dir
|
||||
cd ..
|
||||
mv FabledScryer Roundtable
|
||||
cd Roundtable
|
||||
mv FabledScryer Steward
|
||||
cd Steward
|
||||
```
|
||||
|
||||
- [ ] **Step 4: No commit.**
|
||||
|
||||
**PR 5 done.** Open PR with title `docs: roundtable rename sweep`.
|
||||
**PR 5 done.** Open PR with title `docs: steward rename sweep`.
|
||||
|
||||
---
|
||||
|
||||
@@ -877,11 +877,11 @@ cd Roundtable
|
||||
### Task 25: Remove env var fallback shim
|
||||
|
||||
**Files:**
|
||||
- Modify: `roundtable/config.py`
|
||||
- Modify: `steward/config.py`
|
||||
|
||||
- [ ] **Step 1: Delete `_env_with_fallback` and inline direct `os.environ.get("ROUNDTABLE_*")` lookups**
|
||||
- [ ] **Step 1: Delete `_env_with_fallback` and inline direct `os.environ.get("STEWARD_*")` lookups**
|
||||
|
||||
Replace each `_env_with_fallback("ROUNDTABLE_X", "FABLEDSCRYER_X")` call with `os.environ.get("ROUNDTABLE_X")`. Delete the helper.
|
||||
Replace each `_env_with_fallback("STEWARD_X", "FABLEDSCRYER_X")` call with `os.environ.get("STEWARD_X")`. Delete the helper.
|
||||
|
||||
- [ ] **Step 2: Grep to confirm no `FABLEDSCRYER_` references remain**
|
||||
|
||||
@@ -894,7 +894,7 @@ Expected: zero hits (or only historical commit messages, which don't show up in
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add roundtable/config.py
|
||||
git add steward/config.py
|
||||
git commit -m "refactor: remove FABLEDSCRYER_* env var fallback shim"
|
||||
```
|
||||
|
||||
@@ -923,5 +923,5 @@ git commit -m "refactor: final rename sweep"
|
||||
|
||||
- Spec coverage: all 5 spec sections mapped to PRs 1–5; PR 6 handles the cleanup implied by the shim in PR 3.
|
||||
- No placeholders — every template replacement includes concrete code; every grep includes the actual pattern.
|
||||
- Type consistency: config helper is `_env_with_fallback` in both Task 15 and Task 25; env var names are `ROUNDTABLE_DATABASE_URL`, `ROUNDTABLE_PLUGIN_DIR`, `ROUNDTABLE_SECRET_KEY` consistently.
|
||||
- Type consistency: config helper is `_env_with_fallback` in both Task 15 and Task 25; env var names are `STEWARD_DATABASE_URL`, `STEWARD_PLUGIN_DIR`, `STEWARD_SECRET_KEY` consistently.
|
||||
- Tests intentionally skipped per project feedback; verification is manual (browser + container).
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,21 +1,21 @@
|
||||
# Roundtable Rebrand — Design
|
||||
# Steward Rebrand — Design
|
||||
|
||||
**Date:** 2026-04-13
|
||||
**Status:** Approved for planning
|
||||
|
||||
## 1. Identity
|
||||
|
||||
**Name:** Roundtable (dropping the "Fabled" prefix).
|
||||
**Name:** Steward (dropping the "Fabled" prefix).
|
||||
|
||||
**Why the change:** The old name `FabledScryer` framed the app as a single seer peering into one crystal ball. The actual product is a gathering point — hosts, metrics, alerts, plugins, and dashboards from across the homelab converge here. "Roundtable" captures that: a place where the realm's tools and information meet.
|
||||
**Why the change:** The old name `FabledScryer` framed the app as a single seer peering into one crystal ball. The actual product is a gathering point — hosts, metrics, alerts, plugins, and dashboards from across the homelab converge here. "Steward" captures that: a place where the realm's tools and information meet.
|
||||
|
||||
**Name conflict check:**
|
||||
- *Roundtable Software* — a Progress ABL tooling vendor. Different demographic, no overlap.
|
||||
- *Steward Software* — a Progress ABL tooling vendor. Different demographic, no overlap.
|
||||
- *Fabled.gg* — a virtual tabletop product. Closest concern, and the reason the "Fabled" prefix is dropped entirely to avoid confusion inside the tabletop/fantasy space.
|
||||
|
||||
**Umbrella:** `fabledsword.com` remains the family umbrella domain; `git.fabledsword.com` remains the plugin catalog host. Roundtable becomes one product under that umbrella.
|
||||
**Umbrella:** `fabledsword.com` remains the family umbrella domain; `git.fabledsword.com` remains the plugin catalog host. Steward becomes one product under that umbrella.
|
||||
|
||||
**Tone:** Heraldic / Arthurian register. The roundtable is the gathering place of those who keep watch over the realm.
|
||||
**Tone:** Heraldic / Arthurian register. The steward is the gathering place of those who keep watch over the realm.
|
||||
|
||||
## 2. Visual System
|
||||
|
||||
@@ -78,7 +78,7 @@ Candlelit glow — a soft, warm radial wash behind the main content area. Replac
|
||||
|
||||
### Wordmark
|
||||
|
||||
"Roundtable" in EB Garamond, gold (`#c8a840`), paired with the seal mark to the left in the nav header.
|
||||
"Steward" in EB Garamond, gold (`#c8a840`), paired with the seal mark to the left in the nav header.
|
||||
|
||||
## 3. Voice & Copy
|
||||
|
||||
@@ -100,21 +100,21 @@ Candlelit glow — a soft, warm radial wash behind the main content area. Replac
|
||||
## 4. Rename Mechanics
|
||||
|
||||
**Code & packaging**
|
||||
- `fabledscryer/` package directory → `roundtable/`
|
||||
- All `from fabledscryer...` / `import fabledscryer...` → `roundtable`
|
||||
- `pyproject.toml`: project name, CLI entry point (`fabledscryer = "fabledscryer.cli:main"` → `roundtable = "roundtable.cli:main"`), tool sections
|
||||
- `fabledscryer/` package directory → `steward/`
|
||||
- All `from fabledscryer...` / `import fabledscryer...` → `steward`
|
||||
- `pyproject.toml`: project name, CLI entry point (`fabledscryer = "fabledscryer.cli:main"` → `steward = "steward.cli:main"`), tool sections
|
||||
- `__init__.py` package metadata
|
||||
|
||||
**Runtime config**
|
||||
- Env vars: `FABLEDSCRYER_*` → `ROUNDTABLE_*`
|
||||
- Env vars: `FABLEDSCRYER_*` → `STEWARD_*`
|
||||
- Residual `FABLEDNETMON_*` env vars removed
|
||||
- Config dir: `~/.config/fabledscryer` → `~/.config/roundtable` (one-shot copy on first boot)
|
||||
- Config dir: `~/.config/fabledscryer` → `~/.config/steward` (one-shot copy on first boot)
|
||||
- Log file names, systemd unit names if any
|
||||
|
||||
**Container & deploy**
|
||||
- `Dockerfile` labels, workdir
|
||||
- `docker-compose.yml` service name, image tag, volume names, `container_name`
|
||||
- Published image: `fabledscryer:latest` → `roundtable:latest`
|
||||
- Published image: `fabledscryer:latest` → `steward:latest`
|
||||
|
||||
**Database**
|
||||
- Alembic `script_location` and any customized version table
|
||||
@@ -144,10 +144,10 @@ Staged so `main` stays runnable between PRs.
|
||||
New palette tokens, locked logo SVG, EB Garamond font swap, candlelit glow background, themed edge copy (login, empty states, 404/500, dashboard hero "Under Watch, Under Care"). Still named FabledScryer internally. Ships independently, low risk.
|
||||
|
||||
**PR 2 — Python package rename**
|
||||
`fabledscryer/` → `roundtable/`, all imports, `pyproject.toml` name + entry point, `__init__.py` metadata. Docker/config untouched. Package installable as `roundtable` locally.
|
||||
`fabledscryer/` → `steward/`, all imports, `pyproject.toml` name + entry point, `__init__.py` metadata. Docker/config untouched. Package installable as `steward` locally.
|
||||
|
||||
**PR 3 — Config & env vars**
|
||||
`FABLEDSCRYER_*` → `ROUNDTABLE_*` with a short-lived fallback reader (accept both, warn on old) so self-hosters don't break mid-upgrade. Config dir migration on first boot. Drop `FABLEDNETMON_*` residue.
|
||||
`FABLEDSCRYER_*` → `STEWARD_*` with a short-lived fallback reader (accept both, warn on old) so self-hosters don't break mid-upgrade. Config dir migration on first boot. Drop `FABLEDNETMON_*` residue.
|
||||
|
||||
**PR 4 — Container & deploy**
|
||||
`Dockerfile`, `docker-compose.yml` service/image/volume/container names. Published image tag switch. Template wordmark + `<title>` + meta (the last user-visible string flip).
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Roundtable container entrypoint.
|
||||
# Steward container entrypoint.
|
||||
#
|
||||
# Drops from root to the 'app' user via gosu, then runs the command.
|
||||
# The container starts as root only so /data permissions can be fixed up
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# plugins/__init__.py
|
||||
@@ -0,0 +1,71 @@
|
||||
# plugins/docker/__init__.py
|
||||
from __future__ import annotations
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from quart import Quart
|
||||
|
||||
_app: "Quart | None" = None
|
||||
|
||||
|
||||
def setup(app: "Quart") -> None:
|
||||
global _app
|
||||
_app = app
|
||||
from .models import DockerContainer, DockerMetric # noqa: F401 — register with Base
|
||||
|
||||
# Publish the per-host persist hook so the host agent can hand off the
|
||||
# `docker` array from each sample without importing our models (opportunistic
|
||||
# synergy — host_agent gates on has_capability and no-ops if we're disabled).
|
||||
# required_role=viewer: this is a trusted server-side data-plane write driven
|
||||
# by the authenticated agent ingest, not a user-facing privileged action.
|
||||
from steward.core.capabilities import register_capability
|
||||
from steward.models.users import UserRole
|
||||
from .ingest import persist_host_docker
|
||||
from .retention import run_docker_retention
|
||||
register_capability(
|
||||
"docker.persist_host_samples", persist_host_docker,
|
||||
label="Persist host Docker samples",
|
||||
description="Store per-host container state + metrics pushed by the host agent.",
|
||||
required_role=UserRole.viewer,
|
||||
)
|
||||
# Roll up + prune Docker time-series, driven by the core cleanup task without
|
||||
# it importing our models. Same trusted server-side data-plane role as above.
|
||||
register_capability(
|
||||
"docker.run_retention", run_docker_retention,
|
||||
label="Run Docker retention",
|
||||
description="Roll up old docker_metrics to hourly + prune stale metrics/events.",
|
||||
required_role=UserRole.viewer,
|
||||
)
|
||||
|
||||
|
||||
def get_scheduled_tasks() -> list:
|
||||
# Collection is now agent-driven (pushed to the host_agent ingest); the
|
||||
# central socket scrape was removed. No periodic task to register.
|
||||
return []
|
||||
|
||||
|
||||
def get_blueprint():
|
||||
from .routes import docker_bp
|
||||
return docker_bp
|
||||
|
||||
|
||||
def register_mcp_tools(register) -> None:
|
||||
from .mcp_tools import register_tools
|
||||
register_tools(register)
|
||||
|
||||
|
||||
async def mcp_overview(db) -> list[dict]:
|
||||
from .mcp_tools import mcp_overview as findings
|
||||
return await findings(db)
|
||||
|
||||
|
||||
def get_nav() -> list[dict]:
|
||||
# Surfaces in the sidebar's Infrastructure group (fleet-wide container view).
|
||||
return [{"label": "Docker", "href": "/plugins/docker/"}]
|
||||
|
||||
|
||||
def get_alert_metrics() -> dict[str, list[str]]:
|
||||
"""What ingest.py records per container (resource "<host>/<container>"):
|
||||
restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 /
|
||||
0.0 from the container's HEALTHCHECK, only for containers that define one."""
|
||||
return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy"]}
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Model-free Docker view helpers.
|
||||
|
||||
Kept separate from routes.py/models.py so tests can import it directly: importing
|
||||
a module that re-imports plugins.docker.models AFTER the app has loaded the docker
|
||||
plugin raises "Table 'docker_containers' is already defined". This module touches
|
||||
no ORM models, so it's safe to import in either environment.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def dedup_by_container_id(containers: list) -> list:
|
||||
"""Collapse containers double-reported across hosts. Swarm-aware agents on
|
||||
every manager list cluster-wide tasks, so the same container (identical
|
||||
container_id) arrives once per manager and would otherwise be counted N times.
|
||||
Keep the first occurrence per non-empty container_id — callers order
|
||||
running-first, so a running report wins over a stale stopped one. Rows without
|
||||
a container_id (older agents) can't be matched, so they're kept as-is."""
|
||||
seen: set[str] = set()
|
||||
out = []
|
||||
for c in containers:
|
||||
cid = (c.container_id or "").strip()
|
||||
if cid:
|
||||
if cid in seen:
|
||||
continue
|
||||
seen.add(cid)
|
||||
out.append(c)
|
||||
return out
|
||||
@@ -0,0 +1,466 @@
|
||||
# plugins/docker/ingest.py
|
||||
"""Persist host-scoped Docker samples pushed by the host agent.
|
||||
|
||||
Published as the "docker.persist_host_samples" capability (see __init__.setup),
|
||||
so the host_agent plugin can hand off a sample's `docker` array (and, on a swarm
|
||||
manager, its `swarm` object) WITHOUT importing the docker models — the coupling
|
||||
is opportunistic and degrades to a no-op when the docker plugin is disabled.
|
||||
Runs inside the caller's open transaction (the ingest handler's SAVEPOINT);
|
||||
never opens or commits its own.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
|
||||
# Stopped/terminal container states (anything not "running"/"paused"/"restarting").
|
||||
_STOPPED_STATES = {"exited", "dead", "stopped"}
|
||||
|
||||
|
||||
def _parse_started_at(value) -> datetime | None:
|
||||
"""Parse the agent's ISO-8601 started_at string back to a datetime."""
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _derive_events(old_state: dict, new_containers: list) -> list:
|
||||
"""Diff a fresh snapshot against stored per-container state → lifecycle events.
|
||||
|
||||
Pure function (no DB) so it's unit-testable. `old_state` maps name → the
|
||||
previously stored {status, health, oom_killed, exit_code, restart_count}; `new_containers`
|
||||
is the newest snapshot's list of container dicts. Returns a list of
|
||||
(container_name, event, detail) tuples:
|
||||
|
||||
start — a container transitions into running (or a genuinely new
|
||||
container appears already running)
|
||||
die — also for a new container first seen already exited
|
||||
non-zero: a task that crashed between two samples (#5049)
|
||||
restart — Docker restarted it (its restart count went up); said
|
||||
instead of a start, since that is what happened
|
||||
stop — running → not-running, or a running container is removed
|
||||
die — same as stop but with a non-zero exit code (abnormal)
|
||||
oom — OOMKilled flips False→True
|
||||
health_change — HEALTHCHECK status string changes
|
||||
|
||||
The CALLER skips this entirely on a host's first-ever snapshot (empty
|
||||
old_state) so the baseline doesn't emit a spurious "start" per existing
|
||||
container — a later-appearing container still gets one because by then
|
||||
old_state is populated and that container's prior entry is simply absent.
|
||||
"""
|
||||
events: list = []
|
||||
new_by_name = {c["name"]: c for c in new_containers if c.get("name")}
|
||||
|
||||
for name, c in new_by_name.items():
|
||||
new_status = (c.get("status") or "").lower()
|
||||
new_running = new_status == "running"
|
||||
new_oom = bool(c.get("oom_killed"))
|
||||
new_health = c.get("health")
|
||||
new_exit = c.get("exit_code")
|
||||
old = old_state.get(name)
|
||||
|
||||
if old is None:
|
||||
# Newly observed container: a start, or, when it is already down
|
||||
# with a non-zero exit, the death that happened between samples.
|
||||
if new_running:
|
||||
events.append((name, "start", c.get("image") or None))
|
||||
elif new_status in _STOPPED_STATES and new_exit not in (None, 0):
|
||||
events.append((name, "die", f"exit {new_exit}"))
|
||||
continue
|
||||
|
||||
old_running = (old.get("status") or "").lower() == "running"
|
||||
new_restarts = c.get("restart_count") or 0
|
||||
old_restarts = old.get("restart_count") or 0
|
||||
if new_restarts > old_restarts:
|
||||
gained = new_restarts - old_restarts
|
||||
events.append((name, "restart", f"restart #{new_restarts}" if gained == 1
|
||||
else f"{gained} restarts, now #{new_restarts}"))
|
||||
if new_running and not old_running and new_restarts <= old_restarts:
|
||||
events.append((name, "start", c.get("image") or None))
|
||||
elif old_running and not new_running:
|
||||
if new_exit not in (None, 0):
|
||||
events.append((name, "die", f"exit {new_exit}"))
|
||||
else:
|
||||
events.append((name, "stop", None))
|
||||
|
||||
if new_oom and not bool(old.get("oom_killed")):
|
||||
events.append((name, "oom", None))
|
||||
|
||||
if new_health != old.get("health") and (new_health or old.get("health")):
|
||||
events.append((name, "health_change",
|
||||
f"{old.get('health') or '?'}→{new_health or '?'}"))
|
||||
|
||||
# A running container that vanished from the listing entirely (removed).
|
||||
for name, old in old_state.items():
|
||||
if name not in new_by_name and (old.get("status") or "").lower() == "running":
|
||||
events.append((name, "stop", "removed"))
|
||||
|
||||
return events
|
||||
|
||||
|
||||
def _log_rows(log_batches, host_id: str):
|
||||
"""Pure: flatten the agent's log batches → docker_logs row kwargs (no DB).
|
||||
|
||||
`log_batches` is a list of (recorded_at, records) where each record is
|
||||
{"container", "stream", "ts", "line"}. A line's own Docker timestamp wins;
|
||||
recorded_at is the fallback when the agent couldn't parse one. Drops the
|
||||
agent's advisory truncation marker (container "_steward" — it signals a
|
||||
deferral, not a real container), malformed records, and lineless records;
|
||||
normalises an unknown stream to stdout. Unit-testable in isolation.
|
||||
"""
|
||||
for recorded_at, records in log_batches:
|
||||
if not isinstance(records, list):
|
||||
continue
|
||||
for rec in records:
|
||||
if not isinstance(rec, dict):
|
||||
continue
|
||||
name = rec.get("container")
|
||||
line = rec.get("line")
|
||||
if not name or name == "_steward" or line is None:
|
||||
continue
|
||||
ts = _parse_started_at(rec.get("ts")) or recorded_at
|
||||
stream = rec.get("stream")
|
||||
if stream not in ("stdout", "stderr"):
|
||||
stream = "stdout"
|
||||
yield {"host_id": host_id, "container_name": str(name)[:255],
|
||||
"ts": ts, "stream": stream, "line": str(line)}
|
||||
|
||||
|
||||
async def _persist_logs(session, host, log_batches) -> None:
|
||||
"""Append pushed container log lines (one row per line) for this host.
|
||||
|
||||
Time-series / append-only — the per-container size+age ring (retention)
|
||||
bounds growth, so a chatty container just keeps a shorter window. The global
|
||||
toggle + exclude list (Settings, no restart) are enforced here: this is the
|
||||
authoritative drop point, since the push model has no channel to tell an
|
||||
agent to stop collecting.
|
||||
"""
|
||||
from steward.core.settings import get_setting
|
||||
from .models import DockerLog
|
||||
|
||||
if not await get_setting(session, "docker.logs.enabled"):
|
||||
return
|
||||
exclude = set(await get_setting(session, "docker.logs.exclude") or ())
|
||||
for row in _log_rows(log_batches, host.id):
|
||||
if row["container_name"] in exclude:
|
||||
continue
|
||||
session.add(DockerLog(**row))
|
||||
|
||||
|
||||
async def _persist_swarm(session, host, swarm: dict) -> None:
|
||||
"""Upsert this manager's swarm topology; drop rows no longer reported.
|
||||
|
||||
Current-state tables (not time-series): a manager re-reports its full
|
||||
services/nodes set every sample, so we upsert what's present and delete what
|
||||
isn't (scoped to this host so two managers don't clobber each other).
|
||||
"""
|
||||
from datetime import timezone
|
||||
from .models import DockerSwarmNode, DockerSwarmService
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
services = swarm.get("services") or []
|
||||
seen_services: set[str] = set()
|
||||
for s in services:
|
||||
name = s.get("service_name")
|
||||
if not name:
|
||||
continue
|
||||
seen_services.add(name)
|
||||
row = await session.get(DockerSwarmService, (host.id, name))
|
||||
if row is None:
|
||||
row = DockerSwarmService(host_id=host.id, service_name=name)
|
||||
session.add(row)
|
||||
row.mode = s.get("mode") or "replicated"
|
||||
row.desired = int(s.get("desired") or 0)
|
||||
row.running = int(s.get("running") or 0)
|
||||
row.image = s.get("image")
|
||||
row.placement_json = json.dumps(s.get("placement") or [])
|
||||
row.tasks_json = json.dumps(s.get("tasks") or [])
|
||||
row.updated_at = now
|
||||
stale_services = delete(DockerSwarmService).where(
|
||||
DockerSwarmService.host_id == host.id)
|
||||
if seen_services:
|
||||
stale_services = stale_services.where(
|
||||
DockerSwarmService.service_name.notin_(seen_services))
|
||||
await session.execute(stale_services)
|
||||
|
||||
nodes = swarm.get("nodes") or []
|
||||
seen_nodes: set[str] = set()
|
||||
for n in nodes:
|
||||
nid = n.get("node_id")
|
||||
if not nid:
|
||||
continue
|
||||
seen_nodes.add(nid)
|
||||
row = await session.get(DockerSwarmNode, (host.id, nid))
|
||||
if row is None:
|
||||
row = DockerSwarmNode(host_id=host.id, node_id=nid)
|
||||
session.add(row)
|
||||
row.hostname = n.get("hostname") or ""
|
||||
row.role = n.get("role") or "worker"
|
||||
row.availability = n.get("availability") or "active"
|
||||
row.status = n.get("status") or "unknown"
|
||||
row.leader = bool(n.get("leader", False))
|
||||
row.updated_at = now
|
||||
stale_nodes = delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id)
|
||||
if seen_nodes:
|
||||
stale_nodes = stale_nodes.where(DockerSwarmNode.node_id.notin_(seen_nodes))
|
||||
await session.execute(stale_nodes)
|
||||
|
||||
|
||||
async def _persist_disk(session, host, disk: dict) -> None:
|
||||
"""Upsert this host's /system/df summary + replace its image storage rows.
|
||||
|
||||
Current-state (not time-series): the agent re-reports the full summary each
|
||||
interval, so we overwrite the single summary row and replace the image set
|
||||
(delete rows no longer present, upsert the rest), scoped to this host.
|
||||
"""
|
||||
from datetime import timezone
|
||||
from .models import DockerDiskUsage, DockerImage
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
summary = await session.get(DockerDiskUsage, host.id)
|
||||
if summary is None:
|
||||
summary = DockerDiskUsage(host_id=host.id)
|
||||
session.add(summary)
|
||||
summary.layers_size = int(disk.get("layers_size") or 0)
|
||||
summary.images_size = int(disk.get("images_size") or 0)
|
||||
summary.images_reclaimable = int(disk.get("images_reclaimable") or 0)
|
||||
summary.containers_size = int(disk.get("containers_size") or 0)
|
||||
summary.volumes_size = int(disk.get("volumes_size") or 0)
|
||||
summary.build_cache_size = int(disk.get("build_cache_size") or 0)
|
||||
summary.images_total = int(disk.get("images_total") or 0)
|
||||
summary.images_active = int(disk.get("images_active") or 0)
|
||||
summary.containers_count = int(disk.get("containers_count") or 0)
|
||||
summary.volumes_count = int(disk.get("volumes_count") or 0)
|
||||
summary.updated_at = now
|
||||
|
||||
images = disk.get("images") or []
|
||||
seen: set[str] = set()
|
||||
for im in images:
|
||||
iid = im.get("image_id")
|
||||
if not iid or iid in seen:
|
||||
continue
|
||||
seen.add(iid)
|
||||
row = await session.get(DockerImage, (host.id, iid))
|
||||
if row is None:
|
||||
row = DockerImage(host_id=host.id, image_id=iid)
|
||||
session.add(row)
|
||||
row.repo_tag = (im.get("repo_tag") or "<none>")[:512]
|
||||
row.size = int(im.get("size") or 0)
|
||||
row.shared_size = int(im.get("shared_size") or 0)
|
||||
row.containers = int(im.get("containers") or 0)
|
||||
row.updated_at = now
|
||||
stale = delete(DockerImage).where(DockerImage.host_id == host.id)
|
||||
if seen:
|
||||
stale = stale.where(DockerImage.image_id.notin_(seen))
|
||||
await session.execute(stale)
|
||||
|
||||
|
||||
async def _persist_status(session, host, status: dict) -> None:
|
||||
"""Record whether Docker answered on this host, and why not.
|
||||
|
||||
A host reporting "unavailable" with no row yet never had Docker: nothing to
|
||||
record. Once a host has had Docker, losing it is recorded, so its last
|
||||
containers read as stale rather than as running."""
|
||||
from .models import DockerHostStatus
|
||||
|
||||
row = await session.get(DockerHostStatus, host.id)
|
||||
if row is None:
|
||||
if status["status"] == "unavailable":
|
||||
return
|
||||
row = DockerHostStatus(host_id=host.id)
|
||||
session.add(row)
|
||||
if row.status != status["status"]:
|
||||
row.since = status["at"]
|
||||
row.status = status["status"]
|
||||
row.detail = (status.get("detail") or "")[:255] or None
|
||||
row.reported_at = status["at"]
|
||||
|
||||
|
||||
async def _expire_swarm(session, host) -> None:
|
||||
"""Drop the swarm rows a host reported while it was a manager."""
|
||||
from .models import DockerSwarmNode, DockerSwarmService
|
||||
|
||||
await session.execute(delete(DockerSwarmService).where(DockerSwarmService.host_id == host.id))
|
||||
await session.execute(delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id))
|
||||
|
||||
|
||||
async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
logs=None, status=None, phases=None) -> None:
|
||||
"""Upsert containers + time-series + lifecycle events + swarm for one host.
|
||||
|
||||
`snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
|
||||
one entry per ingested sample carrying docker data (usually one; more when
|
||||
the agent flushes a backlog). Every snapshot contributes time-series
|
||||
DockerMetric rows; the newest snapshot drives current container state, the
|
||||
alert pipeline, and lifecycle-event derivation. `swarm` is the newest
|
||||
sample's swarm object (or None off managers) — persisted when present.
|
||||
`disk` is the newest sample's /system/df summary (or None on Docker-less
|
||||
hosts) — persisted when present. `logs` is a list of (recorded_at, records)
|
||||
log batches (m79) — appended one row per line when present, independent of
|
||||
whether this sample also carried container metrics. `status` is the newest
|
||||
sample's {"status", "detail", "at", "swarm_manager"} (agent ≥ 1.9.0): an
|
||||
"ok" with an empty snapshot clears the host's containers, and a node that
|
||||
says it is no longer a manager loses the swarm rows it reported as one.
|
||||
|
||||
`phases`, when given, is marked after each part (docker.status, .swarm,
|
||||
.disk, .logs, .metrics, .events, .containers, .alerts, .reap), each
|
||||
flushed before its mark, so a slow ingest names the part (#5181).
|
||||
"""
|
||||
from steward.core.alerts import MetricBatch
|
||||
from .models import DockerContainer, DockerEvent, DockerMetric
|
||||
|
||||
async def mark(name: str) -> None:
|
||||
if phases is not None:
|
||||
await session.flush()
|
||||
phases.mark(f"docker.{name}")
|
||||
|
||||
if status is not None:
|
||||
await _persist_status(session, host, status)
|
||||
if status.get("swarm_manager") is False:
|
||||
await _expire_swarm(session, host)
|
||||
await mark("status")
|
||||
if swarm is not None:
|
||||
await _persist_swarm(session, host, swarm)
|
||||
await mark("swarm")
|
||||
if disk is not None:
|
||||
await _persist_disk(session, host, disk)
|
||||
await mark("disk")
|
||||
if logs:
|
||||
await _persist_logs(session, host, logs)
|
||||
await mark("logs")
|
||||
|
||||
if not snapshots:
|
||||
return
|
||||
ordered = sorted(snapshots, key=lambda s: s[0])
|
||||
latest_at, latest_containers = ordered[-1]
|
||||
|
||||
# Time-series points for every snapshot (running containers with a CPU read).
|
||||
for recorded_at, containers in ordered:
|
||||
for c in containers:
|
||||
if c.get("status") == "running" and c.get("cpu_pct") is not None:
|
||||
session.add(DockerMetric(
|
||||
host_id=host.id,
|
||||
container_name=c["name"],
|
||||
scraped_at=recorded_at,
|
||||
cpu_pct=c["cpu_pct"],
|
||||
mem_pct=c.get("mem_pct") or 0.0,
|
||||
mem_usage_bytes=c.get("mem_usage_bytes") or 0,
|
||||
))
|
||||
await mark("metrics")
|
||||
|
||||
# Snapshot of stored per-container state BEFORE the upsert overwrites it —
|
||||
# used to diff lifecycle events. Skip event derivation on the host's first
|
||||
# snapshot (empty state) so we don't emit a start per pre-existing container.
|
||||
old_rows = (await session.execute(
|
||||
select(DockerContainer).where(DockerContainer.host_id == host.id)
|
||||
)).scalars().all()
|
||||
old_state = {
|
||||
r.name: {"status": r.status, "health": r.health,
|
||||
"oom_killed": r.oom_killed, "exit_code": r.exit_code,
|
||||
"restart_count": r.restart_count}
|
||||
for r in old_rows
|
||||
}
|
||||
if old_state:
|
||||
# A stop is dated when Docker says it happened (agent ≥ 1.11), not when
|
||||
# the sample that noticed it was taken.
|
||||
finished = {c.get("name"): _parse_started_at(c.get("finished_at"))
|
||||
for c in latest_containers}
|
||||
for name, event, detail in _derive_events(old_state, latest_containers):
|
||||
at = finished.get(name) if event in ("die", "stop") else None
|
||||
session.add(DockerEvent(
|
||||
host_id=host.id, container_name=name,
|
||||
event=event, detail=detail, at=at or latest_at,
|
||||
))
|
||||
await mark("events")
|
||||
|
||||
# Current state + alerts from the newest snapshot only.
|
||||
batch = MetricBatch()
|
||||
for c in latest_containers:
|
||||
name = c.get("name")
|
||||
if not name:
|
||||
continue
|
||||
existing = await session.get(DockerContainer, (host.id, name))
|
||||
if existing is None:
|
||||
existing = DockerContainer(host_id=host.id, name=name)
|
||||
session.add(existing)
|
||||
existing.container_id = c.get("container_id", "") or ""
|
||||
existing.image = c.get("image", "") or ""
|
||||
existing.status = c.get("status", "unknown") or "unknown"
|
||||
existing.cpu_pct = c.get("cpu_pct")
|
||||
existing.mem_usage_bytes = c.get("mem_usage_bytes")
|
||||
existing.mem_limit_bytes = c.get("mem_limit_bytes")
|
||||
existing.mem_pct = c.get("mem_pct")
|
||||
existing.ports_json = json.dumps(c.get("ports") or [])
|
||||
existing.started_at = _parse_started_at(c.get("started_at"))
|
||||
existing.scraped_at = latest_at
|
||||
# Enrichment (agent ≥ 1.4.0; .get keeps older agents working — fields
|
||||
# stay None/0 when absent from the payload).
|
||||
existing.restart_count = c.get("restart_count", 0) or 0
|
||||
existing.health = c.get("health")
|
||||
existing.exit_code = c.get("exit_code")
|
||||
existing.oom_killed = bool(c.get("oom_killed", False))
|
||||
existing.compose_project = c.get("compose_project")
|
||||
existing.service_name = c.get("service_name")
|
||||
existing.task_id = c.get("task_id")
|
||||
existing.node_id = c.get("node_id")
|
||||
existing.net_rx_bytes = c.get("net_rx_bytes")
|
||||
existing.net_tx_bytes = c.get("net_tx_bytes")
|
||||
existing.blk_read_bytes = c.get("blk_read_bytes")
|
||||
existing.blk_write_bytes = c.get("blk_write_bytes")
|
||||
|
||||
# Alert pipeline — resource is host-scoped so containers of the same name
|
||||
# on different hosts don't collide in the metric/alert namespace.
|
||||
resource = f"{host.name}/{name}"
|
||||
if c.get("status") == "running" and c.get("cpu_pct") is not None:
|
||||
batch.add(
|
||||
source_module="docker",
|
||||
resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
|
||||
)
|
||||
if c.get("mem_pct") is not None:
|
||||
batch.add(
|
||||
source_module="docker",
|
||||
resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
|
||||
)
|
||||
# Restart count is alertable regardless of state (crash-looping matters
|
||||
# most while the container is down/restarting).
|
||||
batch.add(
|
||||
source_module="docker",
|
||||
resource_name=resource, metric_name="restart_count",
|
||||
value=float(c.get("restart_count", 0) or 0),
|
||||
)
|
||||
# Health → 1.0/0.0 only for containers that actually define a HEALTHCHECK
|
||||
# (health is None otherwise — recording 0 would false-alarm every plain
|
||||
# container).
|
||||
health = c.get("health")
|
||||
if health in ("healthy", "unhealthy", "starting"):
|
||||
batch.add(
|
||||
source_module="docker",
|
||||
resource_name=resource, metric_name="is_healthy",
|
||||
value=1.0 if health == "healthy" else 0.0,
|
||||
)
|
||||
|
||||
await mark("containers")
|
||||
await batch.record(session)
|
||||
await mark("alerts")
|
||||
|
||||
# Reap containers that vanished from the host's latest listing. Without this,
|
||||
# removed one-shot containers (CI job runners, buildkit builders, codex jobs)
|
||||
# accumulate forever as permanent "stopped" rows and inflate every count.
|
||||
# The listing is authoritative (event derivation already treats absence as
|
||||
# removal), so anything not in the newest snapshot no longer exists on the
|
||||
# host. Mirrors the image/swarm/disk persisters. An empty snapshot legitimately
|
||||
# means the host has no containers, so the unfiltered delete is correct.
|
||||
seen_names = {c["name"] for c in latest_containers if c.get("name")}
|
||||
reap = delete(DockerContainer).where(DockerContainer.host_id == host.id)
|
||||
if seen_names:
|
||||
reap = reap.where(DockerContainer.name.notin_(seen_names))
|
||||
await session.execute(reap)
|
||||
await mark("reap")
|
||||
@@ -0,0 +1,642 @@
|
||||
"""The docker plugin's MCP tools and its get_overview findings.
|
||||
|
||||
Registered through register_mcp_tools() in __init__.py, so they exist only
|
||||
while the plugin is enabled. Reads go through service.py, the same functions
|
||||
the plugin's pages use.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from steward.ansible.runner import trigger_run
|
||||
from steward.ansible.sources import BUILTIN_SOURCE_NAME
|
||||
from steward.mcp._context import current_caller, mcp_app
|
||||
from steward.mcp.tools._audit import audit_mcp
|
||||
from steward.mcp.tools._shape import (
|
||||
age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, summarize,
|
||||
)
|
||||
from steward.mcp.tools.overview import finding
|
||||
from steward.models.ansible_inventory import AnsibleTarget
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.users import UserRole
|
||||
|
||||
from . import service as docker_svc
|
||||
from .service import PROBLEM_SAYS, PROBLEM_SEVERITY, PROBLEMS, container_problems, problem_since
|
||||
from .dedup import dedup_by_container_id
|
||||
from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
|
||||
from .swarm_view import build_swarm_services, replaced_tasks, task_time
|
||||
|
||||
CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
|
||||
LOG_LINE_CHARS = 2000
|
||||
|
||||
|
||||
# ── pure helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
|
||||
"""Keep both ends of an over-long line and say how much of the middle went."""
|
||||
if text is None or len(text) <= limit:
|
||||
return text
|
||||
keep = limit // 2
|
||||
return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
|
||||
|
||||
|
||||
def container_row(c, host_names: dict, problems: list[str], now: datetime,
|
||||
replaced: bool = False, since: datetime | None = None) -> dict:
|
||||
running = (c.status or "").lower() == "running"
|
||||
return {
|
||||
"container_name": c.name,
|
||||
"host_id": c.host_id,
|
||||
"host_name": host_names.get(c.host_id),
|
||||
"image": c.image,
|
||||
"status": c.status,
|
||||
"health": c.health,
|
||||
"problems": problems,
|
||||
# When the worst problem began, from the container's retained events.
|
||||
"problem_since": iso(since),
|
||||
"problem_age_s": age_s(since, now),
|
||||
"exit_code": c.exit_code,
|
||||
"oom_killed": bool(c.oom_killed),
|
||||
"restart_count": c.restart_count,
|
||||
"cpu_pct": rnd(c.cpu_pct),
|
||||
"mem_pct": rnd(c.mem_pct),
|
||||
"mem_usage_bytes": c.mem_usage_bytes,
|
||||
"mem_limit_bytes": c.mem_limit_bytes,
|
||||
"started_at": iso(c.started_at),
|
||||
"uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
|
||||
"compose_project": c.compose_project,
|
||||
"swarm_service": c.service_name,
|
||||
# A swarm task Swarm has replaced: history, judged on its service.
|
||||
"swarm_replaced": replaced if c.service_name else None,
|
||||
"reported_at": iso(c.scraped_at),
|
||||
"reported_age_s": age_s(c.scraped_at, now),
|
||||
}
|
||||
|
||||
|
||||
def _from_iso(text: str | None) -> datetime | None:
|
||||
return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
|
||||
|
||||
|
||||
def container_findings(rows: list[dict]) -> list[dict]:
|
||||
out = []
|
||||
for r in rows:
|
||||
if not r["problems"]:
|
||||
continue
|
||||
worst = r["problems"][0]
|
||||
where = f" on {r['host_name']}" if r["host_name"] else ""
|
||||
detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
|
||||
out.append(finding(
|
||||
PROBLEM_SEVERITY[worst], f"container_{worst}", r["container_name"],
|
||||
f"container {r['container_name']}{where} {' and '.join(PROBLEM_SAYS[p] for p in r['problems'])}{detail}",
|
||||
since=_from_iso(r["problem_since"]),
|
||||
ids={"host_id": r["host_id"], "container_name": r["container_name"]},
|
||||
next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
|
||||
return out
|
||||
|
||||
|
||||
def task_error_text(err: dict | None) -> str:
|
||||
"""Swarm's own account of a task's end, as one clause."""
|
||||
if not err:
|
||||
return ""
|
||||
why = err.get("error") or err.get("message") or err.get("state") or "unknown"
|
||||
code = f", exit {err['exit_code']}" if err.get("exit_code") not in (None, 0) else ""
|
||||
where = f" on {err['node']}" if err.get("node") else ""
|
||||
return f"{why}{code}{where}"
|
||||
|
||||
|
||||
def service_findings(services: list[dict]) -> list[dict]:
|
||||
"""Swarm services that are failing: crash-looping, under-replicated, or that
|
||||
lost a task to an error in the last 24h. Each names Swarm's last task error."""
|
||||
out = []
|
||||
for s in services:
|
||||
replicas = f"{s['running']} of {s['desired']} replicas running"
|
||||
last = task_error_text(s.get("last_error"))
|
||||
tail = f"; last task error: {last}" if last else ""
|
||||
ids = {"service_name": s["name"]}
|
||||
since = s.get("trouble_since")
|
||||
if s.get("crash_looping"):
|
||||
out.append(finding(
|
||||
"critical", "swarm_service_crash_looping", s["name"],
|
||||
f"swarm service {s['name']} keeps failing: {s['failures_1h']} tasks failed in the "
|
||||
f"last hour, {replicas}{tail}", since=since, ids=ids,
|
||||
next="list_swarm_services()"))
|
||||
elif s["state"] != "healthy":
|
||||
out.append(finding(
|
||||
"critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
|
||||
s["name"], f"swarm service {s['name']} has {replicas}{tail}",
|
||||
since=since, ids=ids, next="list_swarm_services()"))
|
||||
elif s.get("failures_24h"):
|
||||
n = s["failures_24h"]
|
||||
out.append(finding(
|
||||
"warning", "swarm_task_failed", s["name"],
|
||||
f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
|
||||
f"24h and is now {replicas}{tail}", since=since, ids=ids,
|
||||
next="list_swarm_services()"))
|
||||
return out
|
||||
|
||||
|
||||
# ── loading ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async def _host_names(db) -> dict[str, str]:
|
||||
return dict((await db.execute(select(Host.id, Host.name))).all())
|
||||
|
||||
|
||||
def _crash_loop_deaths() -> int:
|
||||
return docker_svc.crash_loop_deaths(mcp_app().config)
|
||||
|
||||
|
||||
async def _replaced(db, containers) -> set[tuple]:
|
||||
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
||||
return replaced_tasks(containers, services)
|
||||
|
||||
|
||||
async def _rows(db, now: datetime) -> list[dict]:
|
||||
"""Every container with its problems. A swarm task Swarm replaced has none:
|
||||
its failure, if it was one, belongs to its service (service_findings)."""
|
||||
host_names = await _host_names(db)
|
||||
return [container_row(r["container"], host_names, r["problems"], now,
|
||||
replaced=r["replaced"], since=r["since"])
|
||||
for r in await docker_svc.ranked_containers(db, now, _crash_loop_deaths())]
|
||||
|
||||
|
||||
async def _swarm(db, now: datetime):
|
||||
containers = dedup_by_container_id(await docker_svc.list_containers(db))
|
||||
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
||||
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
||||
host_names = await _host_names(db)
|
||||
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
||||
built = build_swarm_services(containers, services, node_hostname, host_names,
|
||||
now=now, crash_loop_deaths=_crash_loop_deaths())
|
||||
return built["services"], nodes, host_names
|
||||
|
||||
|
||||
async def mcp_overview(db) -> list[dict]:
|
||||
"""get_overview findings: failing containers and under-replicated swarm services."""
|
||||
now = now_utc()
|
||||
rows = await _rows(db, now)
|
||||
services, _, _ = await _swarm(db, now)
|
||||
return container_findings(rows) + service_findings(services)
|
||||
|
||||
|
||||
# ── tools ────────────────────────────────────────────────────────────────────
|
||||
|
||||
async def list_containers(host: str = "", status: str = "", problem: str = "",
|
||||
compose_project: str = "", limit: int = 100) -> dict:
|
||||
"""Docker containers across every host that runs a Steward agent with Docker.
|
||||
|
||||
host: a host's name or id (list_hosts); empty for all hosts.
|
||||
status: Docker's state — "running", "exited", "restarting", "paused",
|
||||
"created" or "dead"; empty for all.
|
||||
problem: "any" for containers with something wrong, or one of
|
||||
"restarting", "crash_looping" (died repeatedly within the last hour;
|
||||
the count is set under Settings → Thresholds, default 3), "unhealthy"
|
||||
(failing its healthcheck), "oom_killed", "exited_nonzero" (died with an
|
||||
error in the last 24h; exit 137/143 without an OOM kill is a stop, not
|
||||
an error); empty for no filter.
|
||||
compose_project: only containers from this Compose project.
|
||||
limit: default 100, max 500.
|
||||
|
||||
Containers with problems come first. Each row's `problems` lists what is
|
||||
wrong, most severe first, and `problem_since` / `problem_age_s` say when
|
||||
the worst of them began (null when Steward's retained events cannot say).
|
||||
`reported_age_s` is how old the agent's last report is, so a large value
|
||||
means the row may be stale.
|
||||
|
||||
A swarm task's container that Swarm has replaced (a rolling update, or a
|
||||
retry after a failure) has `swarm_replaced: true` and no problems: Swarm
|
||||
restarts a task as a new container, so failures are counted on the
|
||||
service — list_swarm_services shows them.
|
||||
"""
|
||||
if problem and problem != "any" and problem not in PROBLEMS:
|
||||
raise ValueError(f"problem {problem!r} is not 'any' or one of {', '.join(PROBLEMS)}.")
|
||||
limit = clamp_limit(limit, 100, 500)
|
||||
now = now_utc()
|
||||
async with db_session() as db:
|
||||
rows = await _rows(db, now)
|
||||
if host:
|
||||
hosts = list((await db.execute(select(Host))).scalars())
|
||||
h = pick(host, hosts, kind="host", list_tool="list_hosts")
|
||||
rows = [r for r in rows if r["host_id"] == h.id]
|
||||
if status:
|
||||
rows = [r for r in rows if (r["status"] or "").lower() == status.lower()]
|
||||
if problem == "any":
|
||||
rows = [r for r in rows if r["problems"]]
|
||||
elif problem:
|
||||
rows = [r for r in rows if problem in r["problems"]]
|
||||
if compose_project:
|
||||
rows = [r for r in rows if r["compose_project"] == compose_project]
|
||||
rows.sort(key=lambda r: (not r["problems"], (r["status"] or "") != "running",
|
||||
(r["host_name"] or "").lower(), r["container_name"]))
|
||||
return bounded(rows, limit, key="containers",
|
||||
more="Pass a larger `limit` (max 500), or filter by `host`, `status` or `problem`.")
|
||||
|
||||
|
||||
@dataclass
|
||||
class RemovedContainer:
|
||||
"""A container Docker has removed, known only by its retained logs and events."""
|
||||
host_id: str
|
||||
name: str
|
||||
last_seen: datetime
|
||||
|
||||
|
||||
_REMOVED_HINT = ("It may have been removed: Swarm replaces a dead task with a new container under "
|
||||
"a new name. list_containers(status=\"exited\") lists stopped containers, and "
|
||||
"search_container_logs(host=...) searches a whole host.")
|
||||
|
||||
|
||||
async def _pick_container(db, container: str, host: str, *, removed: bool = False):
|
||||
"""The container named `container`: a live one, else (with `removed`) one
|
||||
Docker has removed whose logs or events Steward still holds, returned as a
|
||||
RemovedContainer. Exact names beat partial ones, and live beats removed."""
|
||||
containers = await docker_svc.list_containers(db)
|
||||
host_id = None
|
||||
if host:
|
||||
hosts = list((await db.execute(select(Host))).scalars())
|
||||
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
||||
containers = [c for c in containers if c.host_id == host_id]
|
||||
want = (container or "").strip()
|
||||
if not want:
|
||||
raise ValueError("pass a container name; list_containers lists them.")
|
||||
live = {(c.host_id, c.name) for c in containers}
|
||||
|
||||
def _one(found, what):
|
||||
if len(found) > 1:
|
||||
raise ValueError(f"{len(found)} hosts have {what} named {want!r}; pass `host` "
|
||||
f"(one of: {', '.join(sorted(f[0] for f in found))}).")
|
||||
return found[0] if found else None
|
||||
|
||||
same = [c for c in containers if c.name == want] or \
|
||||
[c for c in containers if c.name.lower() == want.lower()]
|
||||
if same:
|
||||
return _one([(c.host_id, c) for c in same], "a container")[1]
|
||||
if removed:
|
||||
gone = [r for r in await docker_svc.retained_names(db, want, exact=True, host_id=host_id)
|
||||
if (r[0], r[1]) not in live]
|
||||
if gone:
|
||||
return RemovedContainer(*_one(gone, "a removed container"))
|
||||
# One more chance: a unique container whose name contains it (swarm task
|
||||
# names look like "stack_svc.1.k2j…").
|
||||
partial = [c for c in containers if want.lower() in c.name.lower()]
|
||||
if len(partial) == 1:
|
||||
return partial[0]
|
||||
gone = []
|
||||
if removed and not partial:
|
||||
gone = [r for r in await docker_svc.retained_names(db, want, exact=False, host_id=host_id)
|
||||
if (r[0], r[1]) not in live]
|
||||
if len(gone) == 1:
|
||||
return RemovedContainer(*gone[0])
|
||||
names = sorted(c.name for c in partial)[:10] + [f"{r[1]} (removed)" for r in gone[:10]]
|
||||
hint = f" Containers whose names contain it: {', '.join(names)}." if names else ""
|
||||
raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} "
|
||||
f"{'' if names else _REMOVED_HINT + ' '}list_containers lists them.")
|
||||
|
||||
|
||||
def _removed_note(c: RemovedContainer) -> str:
|
||||
return (f"{c.name} no longer exists: Docker removed it (Swarm replaces a failed task with a new "
|
||||
f"container). This is what Steward retained; it was last seen {iso(c.last_seen)}.")
|
||||
|
||||
|
||||
async def get_container(container: str, host: str = "", range: str = "24h", events: int = 20) -> dict:
|
||||
"""One container in depth: state, resources over time and lifecycle events.
|
||||
|
||||
container: the container's name, as list_containers shows. A unique part of
|
||||
a name also works (swarm task names are long).
|
||||
host: the host's name or id. Needed only when several hosts run a
|
||||
container with this name.
|
||||
range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
|
||||
max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
|
||||
"7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
|
||||
events: how many lifecycle events (start, restart, stop, die, oom, health_change)
|
||||
to return, newest first (default 20, max 100).
|
||||
|
||||
A container Docker has since removed (a dead Swarm task, a `docker rm`)
|
||||
still resolves by its exact name, or by a part of it that matches one
|
||||
removed container, while Steward retains its logs or events: the reply
|
||||
has `removed: true`, `last_seen_at`, its history and its events, and no
|
||||
live state. A live container has `removed: false`.
|
||||
"""
|
||||
since, key = lookback(range)
|
||||
events = clamp_limit(events, 20, 100)
|
||||
now = now_utc()
|
||||
async with db_session() as db:
|
||||
c = await _pick_container(db, container, host, removed=True)
|
||||
if isinstance(c, RemovedContainer):
|
||||
return await _removed_container(db, c, since, key, events, now)
|
||||
gone = (c.host_id, c.name) in await _replaced(db, await docker_svc.list_containers(db))
|
||||
failures = (await docker_svc.recent_failures_by_container(
|
||||
db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
|
||||
host_names = await _host_names(db)
|
||||
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
|
||||
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
|
||||
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
|
||||
sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)])
|
||||
hour_ago = now - timedelta(hours=1)
|
||||
recent = sum(1 for e in failures if e.at >= hour_ago)
|
||||
try:
|
||||
ports = json.loads(c.ports_json or "[]")
|
||||
except ValueError:
|
||||
ports = []
|
||||
problems = [] if gone else container_problems(c, recent, bool(failures), _crash_loop_deaths())
|
||||
began = problem_since(problems, failures, hour_ago, sick.get((c.host_id, c.name)))
|
||||
return {
|
||||
**container_row(c, host_names, problems, now, replaced=gone, since=began),
|
||||
"removed": False,
|
||||
"container_id": c.container_id,
|
||||
"ports": ports,
|
||||
"net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
|
||||
"blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
|
||||
"range": key,
|
||||
"cpu_pct_history": summarize(cpu),
|
||||
"mem_pct_history": summarize(mem),
|
||||
"events": _event_rows(evs, now),
|
||||
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
|
||||
}
|
||||
|
||||
|
||||
def _event_rows(evs, now: datetime) -> list[dict]:
|
||||
return [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)}
|
||||
for e in evs]
|
||||
|
||||
|
||||
async def _removed_container(db, c: RemovedContainer, since, key: str, events: int,
|
||||
now: datetime) -> dict:
|
||||
"""get_container for a container Docker has removed: what Steward kept."""
|
||||
host_names = await _host_names(db)
|
||||
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
|
||||
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
|
||||
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
|
||||
return {
|
||||
"container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id),
|
||||
"removed": True, "note": _removed_note(c),
|
||||
"last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now),
|
||||
"range": key,
|
||||
"cpu_pct_history": summarize(cpu),
|
||||
"mem_pct_history": summarize(mem),
|
||||
"events": _event_rows(evs, now),
|
||||
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")',
|
||||
}
|
||||
|
||||
|
||||
async def list_swarm_services(limit: int = 100) -> dict:
|
||||
"""Docker Swarm services with their replicas per node, and the swarm's nodes.
|
||||
|
||||
`state` is "healthy" (all desired replicas running), "degraded" (some) or
|
||||
"down" (none). `replicas` are the service's current tasks: each names its
|
||||
host and container status, and one marked `ghost` runs on a node with no
|
||||
Steward agent, so Steward knows only how many tasks are placed there.
|
||||
Tasks Swarm replaced (rolling updates, retries) are left out and counted
|
||||
in `previous_tasks`.
|
||||
|
||||
From Swarm's own task records (a manager's agent 1.8.0 or later;
|
||||
`tasks_known: false` and nulls otherwise):
|
||||
- `failures_1h` / `failures_24h`: tasks that failed or were rejected.
|
||||
Swarm retries a failed task as a new one, so this, not any container's
|
||||
restart count, is how a crash loop shows. Swarm keeps about 5 old tasks
|
||||
per replica, which bounds the count.
|
||||
- `crash_looping`: failures_1h reached the crash-loop count set under
|
||||
Settings → Thresholds (default 3).
|
||||
- `last_error`: the newest task that failed, was rejected, or carries an
|
||||
error while Swarm retries it — `error` is Swarm's own text (e.g. "No
|
||||
such image", "task: non-zero exit (1)"), with `exit_code`, `node`, `at`.
|
||||
- `trouble_since` / `trouble_age_s`: when the current trouble began — a
|
||||
crash loop's first failure in the hour, a short service's first failed
|
||||
task in its failing slots, a recovered service's first failure in 24h.
|
||||
Swarm keeps few old tasks, so a long outage reads as a lower bound.
|
||||
|
||||
Services with a problem come first. `nodes` lists every swarm node once,
|
||||
with its role (manager/worker), availability and status.
|
||||
limit: default 100, max 500.
|
||||
"""
|
||||
limit = clamp_limit(limit, 100, 500)
|
||||
now = now_utc()
|
||||
async with db_session() as db:
|
||||
services, nodes, host_names = await _swarm(db, now)
|
||||
seen, node_rows = set(), []
|
||||
for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
|
||||
if n.node_id in seen:
|
||||
continue
|
||||
seen.add(n.node_id)
|
||||
node_rows.append({"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
|
||||
"leader": n.leader, "availability": n.availability, "status": n.status,
|
||||
"reported_by": host_names.get(n.host_id)})
|
||||
node_rows.sort(key=lambda n: n["hostname"].lower())
|
||||
order = {"down": 0, "degraded": 1, "healthy": 2}
|
||||
|
||||
def _rank(s):
|
||||
return (not s["crash_looping"], order.get(s["state"], 3), not s["failures_24h"], s["name"])
|
||||
|
||||
def _error_row(err):
|
||||
if err is None:
|
||||
return None
|
||||
at = task_time(err)
|
||||
return {"error": err["error"], "message": err["message"], "state": err["state"],
|
||||
"exit_code": err["exit_code"], "node": err["node"],
|
||||
"at": iso(at), "age_s": age_s(at, now) if at else None}
|
||||
|
||||
rows = [
|
||||
{
|
||||
"service_name": s["name"], "mode": s["mode"], "state": s["state"],
|
||||
"running": s["running"], "desired": s["desired"],
|
||||
"crash_looping": s["crash_looping"],
|
||||
"failures_1h": s["failures_1h"], "failures_24h": s["failures_24h"],
|
||||
"last_error": _error_row(s["last_error"]),
|
||||
"trouble_since": iso(s["trouble_since"]),
|
||||
"trouble_age_s": age_s(s["trouble_since"], now),
|
||||
"previous_tasks": s["previous_tasks"], "tasks_known": s["tasks_known"],
|
||||
"replicas": [
|
||||
{"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
|
||||
{"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
|
||||
"status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
|
||||
"cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
|
||||
for r in s["replicas"]
|
||||
],
|
||||
}
|
||||
for s in sorted(services, key=_rank)
|
||||
]
|
||||
return {"nodes": node_rows, **bounded(rows, limit, key="services")}
|
||||
|
||||
|
||||
async def search_container_logs(container: str = "", host: str = "", text: str = "",
|
||||
range: str = "1h", stream: str = "", before: str = "",
|
||||
context: int = 0, limit: int = 50) -> dict:
|
||||
"""Search the container log lines Steward has retained, newest first.
|
||||
|
||||
container: one container's name (as list_containers shows); empty searches
|
||||
every container (narrow with `host`). A container Docker has removed
|
||||
still works while its lines are retained: the reply then has
|
||||
`container_removed: true` and a `note` saying when it was last seen.
|
||||
host: a host's name or id; empty for all hosts.
|
||||
text: case-insensitive substring to match; empty matches every line.
|
||||
range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are
|
||||
kept per container up to a size and age limit, so older lines may be gone.
|
||||
stream: "stdout" or "stderr"; empty for both.
|
||||
before: an ISO time; returns only lines older than it. Pass a response's
|
||||
`next_before` to page further back.
|
||||
context: lines of surrounding log (0-5) to include with each match, from
|
||||
the same container. Only with `container` set.
|
||||
limit: matches to return, default 50, max 200.
|
||||
|
||||
Lines longer than 2000 characters keep both ends, and the middle is
|
||||
replaced by a marker saying how much was cut.
|
||||
"""
|
||||
since, key = lookback(range)
|
||||
limit = clamp_limit(limit, 50, 200)
|
||||
context = max(0, min(int(context or 0), 5))
|
||||
if context and not container:
|
||||
raise ValueError("context needs `container`: it shows lines around a match within one container's log.")
|
||||
if stream and stream not in ("stdout", "stderr"):
|
||||
raise ValueError(f"stream {stream!r} is not 'stdout' or 'stderr'.")
|
||||
before_dt = None
|
||||
if before:
|
||||
try:
|
||||
before_dt = datetime.fromisoformat(before.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc
|
||||
async with db_session() as db:
|
||||
host_id = name = removed = None
|
||||
if container:
|
||||
c = await _pick_container(db, container, host, removed=True)
|
||||
host_id, name = c.host_id, c.name
|
||||
removed = c if isinstance(c, RemovedContainer) else None
|
||||
elif host:
|
||||
hosts = list((await db.execute(select(Host))).scalars())
|
||||
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
||||
rows = await docker_svc.search_logs(db, since=since, host_id=host_id, name=name, text=text,
|
||||
stream=stream, before=before_dt, limit=limit + 1)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
host_names = await _host_names(db)
|
||||
matches = []
|
||||
for r in rows:
|
||||
m = {"at": iso(r.ts), "container_name": r.container_name, "host_id": r.host_id,
|
||||
"host_name": host_names.get(r.host_id), "stream": r.stream, "line": cut_middle(r.line)}
|
||||
if context:
|
||||
pre, post = await docker_svc.log_context(db, r.host_id, r.container_name, r.ts, context)
|
||||
m["before"] = [cut_middle(x.line) for x in pre]
|
||||
m["after"] = [cut_middle(x.line) for x in post]
|
||||
matches.append(m)
|
||||
out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more}
|
||||
if removed is not None:
|
||||
out["container_removed"] = True
|
||||
out["note"] = _removed_note(removed)
|
||||
if removed.last_seen < since:
|
||||
out["note"] += " That is before this range; pass a longer `range`."
|
||||
if more:
|
||||
# Full precision: `at` is cut to the second for reading, and a cursor
|
||||
# cut the same way would skip lines logged earlier in that second.
|
||||
out["next_before"] = rows[-1].ts.isoformat()
|
||||
out["more"] = "More matches are older; pass next_before as `before`, or narrow with `text`."
|
||||
return out
|
||||
|
||||
|
||||
async def get_docker_disk_usage(host: str = "") -> dict:
|
||||
"""Docker disk usage per host: images, containers, volumes and build cache, in bytes.
|
||||
|
||||
host: a host's name or id; empty for every host. `images_reclaimable_bytes`
|
||||
is what pruning unused images would free. An admin can prune from
|
||||
Steward's Docker → Disk page.
|
||||
"""
|
||||
async with db_session() as db:
|
||||
summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
|
||||
host_names = await _host_names(db)
|
||||
if host:
|
||||
hosts = list((await db.execute(select(Host))).scalars())
|
||||
hid = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
||||
summaries = [s for s in summaries if s.host_id == hid]
|
||||
rows = [
|
||||
{
|
||||
"host_id": s.host_id, "host_name": host_names.get(s.host_id),
|
||||
"images_total": s.images_total, "images_active": s.images_active,
|
||||
"images_bytes": s.images_size, "images_reclaimable_bytes": s.images_reclaimable,
|
||||
"containers_count": s.containers_count, "containers_bytes": s.containers_size,
|
||||
"volumes_count": s.volumes_count, "volumes_bytes": s.volumes_size,
|
||||
"build_cache_bytes": s.build_cache_size,
|
||||
"reported_at": iso(s.updated_at), "reported_age_s": age_s(s.updated_at),
|
||||
}
|
||||
for s in summaries
|
||||
]
|
||||
rows.sort(key=lambda r: -(r["images_reclaimable_bytes"] or 0))
|
||||
return {"hosts": rows, "total": len(rows)}
|
||||
|
||||
|
||||
PRUNE_BUILTIN_PLAYBOOK = "maintenance/docker_prune.yml"
|
||||
_STOPPED = ("exited", "created", "dead")
|
||||
|
||||
|
||||
def prune_preview(what: str, usage, stopped: int) -> dict:
|
||||
"""What a prune of `what` would remove, from the host's latest disk report."""
|
||||
out: dict = {}
|
||||
if what in ("containers", "system"):
|
||||
out["stopped_containers"] = stopped
|
||||
if what == "images":
|
||||
out["unused_images_bytes"] = usage.images_reclaimable if usage else None
|
||||
if what == "system":
|
||||
out["dangling_images"] = "yes (unused but tagged images stay)"
|
||||
out["build_cache_bytes"] = usage.build_cache_size if usage else None
|
||||
out["unused_networks"] = "yes"
|
||||
return out
|
||||
|
||||
|
||||
async def prune_docker(host: str, what: str, check: bool = True) -> dict:
|
||||
"""Reclaim Docker disk on a host. Previews unless check=false.
|
||||
|
||||
host: the Steward host's name or id. It needs a linked Ansible target
|
||||
(list_inventory shows the links); the prune runs through Ansible as
|
||||
Steward's built-in docker_prune playbook.
|
||||
what: "containers" removes stopped containers; "images" removes ALL
|
||||
unused images, not only dangling ones; "system" is `docker system
|
||||
prune -f`: stopped containers, dangling images, unused networks and
|
||||
build cache. Volumes are never pruned.
|
||||
check: true (default) starts nothing and returns `would_remove`, sized
|
||||
from the host's latest Docker disk report. Ansible's own check mode
|
||||
skips the prune commands, so a check-mode run would show nothing.
|
||||
Pass false to run the prune.
|
||||
|
||||
Admin only, as on the Docker → Disk page. A real prune returns `run_id`
|
||||
(follow it with wait_for_run) and is recorded in the audit log.
|
||||
"""
|
||||
what = (what or "").strip()
|
||||
if what not in docker_svc.PRUNE_TARGETS:
|
||||
raise ValueError(f"what {what!r} is not one of {', '.join(docker_svc.PRUNE_TARGETS)}.")
|
||||
async with db_session() as db:
|
||||
h = pick(host, (await db.execute(select(Host))).scalars(), kind="host", list_tool="list_hosts")
|
||||
target = (await db.execute(
|
||||
select(AnsibleTarget).where(AnsibleTarget.host_id == h.id))).scalars().first()
|
||||
usage = await db.get(DockerDiskUsage, h.id)
|
||||
stopped = sum(1 for c in await docker_svc.list_containers(db, h.id)
|
||||
if (c.status or "").lower() in _STOPPED)
|
||||
if target is None:
|
||||
raise ValueError(f"host {h.name!r} has no linked Ansible target, so Steward cannot reach it "
|
||||
"to prune. Link one under Ansible → Inventory. Nothing was run.")
|
||||
preview = {
|
||||
"host": h.name,
|
||||
"what": what,
|
||||
"ansible_target": target.name,
|
||||
"would_remove": prune_preview(what, usage, stopped),
|
||||
"disk_report_age_s": age_s(usage.updated_at) if usage else None,
|
||||
}
|
||||
if check:
|
||||
return {**preview, "check": True,
|
||||
"summary": f"Nothing was run. Pass check=false to prune {what} on {h.name}."}
|
||||
caller = current_caller()
|
||||
run, _src, err = await trigger_run(
|
||||
mcp_app(), source_name=BUILTIN_SOURCE_NAME, playbook_path=PRUNE_BUILTIN_PLAYBOOK,
|
||||
inventory_scope=f"steward:target:{target.id}",
|
||||
params={"extra_vars_map": docker_svc.prune_extra_vars(what)},
|
||||
triggered_by=caller.user_id, origin="mcp")
|
||||
if err:
|
||||
raise ValueError(f"the prune could not start: {err}.")
|
||||
await audit_mcp("docker.prune.started", "host", h.name,
|
||||
{"host_id": h.id, "what": what, "run_id": run.id})
|
||||
return {**preview, "check": False, "run_id": run.id,
|
||||
"next": f'wait_for_run(run_id="{run.id}")'}
|
||||
|
||||
|
||||
def register_tools(register) -> None:
|
||||
for fn in (list_containers, get_container, list_swarm_services,
|
||||
search_container_logs, get_docker_disk_usage):
|
||||
register(fn, role=UserRole.viewer, read_only=True)
|
||||
register(prune_docker, role=UserRole.admin, read_only=False)
|
||||
@@ -0,0 +1,70 @@
|
||||
# plugins/docker/migrations/env.py
|
||||
"""Alembic env.py for the Docker plugin."""
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||
from alembic import context
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
|
||||
|
||||
from steward.models.base import Base
|
||||
import steward.models # noqa: F401
|
||||
from plugins.docker.models import DockerContainer, DockerMetric # noqa: F401
|
||||
|
||||
config = context.config
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def _get_url() -> str:
|
||||
import yaml
|
||||
cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml")
|
||||
try:
|
||||
with open(cfg_path) as f:
|
||||
cfg = yaml.safe_load(f) or {}
|
||||
url = cfg.get("database", {}).get("url", "")
|
||||
except FileNotFoundError:
|
||||
url = ""
|
||||
return os.environ.get("STEWARD_DATABASE__URL", url)
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
context.configure(
|
||||
url=_get_url(), target_metadata=target_metadata,
|
||||
literal_binds=True, dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def do_run_migrations(connection: Connection) -> None:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
async def run_async_migrations() -> None:
|
||||
cfg = config.get_section(config.config_ini_section, {})
|
||||
cfg["sqlalchemy.url"] = _get_url()
|
||||
connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool)
|
||||
async with connectable.connect() as connection:
|
||||
await connection.run_sync(do_run_migrations)
|
||||
await connectable.dispose()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
asyncio.run(run_async_migrations())
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Docker plugin initial tables
|
||||
|
||||
Revision ID: docker_001_initial
|
||||
Revises: (none — branch off core via depends_on)
|
||||
Create Date: 2026-03-22
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "docker_001_initial"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = "docker"
|
||||
depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"docker_containers",
|
||||
sa.Column("name", sa.String(255), primary_key=True),
|
||||
sa.Column("container_id", sa.String(64), nullable=False, server_default=""),
|
||||
sa.Column("image", sa.String(512), nullable=False, server_default=""),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
|
||||
sa.Column("cpu_pct", sa.Float, nullable=True),
|
||||
sa.Column("mem_usage_bytes", sa.Integer, nullable=True),
|
||||
sa.Column("mem_limit_bytes", sa.Integer, nullable=True),
|
||||
sa.Column("mem_pct", sa.Float, nullable=True),
|
||||
sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"docker_metrics",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("container_name", sa.String(255), nullable=False, index=True),
|
||||
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True),
|
||||
sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"),
|
||||
sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"),
|
||||
sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("docker_metrics")
|
||||
op.drop_table("docker_containers")
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Docker collection goes per-host: add host_id, re-key by (host_id, name)
|
||||
|
||||
Collection moved from the central single-socket scrape to the host agent, so
|
||||
containers are now scoped to the host that reported them. docker_containers is
|
||||
re-keyed (host_id, name) and docker_metrics gains host_id.
|
||||
|
||||
Dev-only posture (rule 122): the old tables only ever held the Steward box's
|
||||
own containers (a single global namespace), which are disposable — so this
|
||||
DROP+recreates rather than backfilling a host_id onto orphan rows.
|
||||
|
||||
Revision ID: docker_002_host_scoped
|
||||
Revises: docker_001_initial
|
||||
Create Date: 2026-06-18
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "docker_002_host_scoped"
|
||||
down_revision: Union[str, None] = "docker_001_initial"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
# FK targets hosts.id (created in 0002_core_monitors) — make the edge explicit.
|
||||
depends_on: Union[str, Sequence[str], None] = ("0002_core_monitors",)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_table("docker_metrics")
|
||||
op.drop_table("docker_containers")
|
||||
|
||||
op.create_table(
|
||||
"docker_containers",
|
||||
sa.Column("host_id", sa.String(36),
|
||||
sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True),
|
||||
sa.Column("name", sa.String(255), primary_key=True),
|
||||
sa.Column("container_id", sa.String(64), nullable=False, server_default=""),
|
||||
sa.Column("image", sa.String(512), nullable=False, server_default=""),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
|
||||
sa.Column("cpu_pct", sa.Float, nullable=True),
|
||||
sa.Column("mem_usage_bytes", sa.Integer, nullable=True),
|
||||
sa.Column("mem_limit_bytes", sa.Integer, nullable=True),
|
||||
sa.Column("mem_pct", sa.Float, nullable=True),
|
||||
sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"docker_metrics",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("host_id", sa.String(36),
|
||||
sa.ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("container_name", sa.String(255), nullable=False),
|
||||
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"),
|
||||
sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"),
|
||||
sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"),
|
||||
)
|
||||
op.create_index("ix_docker_metrics_host_id", "docker_metrics", ["host_id"])
|
||||
op.create_index("ix_docker_metrics_container_name", "docker_metrics",
|
||||
["container_name"])
|
||||
op.create_index("ix_docker_metrics_scraped_at", "docker_metrics", ["scraped_at"])
|
||||
op.create_index("ix_docker_metrics_host_container_time", "docker_metrics",
|
||||
["host_id", "container_name", "scraped_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("docker_metrics")
|
||||
op.drop_table("docker_containers")
|
||||
|
||||
op.create_table(
|
||||
"docker_containers",
|
||||
sa.Column("name", sa.String(255), primary_key=True),
|
||||
sa.Column("container_id", sa.String(64), nullable=False, server_default=""),
|
||||
sa.Column("image", sa.String(512), nullable=False, server_default=""),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
|
||||
sa.Column("cpu_pct", sa.Float, nullable=True),
|
||||
sa.Column("mem_usage_bytes", sa.Integer, nullable=True),
|
||||
sa.Column("mem_limit_bytes", sa.Integer, nullable=True),
|
||||
sa.Column("mem_pct", sa.Float, nullable=True),
|
||||
sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_table(
|
||||
"docker_metrics",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("container_name", sa.String(255), nullable=False, index=True),
|
||||
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True),
|
||||
sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"),
|
||||
sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"),
|
||||
sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"),
|
||||
)
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Docker container enrichment: health, exit/restart, grouping, I/O counters
|
||||
|
||||
Adds the fields the agent (≥1.4.0) now reports per container beyond the basic
|
||||
state: health status, exit code, OOM flag, compose/swarm grouping labels, and
|
||||
cumulative network/block I/O counters. Additive columns — no data loss, so no
|
||||
DROP+recreate needed here.
|
||||
|
||||
Revision ID: docker_003_container_enrichment
|
||||
Revises: docker_002_host_scoped
|
||||
Create Date: 2026-06-19
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "docker_003_container_enrichment"
|
||||
down_revision: Union[str, None] = "docker_002_host_scoped"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("docker_containers", sa.Column("health", sa.String(16), nullable=True))
|
||||
op.add_column("docker_containers", sa.Column("exit_code", sa.Integer, nullable=True))
|
||||
op.add_column("docker_containers",
|
||||
sa.Column("oom_killed", sa.Boolean, nullable=False, server_default=sa.false()))
|
||||
op.add_column("docker_containers", sa.Column("compose_project", sa.String(255), nullable=True))
|
||||
op.add_column("docker_containers", sa.Column("service_name", sa.String(255), nullable=True))
|
||||
op.add_column("docker_containers", sa.Column("task_id", sa.String(64), nullable=True))
|
||||
op.add_column("docker_containers", sa.Column("node_id", sa.String(64), nullable=True))
|
||||
op.add_column("docker_containers", sa.Column("net_rx_bytes", sa.BigInteger, nullable=True))
|
||||
op.add_column("docker_containers", sa.Column("net_tx_bytes", sa.BigInteger, nullable=True))
|
||||
op.add_column("docker_containers", sa.Column("blk_read_bytes", sa.BigInteger, nullable=True))
|
||||
op.add_column("docker_containers", sa.Column("blk_write_bytes", sa.BigInteger, nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for col in ("blk_write_bytes", "blk_read_bytes", "net_tx_bytes", "net_rx_bytes",
|
||||
"node_id", "task_id", "service_name", "compose_project",
|
||||
"oom_killed", "exit_code", "health"):
|
||||
op.drop_column("docker_containers", col)
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Docker lifecycle events + swarm topology tables
|
||||
|
||||
Adds the storage for milestone-77 monitoring depth that doesn't fit on the
|
||||
per-container row:
|
||||
|
||||
* docker_events — lifecycle (start/stop/die/oom/health_change) derived by
|
||||
diffing consecutive host snapshots; host-scoped, indexed for both timeline
|
||||
lookups (host_id, container_name, at) and retention pruning (at).
|
||||
* docker_swarm_services / docker_swarm_nodes — manager-reported Swarm
|
||||
topology (desired-vs-running replicas, node role/availability/status).
|
||||
|
||||
All new tables — purely additive, so no DROP+recreate. `event`/`mode`/`role`
|
||||
etc. are plain strings (no CHECK whitelist), matching how docker_containers
|
||||
models `status`; keeps the value sets open without a migration per addition
|
||||
(so rule 36 doesn't apply here).
|
||||
|
||||
Revision ID: docker_004_events_swarm
|
||||
Revises: docker_003_container_enrichment
|
||||
Create Date: 2026-06-19
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "docker_004_events_swarm"
|
||||
down_revision: Union[str, None] = "docker_003_container_enrichment"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"docker_events",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("host_id", sa.String(36),
|
||||
sa.ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("container_name", sa.String(255), nullable=False),
|
||||
sa.Column("event", sa.String(16), nullable=False),
|
||||
sa.Column("detail", sa.String(255), nullable=True),
|
||||
sa.Column("at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_docker_events_host_container_time", "docker_events",
|
||||
["host_id", "container_name", "at"])
|
||||
op.create_index("ix_docker_events_at", "docker_events", ["at"])
|
||||
|
||||
op.create_table(
|
||||
"docker_swarm_services",
|
||||
sa.Column("host_id", sa.String(36),
|
||||
sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True),
|
||||
sa.Column("service_name", sa.String(255), primary_key=True),
|
||||
sa.Column("mode", sa.String(16), nullable=False, server_default="replicated"),
|
||||
sa.Column("desired", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("running", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("image", sa.String(512), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"docker_swarm_nodes",
|
||||
sa.Column("host_id", sa.String(36),
|
||||
sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True),
|
||||
sa.Column("node_id", sa.String(64), primary_key=True),
|
||||
sa.Column("hostname", sa.String(255), nullable=False, server_default=""),
|
||||
sa.Column("role", sa.String(16), nullable=False, server_default="worker"),
|
||||
sa.Column("availability", sa.String(16), nullable=False, server_default="active"),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="unknown"),
|
||||
sa.Column("leader", sa.Boolean, nullable=False, server_default=sa.false()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("docker_swarm_nodes")
|
||||
op.drop_table("docker_swarm_services")
|
||||
op.drop_index("ix_docker_events_at", table_name="docker_events")
|
||||
op.drop_index("ix_docker_events_host_container_time", table_name="docker_events")
|
||||
op.drop_table("docker_events")
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Docker swarm service placement column
|
||||
|
||||
Adds docker_swarm_services.placement_json — the task→node placement of a
|
||||
service's running replicas, captured from the agent's swarm payload (a manager
|
||||
sees every task, so this records cross-node placement the local container rows
|
||||
can't). Additive column; no DROP+recreate.
|
||||
|
||||
Revision ID: docker_005_swarm_placement
|
||||
Revises: docker_004_events_swarm
|
||||
Create Date: 2026-06-19
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "docker_005_swarm_placement"
|
||||
down_revision: Union[str, None] = "docker_004_events_swarm"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("docker_swarm_services",
|
||||
sa.Column("placement_json", sa.Text, nullable=False,
|
||||
server_default="[]"))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("docker_swarm_services", "placement_json")
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Docker hourly metric rollup table
|
||||
|
||||
Adds docker_metrics_hourly — the coarse series that retention rolls raw
|
||||
docker_metrics into before pruning them, so multi-day history stays cheap.
|
||||
One row per (host, container, hour bucket); the unique constraint is the
|
||||
conflict target for the idempotent rollup upsert. Additive create_table.
|
||||
|
||||
Revision ID: docker_006_metric_rollup
|
||||
Revises: docker_005_swarm_placement
|
||||
Create Date: 2026-06-19
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "docker_006_metric_rollup"
|
||||
down_revision: Union[str, None] = "docker_005_swarm_placement"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"docker_metrics_hourly",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("host_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("container_name", sa.String(length=255), nullable=False),
|
||||
sa.Column("bucket", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("cpu_pct", sa.Float(), nullable=False, server_default="0"),
|
||||
sa.Column("mem_pct", sa.Float(), nullable=False, server_default="0"),
|
||||
sa.Column("mem_usage_bytes", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("sample_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("host_id", "container_name", "bucket",
|
||||
name="uq_docker_metrics_hourly_bucket"),
|
||||
)
|
||||
op.create_index("ix_docker_metrics_hourly_bucket",
|
||||
"docker_metrics_hourly", ["bucket"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_docker_metrics_hourly_bucket",
|
||||
table_name="docker_metrics_hourly")
|
||||
op.drop_table("docker_metrics_hourly")
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Docker disk usage + image storage tables
|
||||
|
||||
Adds docker_disk_usage (one per-host /system/df summary row) and docker_images
|
||||
(the heavy-hitter image storage records). Both current-state, host-scoped,
|
||||
re-reported each interval. Additive create_table.
|
||||
|
||||
Revision ID: docker_007_disk_usage
|
||||
Revises: docker_006_metric_rollup
|
||||
Create Date: 2026-06-19
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "docker_007_disk_usage"
|
||||
down_revision: Union[str, None] = "docker_006_metric_rollup"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"docker_disk_usage",
|
||||
sa.Column("host_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("layers_size", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("images_size", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("images_reclaimable", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("containers_size", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("volumes_size", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("build_cache_size", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("images_total", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("images_active", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("containers_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("volumes_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("host_id"),
|
||||
)
|
||||
op.create_table(
|
||||
"docker_images",
|
||||
sa.Column("host_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("image_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("repo_tag", sa.String(length=512), nullable=False, server_default="<none>"),
|
||||
sa.Column("size", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("shared_size", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("containers", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("host_id", "image_id"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("docker_images")
|
||||
op.drop_table("docker_disk_usage")
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Widen memory byte columns to BIGINT
|
||||
|
||||
docker_metrics.mem_usage_bytes and docker_containers.mem_usage_bytes /
|
||||
mem_limit_bytes were int4, which overflows for any container using >2.1 GB
|
||||
(asyncpg: "value out of int32 range"), failing the whole ingest batch. Widen to
|
||||
BIGINT — a safe in-place int4→int8 promotion, no data rewrite.
|
||||
|
||||
Revision ID: docker_008_bigint_mem
|
||||
Revises: docker_007_disk_usage
|
||||
Create Date: 2026-06-20
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "docker_008_bigint_mem"
|
||||
down_revision: Union[str, None] = "docker_007_disk_usage"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.alter_column("docker_metrics", "mem_usage_bytes", type_=sa.BigInteger())
|
||||
op.alter_column("docker_containers", "mem_usage_bytes", type_=sa.BigInteger())
|
||||
op.alter_column("docker_containers", "mem_limit_bytes", type_=sa.BigInteger())
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Narrowing back to int4 will error if any stored value exceeds 2^31 — that's
|
||||
# the very condition this migration fixed, so downgrade is best-effort.
|
||||
op.alter_column("docker_containers", "mem_limit_bytes", type_=sa.Integer())
|
||||
op.alter_column("docker_containers", "mem_usage_bytes", type_=sa.Integer())
|
||||
op.alter_column("docker_metrics", "mem_usage_bytes", type_=sa.Integer())
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Docker container logs table
|
||||
|
||||
Adds docker_logs — one row per container log line, pushed by the host agent and
|
||||
folded into its metrics push. Time-series, host-scoped (container names are only
|
||||
unique within a host). Bounded by a per-container size+age ring in retention, so
|
||||
a chatty container keeps a shorter window rather than growing without limit.
|
||||
Additive create_table + twin indexes (viewer lookup by (host, container, ts);
|
||||
age-cutoff prune by ts).
|
||||
|
||||
Revision ID: docker_009_container_logs
|
||||
Revises: docker_008_bigint_mem
|
||||
Create Date: 2026-07-19
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "docker_009_container_logs"
|
||||
down_revision: Union[str, None] = "docker_008_bigint_mem"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"docker_logs",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("host_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("container_name", sa.String(length=255), nullable=False),
|
||||
sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("stream", sa.String(length=8), nullable=False, server_default="stdout"),
|
||||
sa.Column("line", sa.Text(), nullable=False, server_default=""),
|
||||
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index("ix_docker_logs_host_id", "docker_logs", ["host_id"])
|
||||
op.create_index("ix_docker_logs_container_name", "docker_logs", ["container_name"])
|
||||
op.create_index("ix_docker_logs_host_container_time",
|
||||
"docker_logs", ["host_id", "container_name", "ts"])
|
||||
op.create_index("ix_docker_logs_ts", "docker_logs", ["ts"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_docker_logs_ts", table_name="docker_logs")
|
||||
op.drop_index("ix_docker_logs_host_container_time", table_name="docker_logs")
|
||||
op.drop_index("ix_docker_logs_container_name", table_name="docker_logs")
|
||||
op.drop_index("ix_docker_logs_host_id", table_name="docker_logs")
|
||||
op.drop_table("docker_logs")
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Docker swarm service tasks column
|
||||
|
||||
Adds docker_swarm_services.tasks_json — each service's newest tasks as a
|
||||
manager's agent (≥ 1.8.0) reports them: slot, node, desired and current state,
|
||||
Swarm's error and message, exit code and time. Lets Steward tell a task Swarm
|
||||
replaced from one that crashed, count a service's failures across task
|
||||
retries, and say why its last task failed. Additive column; no DROP+recreate.
|
||||
|
||||
Revision ID: docker_010_swarm_tasks
|
||||
Revises: docker_009_container_logs
|
||||
Create Date: 2026-10-06
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "docker_010_swarm_tasks"
|
||||
down_revision: Union[str, None] = "docker_009_container_logs"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("docker_swarm_services",
|
||||
sa.Column("tasks_json", sa.Text, nullable=False,
|
||||
server_default="[]"))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("docker_swarm_services", "tasks_json")
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Docker host status
|
||||
|
||||
Adds docker_host_status: per host, whether Docker answered at the agent's
|
||||
last report (ok / unavailable / error) and why not. Lets Steward show a host
|
||||
whose Docker went away as such, instead of its last containers as running
|
||||
(#5152). Additive create_table.
|
||||
|
||||
Revision ID: docker_011_host_status
|
||||
Revises: docker_010_swarm_tasks
|
||||
Create Date: 2026-10-06
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "docker_011_host_status"
|
||||
down_revision: Union[str, None] = "docker_010_swarm_tasks"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"docker_host_status",
|
||||
sa.Column("host_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("status", sa.String(length=16), nullable=False),
|
||||
sa.Column("detail", sa.String(length=255), nullable=True),
|
||||
sa.Column("since", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("reported_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("host_id"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("docker_host_status")
|
||||
@@ -0,0 +1,327 @@
|
||||
# plugins/docker/models.py
|
||||
from __future__ import annotations
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy import (
|
||||
BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text,
|
||||
UniqueConstraint,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from steward.models.base import Base
|
||||
from steward.core.ids import time_ordered_id
|
||||
|
||||
|
||||
class DockerContainer(Base):
|
||||
"""Latest known state per container, scoped to the host that reported it.
|
||||
|
||||
Collection is per-host via the host agent, so container names are only
|
||||
unique within a host — the natural key is (host_id, name). host_id is NOT
|
||||
NULL: every container arrives through a host_agent ingest that resolves a
|
||||
Host first. Deleting a host cascades its containers away.
|
||||
"""
|
||||
__tablename__ = "docker_containers"
|
||||
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
container_id: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
image: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="unknown")
|
||||
# running | stopped | paused | exited | dead
|
||||
cpu_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
# BigInteger: a container's memory usage/limit routinely exceeds 2^31 bytes
|
||||
# (~2.1 GB), which overflows int4 and fails the insert.
|
||||
mem_usage_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
mem_limit_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
mem_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
restart_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
ports_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
|
||||
# JSON: [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}]
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
scraped_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
# ── Enrichment (agent ≥ 1.4.0) ────────────────────────────────────────────
|
||||
# Health/exit/restart come from `docker inspect` (not the list endpoint);
|
||||
# exit_code is only meaningful for stopped containers.
|
||||
health: Mapped[str | None] = mapped_column(String(16), nullable=True) # healthy|unhealthy|starting
|
||||
exit_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
oom_killed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
# Grouping: compose project + swarm placement, read off container labels.
|
||||
compose_project: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
service_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
task_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
node_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
# Cumulative-since-start I/O counters (BigInteger — they exceed 2^31 quickly).
|
||||
net_rx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
net_tx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
blk_read_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
blk_write_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
|
||||
|
||||
class DockerMetric(Base):
|
||||
"""Time-series CPU/memory per container — one row per sample per running
|
||||
container, scoped to the reporting host."""
|
||||
__tablename__ = "docker_metrics"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36), primary_key=True, default=time_ordered_id
|
||||
)
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
scraped_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
index=True,
|
||||
)
|
||||
cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
# BigInteger: per-container memory usage exceeds 2^31 bytes (~2.1 GB) routinely.
|
||||
mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
|
||||
# Per-container history lookups filter on (host_id, container_name) then sort
|
||||
# by time — a composite index keeps the rows() sparkline queries cheap.
|
||||
__table_args__ = (
|
||||
Index("ix_docker_metrics_host_container_time",
|
||||
"host_id", "container_name", "scraped_at"),
|
||||
)
|
||||
|
||||
|
||||
class DockerMetricHourly(Base):
|
||||
"""Hourly rollup of docker_metrics — avg cpu/mem per container per hour.
|
||||
|
||||
Raw per-sample rows (~2880/container/day at 30s) are pruned beyond a short
|
||||
window; before deletion they're aggregated here so multi-day history stays
|
||||
cheap to store and query. One row per (host, container, hour bucket); the
|
||||
unique constraint lets retention upsert idempotently if it re-runs before the
|
||||
raw rows are deleted. `bucket` is the hour-truncated sample time.
|
||||
"""
|
||||
__tablename__ = "docker_metrics_hourly"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36), primary_key=True, default=time_ordered_id
|
||||
)
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
container_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
__table_args__ = (
|
||||
# One bucket per container per host — the conflict target for the
|
||||
# idempotent rollup upsert; doubles as the history-query index.
|
||||
UniqueConstraint("host_id", "container_name", "bucket",
|
||||
name="uq_docker_metrics_hourly_bucket"),
|
||||
Index("ix_docker_metrics_hourly_bucket", "bucket"),
|
||||
)
|
||||
|
||||
|
||||
class DockerEvent(Base):
|
||||
"""Lifecycle events derived by diffing consecutive host snapshots.
|
||||
|
||||
The agent only reports current state, so Steward synthesises lifecycle by
|
||||
comparing each new snapshot against the stored DockerContainer state for the
|
||||
host: a container that appears → `start`; one that vanishes → `stop`; a
|
||||
transition to a stopped status with a non-zero exit → `die`; an OOM-kill
|
||||
flip → `oom`; a health status change → `health_change`. Scoped to the
|
||||
reporting host (names are only unique within a host). `event` is a plain
|
||||
string (no CHECK whitelist), matching how `status` is modelled on
|
||||
DockerContainer — keeps the value set open without a migration per addition.
|
||||
"""
|
||||
__tablename__ = "docker_events"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36), primary_key=True, default=time_ordered_id
|
||||
)
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
container_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
event: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
# start | restart | stop | die | oom | health_change
|
||||
detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
# e.g. "exit 137", "healthy→unhealthy", image on start
|
||||
at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
# Timeline lookups filter on (host_id, container_name) and sort by time;
|
||||
# retention prunes by `at` alone — index both access paths.
|
||||
__table_args__ = (
|
||||
Index("ix_docker_events_host_container_time",
|
||||
"host_id", "container_name", "at"),
|
||||
Index("ix_docker_events_at", "at"),
|
||||
)
|
||||
|
||||
|
||||
class DockerLog(Base):
|
||||
"""Container log lines pushed by the host agent — one row per line.
|
||||
|
||||
Time-series, scoped to the reporting host (container names are only unique
|
||||
within a host, same identity as docker_metrics). The agent tails each running
|
||||
container incrementally and folds new lines into its metrics push; `ts` is the
|
||||
line's own Docker timestamp. Bounded by a per-container size+age ring
|
||||
(retention), so a chatty container just keeps a shorter window rather than
|
||||
growing without limit.
|
||||
"""
|
||||
__tablename__ = "docker_logs"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36), primary_key=True, default=time_ordered_id
|
||||
)
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
ts: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
stream: Mapped[str] = mapped_column(String(8), nullable=False, default="stdout")
|
||||
# stdout | stderr
|
||||
line: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
|
||||
# Viewer filters on (host_id, container_name) and sorts by time; the ring
|
||||
# prune walks the same key newest-first. A second index on `ts` alone serves
|
||||
# the age-cutoff delete. Twin-index idiom, mirroring docker_events.
|
||||
__table_args__ = (
|
||||
Index("ix_docker_logs_host_container_time",
|
||||
"host_id", "container_name", "ts"),
|
||||
Index("ix_docker_logs_ts", "ts"),
|
||||
)
|
||||
|
||||
|
||||
class DockerSwarmService(Base):
|
||||
"""A Swarm service as seen by a manager host: desired vs running replicas.
|
||||
|
||||
Manager-scoped — only a host that is a Swarm manager reports these, so a
|
||||
single-manager cluster yields one row set keyed by that host. Service names
|
||||
are unique within a swarm; we still key by (host_id, service_name) so two
|
||||
managers reporting independently don't collide.
|
||||
"""
|
||||
__tablename__ = "docker_swarm_services"
|
||||
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
service_name: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
mode: Mapped[str] = mapped_column(String(16), nullable=False, default="replicated")
|
||||
# replicated | global
|
||||
desired: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
running: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
image: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
# task→node placement of running replicas: [{"node_id", "running"}], JSON.
|
||||
# A manager sees every task, so this captures cross-node placement that the
|
||||
# local docker_containers rows (this host only) can't.
|
||||
placement_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
|
||||
# The service's newest tasks, newest first, as the manager's agent shaped
|
||||
# them: [{"task_id", "slot", "node_id", "desired_state", "state", "message",
|
||||
# "error", "exit_code", "at"}], JSON. Swarm's own record of which tasks it
|
||||
# replaced and why each one stopped; "[]" from agents older than 1.8.0.
|
||||
tasks_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
class DockerDiskUsage(Base):
|
||||
"""Per-host Docker disk usage summary from `/system/df` (current state).
|
||||
|
||||
One row per host (the agent re-reports the full summary each interval).
|
||||
Reclaimable = bytes held by images no container references. Sizes are bytes
|
||||
(BigInteger — image caches exceed 2^31 easily).
|
||||
"""
|
||||
__tablename__ = "docker_disk_usage"
|
||||
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
layers_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
images_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
images_reclaimable: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
containers_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
volumes_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
build_cache_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
images_total: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
images_active: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
containers_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
volumes_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
class DockerImage(Base):
|
||||
"""Per-host image storage record (the heavy hitters from `/system/df`).
|
||||
|
||||
Keyed (host_id, image_id); `containers` is the reference count (0 ⇒ the
|
||||
image's size is reclaimable). Replaced wholesale per host on each report.
|
||||
"""
|
||||
__tablename__ = "docker_images"
|
||||
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
image_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
repo_tag: Mapped[str] = mapped_column(String(512), nullable=False, default="<none>")
|
||||
size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
shared_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
containers: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
class DockerSwarmNode(Base):
|
||||
"""A Swarm node as seen by a manager host: role / availability / status."""
|
||||
__tablename__ = "docker_swarm_nodes"
|
||||
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
node_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
hostname: Mapped[str] = mapped_column(String(255), nullable=False, default="")
|
||||
role: Mapped[str] = mapped_column(String(16), nullable=False, default="worker")
|
||||
# manager | worker
|
||||
availability: Mapped[str] = mapped_column(String(16), nullable=False, default="active")
|
||||
# active | pause | drain
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="unknown")
|
||||
# ready | down | unknown
|
||||
leader: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
class DockerHostStatus(Base):
|
||||
"""Whether Docker answered on a host at its agent's last report (agent ≥ 1.9.0).
|
||||
|
||||
"ok" when it did; "unavailable" when there is no Docker to ask (no socket,
|
||||
nothing listening); "error" when Docker is there and the call failed.
|
||||
`detail` says why for the last two. While a host is not "ok" its container
|
||||
rows are what Docker last said, not what is true now, and read as stale.
|
||||
A host that never had Docker gets no row.
|
||||
"""
|
||||
__tablename__ = "docker_host_status"
|
||||
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
# When the status last changed, and the agent's newest report.
|
||||
since: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
reported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
@@ -0,0 +1,16 @@
|
||||
name: docker
|
||||
version: "2.0.0"
|
||||
description: "Per-host Docker container status + resource usage, collected by the host agent"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker"
|
||||
tags:
|
||||
- containers
|
||||
- docker
|
||||
- infrastructure
|
||||
|
||||
# No config: collection is agent-driven (the host agent reads each host's local
|
||||
# socket and pushes containers to the host_agent ingest). This plugin is pure
|
||||
# presentation + storage, so there's no socket path or scrape interval to tune.
|
||||
@@ -0,0 +1,160 @@
|
||||
# plugins/docker/retention.py
|
||||
"""Bound Docker time-series growth: roll up old metrics, prune old rows.
|
||||
|
||||
Published as the "docker.run_retention" capability (see __init__.setup) so the
|
||||
core cleanup task can drive it WITHOUT importing the docker models (same
|
||||
opportunistic-coupling pattern as docker.persist_host_samples). Runs inside the
|
||||
caller's open transaction; never opens or commits its own.
|
||||
|
||||
The scaling concern is docker_metrics: ~2880 rows/container/day at a 30s sample.
|
||||
We keep raw samples for a short window, then aggregate everything older into
|
||||
hourly averages (docker_metrics_hourly) and delete the raw rows — so multi-day
|
||||
history stays cheap to store and query. docker_events is light but unbounded
|
||||
without a cutoff, so it gets a (longer) window too.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
||||
def _hour_floor(dt: datetime) -> datetime:
|
||||
"""Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
|
||||
return dt.replace(minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
|
||||
"""Hour-aligned boundary below which raw metrics get rolled up + deleted.
|
||||
|
||||
Aligning to the hour means we only ever roll up *whole* elapsed hours — a
|
||||
bucket is never split across the keep/roll boundary, so re-running can't
|
||||
produce a partial-then-complete duplicate for the same hour.
|
||||
"""
|
||||
return _hour_floor(now - timedelta(days=raw_days))
|
||||
|
||||
|
||||
async def run_docker_retention(
|
||||
session,
|
||||
*,
|
||||
events_days: int,
|
||||
metrics_raw_days: int,
|
||||
metrics_rollup_days: int,
|
||||
logs_retention_days: int = 3,
|
||||
logs_max_bytes_per_container: int = 5_000_000,
|
||||
now: datetime | None = None,
|
||||
) -> dict:
|
||||
"""Roll up + prune Docker time-series. Returns a counts dict for logging.
|
||||
|
||||
1. Aggregate docker_metrics older than the (hour-aligned) raw window into
|
||||
docker_metrics_hourly (avg cpu/mem per container per hour), upserting so a
|
||||
re-run is idempotent, then delete those raw rows.
|
||||
2. Prune rolled-up rows older than the rollup window.
|
||||
3. Prune docker_events older than the events window.
|
||||
4. Prune docker_logs with a per-container size+age ring (m79): drop lines
|
||||
older than the age window, then keep only the newest ~cap bytes per
|
||||
(host, container).
|
||||
"""
|
||||
from datetime import timezone
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from .models import DockerEvent, DockerLog, DockerMetric, DockerMetricHourly
|
||||
|
||||
if now is None:
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
rolled = rolled_rows = events_pruned = rollup_pruned = 0
|
||||
logs_age_pruned = logs_size_pruned = 0
|
||||
|
||||
# ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
|
||||
raw_cutoff = _rollup_cutoff(now, metrics_raw_days)
|
||||
hour = func.date_trunc("hour", DockerMetric.scraped_at)
|
||||
agg = (
|
||||
select(
|
||||
DockerMetric.host_id,
|
||||
DockerMetric.container_name,
|
||||
hour.label("bucket"),
|
||||
func.avg(DockerMetric.cpu_pct).label("cpu_pct"),
|
||||
func.avg(DockerMetric.mem_pct).label("mem_pct"),
|
||||
func.avg(DockerMetric.mem_usage_bytes).label("mem_usage_bytes"),
|
||||
func.count().label("sample_count"),
|
||||
)
|
||||
.where(DockerMetric.scraped_at < raw_cutoff)
|
||||
.group_by(DockerMetric.host_id, DockerMetric.container_name, hour)
|
||||
)
|
||||
for r in (await session.execute(agg)).all():
|
||||
stmt = (
|
||||
pg_insert(DockerMetricHourly)
|
||||
.values(
|
||||
host_id=r.host_id,
|
||||
container_name=r.container_name,
|
||||
bucket=r.bucket,
|
||||
cpu_pct=float(r.cpu_pct or 0.0),
|
||||
mem_pct=float(r.mem_pct or 0.0),
|
||||
mem_usage_bytes=int(r.mem_usage_bytes or 0),
|
||||
sample_count=int(r.sample_count or 0),
|
||||
)
|
||||
.on_conflict_do_update(
|
||||
constraint="uq_docker_metrics_hourly_bucket",
|
||||
set_={
|
||||
"cpu_pct": float(r.cpu_pct or 0.0),
|
||||
"mem_pct": float(r.mem_pct or 0.0),
|
||||
"mem_usage_bytes": int(r.mem_usage_bytes or 0),
|
||||
"sample_count": int(r.sample_count or 0),
|
||||
},
|
||||
)
|
||||
)
|
||||
await session.execute(stmt)
|
||||
rolled += 1
|
||||
rolled_rows += int(r.sample_count or 0)
|
||||
if rolled:
|
||||
await session.execute(
|
||||
delete(DockerMetric).where(DockerMetric.scraped_at < raw_cutoff)
|
||||
)
|
||||
|
||||
# ── 2. Prune rolled-up rows beyond the rollup window ──
|
||||
rollup_cutoff = now - timedelta(days=metrics_rollup_days)
|
||||
res = await session.execute(
|
||||
delete(DockerMetricHourly).where(DockerMetricHourly.bucket < rollup_cutoff)
|
||||
)
|
||||
rollup_pruned = res.rowcount or 0
|
||||
|
||||
# ── 3. Prune lifecycle events beyond the events window ──
|
||||
events_cutoff = now - timedelta(days=events_days)
|
||||
res = await session.execute(
|
||||
delete(DockerEvent).where(DockerEvent.at < events_cutoff)
|
||||
)
|
||||
events_pruned = res.rowcount or 0
|
||||
|
||||
# ── 4. Container-log ring: age cutoff, then per-container byte cap (m79) ──
|
||||
logs_cutoff = now - timedelta(days=logs_retention_days)
|
||||
res = await session.execute(
|
||||
delete(DockerLog).where(DockerLog.ts < logs_cutoff)
|
||||
)
|
||||
logs_age_pruned = res.rowcount or 0
|
||||
|
||||
# Size ring: per (host, container), sum line bytes newest-first; delete a row
|
||||
# once its strictly-newer siblings already fill the cap. Using the EXCLUSIVE
|
||||
# prefix (running total minus this row) means the newest row always survives,
|
||||
# so a single line larger than the cap is never wiped out.
|
||||
running = func.sum(func.length(DockerLog.line)).over(
|
||||
partition_by=[DockerLog.host_id, DockerLog.container_name],
|
||||
order_by=[DockerLog.ts.desc(), DockerLog.id.desc()],
|
||||
)
|
||||
prefix_excl = (running - func.length(DockerLog.line)).label("prefix_excl")
|
||||
ranked = select(DockerLog.id, prefix_excl).subquery()
|
||||
over_cap = select(ranked.c.id).where(
|
||||
ranked.c.prefix_excl >= logs_max_bytes_per_container)
|
||||
res = await session.execute(
|
||||
delete(DockerLog).where(DockerLog.id.in_(over_cap))
|
||||
)
|
||||
logs_size_pruned = res.rowcount or 0
|
||||
|
||||
return {
|
||||
"buckets_rolled": rolled,
|
||||
"raw_rows_rolled": rolled_rows,
|
||||
"rollup_pruned": rollup_pruned,
|
||||
"events_pruned": events_pruned,
|
||||
"logs_age_pruned": logs_age_pruned,
|
||||
"logs_size_pruned": logs_size_pruned,
|
||||
}
|
||||
@@ -0,0 +1,696 @@
|
||||
# plugins/docker/routes.py
|
||||
from __future__ import annotations
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from quart import (
|
||||
Blueprint, current_app, jsonify, redirect, render_template, request,
|
||||
session, url_for,
|
||||
)
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.models.users import UserRole
|
||||
from steward.models.hosts import Host
|
||||
from steward.core.formatting import human_bytes, human_duration, seconds_since
|
||||
from steward.core.time_range import parse_range, DEFAULT_RANGE
|
||||
from . import service as docker_svc
|
||||
from .dedup import dedup_by_container_id
|
||||
from .swarm_view import (
|
||||
build_swarm_services, failing_services, parse_tasks, replaced_tasks, services_problems_first,
|
||||
task_summary, trouble_since,
|
||||
)
|
||||
from .models import (
|
||||
DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage, DockerMetric,
|
||||
DockerSwarmNode, DockerSwarmService,
|
||||
)
|
||||
|
||||
docker_bp = Blueprint("docker", __name__, template_folder="templates")
|
||||
|
||||
|
||||
def _error(status: int, code: str, detail: str | None = None):
|
||||
body: dict = {"ok": False, "error": code}
|
||||
if detail:
|
||||
body["detail"] = detail
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
def _uptime(started_at: datetime | None) -> str | None:
|
||||
"""How long a container has run ('3d 4h'), or None if unknown."""
|
||||
secs = seconds_since(started_at)
|
||||
return None if secs is None or secs < 0 else human_duration(secs)
|
||||
|
||||
|
||||
def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
|
||||
if len(values) < 2:
|
||||
return f'<svg width="{width}" height="{height}"></svg>'
|
||||
mn, mx = min(values), max(values)
|
||||
if mx == mn:
|
||||
mx = mn + 1.0
|
||||
step = width / (len(values) - 1)
|
||||
pts = []
|
||||
for i, v in enumerate(values):
|
||||
x = i * step
|
||||
y = height - (v - mn) / (mx - mn) * (height - 2) - 1
|
||||
pts.append(f"{x:.1f},{y:.1f}")
|
||||
poly = " ".join(pts)
|
||||
return (
|
||||
f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
|
||||
f'style="vertical-align:middle;">'
|
||||
f'<polyline points="{poly}" fill="none" stroke="#6060c0" stroke-width="1.5"/>'
|
||||
f'</svg>'
|
||||
)
|
||||
|
||||
|
||||
@docker_bp.get("/")
|
||||
@require_role(UserRole.viewer)
|
||||
async def index():
|
||||
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
|
||||
# Show the Swarm link only when a manager has actually reported topology, so
|
||||
# non-swarm installs aren't offered an always-empty page.
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
has_swarm = (await db.execute(
|
||||
select(DockerSwarmService.host_id).limit(1))).first() is not None
|
||||
has_disk = (await db.execute(
|
||||
select(DockerDiskUsage.host_id).limit(1))).first() is not None
|
||||
return await render_template(
|
||||
"docker/index.html",
|
||||
poll_interval=poll_interval,
|
||||
has_swarm=has_swarm,
|
||||
has_disk=has_disk,
|
||||
)
|
||||
|
||||
|
||||
@docker_bp.get("/rows")
|
||||
@require_role(UserRole.viewer)
|
||||
async def rows():
|
||||
"""HTMX fragment: what needs attention first, then swarm services
|
||||
(collapsed, cluster-complete), then non-swarm containers grouped by host."""
|
||||
now = datetime.now(timezone.utc)
|
||||
stale_after = docker_svc.stale_after_seconds(current_app.config)
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
# Hosts whose Docker is not answering: their rows are what it last said.
|
||||
docker_down = {d.host_id: d for d in (await db.execute(
|
||||
select(DockerHostStatus).where(DockerHostStatus.status != "ok"))).scalars()}
|
||||
ranked = await docker_svc.ranked_containers(
|
||||
db, now, docker_svc.crash_loop_deaths(current_app.config))
|
||||
containers = await docker_svc.list_containers(db)
|
||||
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
||||
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
||||
hosts = await docker_svc.host_map(
|
||||
db,
|
||||
{c.host_id for c in containers}
|
||||
| {n.host_id for n in nodes} | {s.host_id for s in services} | set(docker_down),
|
||||
)
|
||||
|
||||
# Swarm tasks collapse into per-service rows (each replica labelled with
|
||||
# its host, plus "ghost" replicas for tasks on agent-less nodes, drawn
|
||||
# from the managers' placement). Non-swarm containers keep the host view.
|
||||
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
||||
host_name = {h.id: h.name for h in hosts.values()}
|
||||
swarm_view = build_swarm_services(
|
||||
containers, services, node_hostname, host_name,
|
||||
crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
|
||||
capacity = await docker_svc.host_capacity(db, {c.host_id for c in containers})
|
||||
failures = docker_svc.failures_24h(
|
||||
await docker_svc.failure_events(db, now - timedelta(hours=24)),
|
||||
swarm_view["services"], host_name)
|
||||
|
||||
standalone = [c for c in containers if not c.service_name]
|
||||
groups: dict[str, dict] = {}
|
||||
for c in standalone:
|
||||
g = groups.get(c.host_id)
|
||||
if g is None:
|
||||
host = hosts.get(c.host_id)
|
||||
g = groups[c.host_id] = {
|
||||
"host": host,
|
||||
"host_id": c.host_id,
|
||||
"host_name": host.name if host else c.host_id,
|
||||
"containers": [], "running": 0, "stopped": 0,
|
||||
}
|
||||
g["containers"].append({
|
||||
"container": c,
|
||||
"ports": json.loads(c.ports_json) if c.ports_json else [],
|
||||
"uptime": _uptime(c.started_at) if c.status == "running" else None,
|
||||
"stale": _is_stale(c, now, stale_after, docker_down),
|
||||
"usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None))),
|
||||
"exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
|
||||
})
|
||||
if c.status == "running":
|
||||
g["running"] += 1
|
||||
else:
|
||||
g["stopped"] += 1
|
||||
|
||||
# Sub-group each host's containers by compose project, preserving the
|
||||
# running-first ordering. Containers with no project fall into an unlabelled
|
||||
# bucket rendered flat, so plain hosts look unchanged.
|
||||
for g in groups.values():
|
||||
subs: dict[str, dict] = {}
|
||||
order: list[str] = []
|
||||
for item in g["containers"]:
|
||||
c = item["container"]
|
||||
label = c.compose_project or None
|
||||
key = label or ""
|
||||
if key not in subs:
|
||||
subs[key] = {"label": label, "containers": []}
|
||||
order.append(key)
|
||||
subs[key]["containers"].append(item)
|
||||
g["subgroups"] = [subs[k] for k in order]
|
||||
g["grouped"] = any(s["label"] for s in g["subgroups"])
|
||||
|
||||
host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
|
||||
swarm_services = services_problems_first(swarm_view["services"])
|
||||
standalone_running = sum(g["running"] for g in host_groups)
|
||||
attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down),
|
||||
label=docker_svc.task_label(r["container"], node_hostname))
|
||||
for r in docker_svc.needs_attention(ranked)]
|
||||
failing = failing_services(swarm_services)
|
||||
return await render_template(
|
||||
"docker/rows.html",
|
||||
host_groups=host_groups,
|
||||
swarm_services=swarm_services,
|
||||
failing_services=failing,
|
||||
failures=failures,
|
||||
services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
|
||||
docker_hosts=len({c.host_id for c in containers}),
|
||||
docker_down=sorted(
|
||||
({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status,
|
||||
"detail": d.detail, "since": d.since} for h, d in docker_down.items()),
|
||||
key=lambda d: d["host_name"].lower()),
|
||||
attention=attention, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
|
||||
unhealthy=sum("unhealthy" in r["problems"] for r in attention),
|
||||
restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
|
||||
stale=sum(1 for g in host_groups for i in g["containers"] if i["stale"]),
|
||||
running=standalone_running,
|
||||
stopped=len(standalone) - standalone_running,
|
||||
)
|
||||
|
||||
|
||||
def _is_stale(c, now: datetime, stale_after: int, docker_down=()) -> bool:
|
||||
"""A container whose last report is older than the agent's staleness
|
||||
window, or whose host's Docker is not answering: what the page shows may
|
||||
no longer be true."""
|
||||
return (c.host_id in docker_down or c.scraped_at is None
|
||||
or (now - c.scraped_at).total_seconds() > stale_after)
|
||||
|
||||
|
||||
@docker_bp.get("/widget")
|
||||
@require_role(UserRole.viewer)
|
||||
async def widget():
|
||||
"""HTMX dashboard widget: swarm services and containers, problems first,
|
||||
with how many things failed in the last 24h (linked to the list)."""
|
||||
show_stopped = request.args.get("show_stopped", "no") == "yes"
|
||||
widget_id = request.args.get("wid", "0")
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
all_containers = dedup_by_container_id(await docker_svc.list_containers(db))
|
||||
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
||||
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
||||
events = await docker_svc.failure_events(db, now - timedelta(hours=24))
|
||||
hosts = await docker_svc.host_map(
|
||||
db,
|
||||
{c.host_id for c in all_containers}
|
||||
| {n.host_id for n in nodes} | {s.host_id for s in services},
|
||||
)
|
||||
capacity = await docker_svc.host_capacity(db, {c.host_id for c in all_containers})
|
||||
|
||||
# Swarm tasks collapse into per-service rows (each replica tagged with its
|
||||
# host); non-swarm containers stay grouped by host.
|
||||
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
||||
host_name = {h.id: h.name for h in hosts.values()}
|
||||
swarm_services = services_problems_first(build_swarm_services(
|
||||
all_containers, services, node_hostname, host_name,
|
||||
crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"])
|
||||
failures = docker_svc.failures_24h(events, swarm_services, host_name)
|
||||
|
||||
standalone = [c for c in all_containers if not c.service_name]
|
||||
running = [c for c in standalone if c.status == "running"]
|
||||
display = standalone if show_stopped else running
|
||||
|
||||
def trouble(c) -> int:
|
||||
if (c.status or "") == "restarting" or c.health == "unhealthy":
|
||||
return 0
|
||||
if c.status != "running" and docker_svc.exit_is_failure(c.exit_code, c.oom_killed):
|
||||
return 1
|
||||
return 2 if c.status == "running" else 3
|
||||
|
||||
# Group for display so multi-host fleets read clearly; single-host stays flat.
|
||||
groups: dict[str, dict] = {}
|
||||
for c in sorted(display, key=lambda c: (trouble(c), c.name)):
|
||||
g = groups.setdefault(c.host_id, {
|
||||
"host_id": c.host_id,
|
||||
"host_name": host_name.get(c.host_id, c.host_id),
|
||||
"containers": [],
|
||||
})
|
||||
g["containers"].append({"c": c, "usage": docker_svc.usage(
|
||||
c, *capacity.get(c.host_id, (None, None))),
|
||||
"exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed)})
|
||||
host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
|
||||
|
||||
return await render_template(
|
||||
"docker/widget.html",
|
||||
host_groups=host_groups,
|
||||
multi_host=len(host_groups) > 1,
|
||||
swarm_services=swarm_services,
|
||||
services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
|
||||
running_count=len(running),
|
||||
failures=failures,
|
||||
total_count=len(all_containers) + len(swarm_services),
|
||||
show_stopped=show_stopped,
|
||||
widget_id=widget_id,
|
||||
)
|
||||
|
||||
|
||||
CPU_AVERAGE_MINUTES = 5
|
||||
|
||||
|
||||
@docker_bp.get("/widget/resources")
|
||||
@require_role(UserRole.viewer)
|
||||
async def widget_resources():
|
||||
"""HTMX dashboard widget: the containers using the most CPU, averaged over
|
||||
the last few minutes so the order holds still between polls, with their
|
||||
CPU in cores and memory in bytes."""
|
||||
limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
|
||||
widget_id = request.args.get("wid", "0")
|
||||
since = datetime.now(timezone.utc) - timedelta(minutes=CPU_AVERAGE_MINUTES)
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
# Dedup before the limit, else swarm tasks reported by both managers can
|
||||
# fill the list with duplicates of the same container.
|
||||
running = dedup_by_container_id([
|
||||
c for c in await docker_svc.list_containers(db) if c.status == "running"])
|
||||
average = {(h, n): v for h, n, v in (await db.execute(
|
||||
select(DockerMetric.host_id, DockerMetric.container_name, func.avg(DockerMetric.cpu_pct))
|
||||
.where(DockerMetric.scraped_at >= since)
|
||||
.group_by(DockerMetric.host_id, DockerMetric.container_name)
|
||||
)).all()}
|
||||
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
||||
hosts = await docker_svc.host_map(db, {c.host_id for c in running})
|
||||
capacity = await docker_svc.host_capacity(db, {c.host_id for c in running})
|
||||
|
||||
def cpu_of(c):
|
||||
v = average.get((c.host_id, c.name))
|
||||
return c.cpu_pct if v is None else float(v)
|
||||
|
||||
busiest = sorted(running, key=lambda c: (-(cpu_of(c) or 0), c.name))[:limit]
|
||||
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
||||
rows_data = [{
|
||||
"c": c,
|
||||
"label": docker_svc.task_label(c, node_hostname),
|
||||
"host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id,
|
||||
"usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c)),
|
||||
} for c in busiest]
|
||||
return await render_template(
|
||||
"docker/widget_resources.html",
|
||||
rows=rows_data,
|
||||
minutes=CPU_AVERAGE_MINUTES,
|
||||
multi_host=len({c.host_id for c in busiest}) > 1,
|
||||
widget_id=widget_id,
|
||||
)
|
||||
|
||||
|
||||
@docker_bp.get("/host/<host_id>")
|
||||
@require_role(UserRole.viewer)
|
||||
async def host_panel(host_id: str):
|
||||
"""Per-host Docker fragment embedded on the core Hosts hub via HTMX:
|
||||
problems first (with exit code and since when), then running, then
|
||||
stopped. Swarm tasks Swarm has already replaced are history and left out.
|
||||
|
||||
Renders nothing when the host reports no containers, so hosts without Docker
|
||||
don't carry an empty card.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
containers = await docker_svc.list_containers(db, host_id)
|
||||
if not containers:
|
||||
return ""
|
||||
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
||||
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
||||
failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
|
||||
capacity = await docker_svc.host_capacity(db, {host_id})
|
||||
|
||||
replaced = replaced_tasks(containers, services)
|
||||
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
||||
hour_ago = now - timedelta(hours=1)
|
||||
crash = docker_svc.crash_loop_deaths(current_app.config)
|
||||
rows = []
|
||||
for c in containers:
|
||||
if (c.host_id, c.name) in replaced:
|
||||
continue
|
||||
events = failures.get((c.host_id, c.name), [])
|
||||
recent = sum(1 for e in events if e.at >= hour_ago)
|
||||
problems = docker_svc.container_problems(c, recent, bool(events), crash)
|
||||
rows.append({
|
||||
"c": c, "label": docker_svc.task_label(c, node_hostname), "problems": problems,
|
||||
"since": docker_svc.problem_since(problems, events, hour_ago),
|
||||
"usage": docker_svc.usage(c, *capacity.get(host_id, (None, None))),
|
||||
"uptime": _uptime(c.started_at) if c.status == "running" else None,
|
||||
})
|
||||
rows.sort(key=lambda r: (
|
||||
docker_svc.PROBLEMS.index(r["problems"][0]) if r["problems"] else len(docker_svc.PROBLEMS),
|
||||
r["c"].status != "running", r["label"].lower()))
|
||||
running = sum(1 for r in rows if r["c"].status == "running")
|
||||
return await render_template(
|
||||
"docker/host_panel.html",
|
||||
rows=rows, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
|
||||
running=running, stopped=len(rows) - running,
|
||||
problems=sum(1 for r in rows if r["problems"]),
|
||||
replaced=len(replaced),
|
||||
)
|
||||
|
||||
|
||||
# Glyph + colour per lifecycle event, so the timeline reads at a glance.
|
||||
_EVENT_STYLE = {
|
||||
"start": ("▲", "var(--green)"),
|
||||
"restart": ("↻", "var(--yellow)"),
|
||||
"stop": ("■", "var(--text-muted)"),
|
||||
"die": ("✕", "var(--red)"),
|
||||
"oom": ("☠", "var(--red)"),
|
||||
"health_change": ("◐", "var(--orange)"),
|
||||
}
|
||||
|
||||
|
||||
@docker_bp.get("/container/<host_id>/<name>")
|
||||
@require_role(UserRole.viewer)
|
||||
async def container_detail(host_id: str, name: str):
|
||||
"""Full detail page for one container: facts, resource history, lifecycle."""
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
c = await db.get(DockerContainer, (host_id, name))
|
||||
host = await db.get(Host, host_id)
|
||||
events = await docker_svc.container_events(db, host_id, name) if c is not None else []
|
||||
node_name = None
|
||||
if c is not None and c.node_id:
|
||||
node_name = (await db.execute(
|
||||
select(DockerSwarmNode.hostname)
|
||||
.where(DockerSwarmNode.node_id == c.node_id, DockerSwarmNode.hostname != "")
|
||||
.limit(1)
|
||||
)).scalar()
|
||||
cpus, mem_total = (await docker_svc.host_capacity(db, [host_id])).get(host_id, (None, None)) \
|
||||
if c is not None else (None, None)
|
||||
|
||||
if c is None:
|
||||
return await render_template(
|
||||
"docker/container_detail.html",
|
||||
container=None, host_id=host_id, name=name,
|
||||
), 404
|
||||
|
||||
return await render_template(
|
||||
"docker/container_detail.html",
|
||||
container=c,
|
||||
host=host,
|
||||
host_id=host_id,
|
||||
name=name,
|
||||
label=docker_svc.task_label(c, {c.node_id: node_name} if node_name else None),
|
||||
node_name=node_name,
|
||||
usage=docker_svc.usage(c, cpus, mem_total),
|
||||
exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
|
||||
ports=json.loads(c.ports_json) if c.ports_json else [],
|
||||
uptime=_uptime(c.started_at) if c.status == "running" else None,
|
||||
net_rx=human_bytes(c.net_rx_bytes), net_tx=human_bytes(c.net_tx_bytes),
|
||||
blk_read=human_bytes(c.blk_read_bytes), blk_write=human_bytes(c.blk_write_bytes),
|
||||
events=[
|
||||
{"event": e.event, "detail": e.detail, "at": e.at,
|
||||
"glyph": _EVENT_STYLE.get(e.event, ("•", "var(--text-muted)"))[0],
|
||||
"colour": _EVENT_STYLE.get(e.event, ("•", "var(--text-muted)"))[1]}
|
||||
for e in events
|
||||
],
|
||||
current_range=request.args.get("range", DEFAULT_RANGE),
|
||||
)
|
||||
|
||||
|
||||
@docker_bp.get("/swarm")
|
||||
@require_role(UserRole.viewer)
|
||||
async def swarm():
|
||||
"""Swarm topology page: services with replica health, nodes, placement.
|
||||
|
||||
Swarm services/nodes are cluster-global — every manager's API returns the
|
||||
same list — but each manager reports them independently (rows keyed by
|
||||
host_id). So we group the reporting managers into swarms (managers that share
|
||||
any node_id are the same cluster) and dedup within each: one row per service
|
||||
(by name) and per node (by node_id), keeping the freshest. Without this, two
|
||||
managers in one cluster list every service/node twice.
|
||||
"""
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
services = list((await db.execute(
|
||||
select(DockerSwarmService).order_by(DockerSwarmService.service_name)
|
||||
)).scalars())
|
||||
nodes = list((await db.execute(
|
||||
select(DockerSwarmNode).order_by(
|
||||
DockerSwarmNode.role, DockerSwarmNode.hostname)
|
||||
)).scalars())
|
||||
hosts = await docker_svc.host_map(db, {s.host_id for s in services} | {n.host_id for n in nodes})
|
||||
|
||||
# ── Group reporting managers into swarms by shared node membership ──────────
|
||||
manager_ids = {s.host_id for s in services} | {n.host_id for n in nodes}
|
||||
nodes_by_mgr: dict[str, set[str]] = {}
|
||||
for n in nodes:
|
||||
nodes_by_mgr.setdefault(n.host_id, set()).add(n.node_id)
|
||||
|
||||
parent = {m: m for m in manager_ids}
|
||||
|
||||
def _find(x: str) -> str:
|
||||
while parent[x] != x:
|
||||
parent[x] = parent[parent[x]]
|
||||
x = parent[x]
|
||||
return x
|
||||
|
||||
mgrs = list(manager_ids)
|
||||
for i in range(len(mgrs)):
|
||||
for j in range(i + 1, len(mgrs)):
|
||||
a, b = mgrs[i], mgrs[j]
|
||||
if nodes_by_mgr.get(a) and nodes_by_mgr.get(b) and (nodes_by_mgr[a] & nodes_by_mgr[b]):
|
||||
parent[_find(a)] = _find(b)
|
||||
|
||||
swarm_members: dict[str, set[str]] = {}
|
||||
for m in manager_ids:
|
||||
swarm_members.setdefault(_find(m), set()).add(m)
|
||||
|
||||
# ── Dedup services (by name) + nodes (by node_id) within each swarm ─────────
|
||||
swarms = []
|
||||
for members in swarm_members.values():
|
||||
svc_by_name: dict[str, DockerSwarmService] = {}
|
||||
for s in services:
|
||||
if s.host_id in members and (
|
||||
s.service_name not in svc_by_name
|
||||
or s.updated_at > svc_by_name[s.service_name].updated_at
|
||||
):
|
||||
svc_by_name[s.service_name] = s
|
||||
node_by_id: dict[str, DockerSwarmNode] = {}
|
||||
for n in nodes:
|
||||
if n.host_id in members and (
|
||||
n.node_id not in node_by_id
|
||||
or n.updated_at > node_by_id[n.node_id].updated_at
|
||||
):
|
||||
node_by_id[n.node_id] = n
|
||||
|
||||
node_names = {nid: (nrow.hostname or nid) for nid, nrow in node_by_id.items()}
|
||||
now = datetime.now(timezone.utc)
|
||||
deaths = docker_svc.crash_loop_deaths(current_app.config)
|
||||
svc_dicts = []
|
||||
for s in svc_by_name.values():
|
||||
tasks = parse_tasks(s)
|
||||
summary = task_summary(tasks, now, deaths, node_names)
|
||||
state = ("healthy" if s.running >= s.desired and s.desired > 0
|
||||
else "degraded" if s.running > 0 else "down")
|
||||
placement = []
|
||||
for p in (json.loads(s.placement_json) if s.placement_json else []):
|
||||
nid = p.get("node_id", "")
|
||||
placement.append({
|
||||
"node": node_names.get(nid, nid[:12] or "?"),
|
||||
"running": p.get("running", 0),
|
||||
})
|
||||
svc_dicts.append({
|
||||
"name": s.service_name, "mode": s.mode,
|
||||
"running": s.running, "desired": s.desired, "image": s.image,
|
||||
"healthy": s.running >= s.desired and s.desired > 0,
|
||||
"degraded": 0 < s.running < s.desired,
|
||||
"down": s.running == 0 and s.desired > 0,
|
||||
"state": state,
|
||||
"placement": placement,
|
||||
**summary,
|
||||
"trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
|
||||
})
|
||||
# Problems first: a node that is down or not taking work, then by role.
|
||||
node_rows = sorted(
|
||||
node_by_id.values(), key=lambda n: (
|
||||
n.status == "ready" and n.availability == "active",
|
||||
n.role, (n.hostname or n.node_id).lower())
|
||||
)
|
||||
manager_names = sorted(hosts[m].name if m in hosts else m for m in members)
|
||||
swarms.append({
|
||||
"managers": manager_names,
|
||||
"services": services_problems_first(svc_dicts),
|
||||
"nodes": node_rows,
|
||||
})
|
||||
|
||||
swarms.sort(key=lambda g: g["managers"][0].lower() if g["managers"] else "")
|
||||
return await render_template("docker/swarm.html", swarms=swarms)
|
||||
|
||||
|
||||
@docker_bp.get("/disk")
|
||||
@require_role(UserRole.viewer)
|
||||
async def disk():
|
||||
"""Image/disk usage page: reclaimable space, per-image sizes, stopped count.
|
||||
|
||||
Admins can reclaim space per host via the prune buttons, which fire the
|
||||
audited bundled prune playbook through Ansible (m78) — the collection agent
|
||||
itself stays read-only.
|
||||
"""
|
||||
from steward.core.capabilities import has_capability
|
||||
from steward.models.ansible_inventory import AnsibleTarget
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
|
||||
images = list((await db.execute(
|
||||
select(DockerImage).order_by(DockerImage.size.desc())
|
||||
)).scalars())
|
||||
# Stopped-container count per host, surfaced alongside reclaimable space.
|
||||
stopped_rows = (await db.execute(
|
||||
select(DockerContainer.host_id)
|
||||
.where(DockerContainer.status != "running")
|
||||
)).scalars()
|
||||
stopped_by_host: dict[str, int] = {}
|
||||
for hid in stopped_rows:
|
||||
stopped_by_host[hid] = stopped_by_host.get(hid, 0) + 1
|
||||
hosts = await docker_svc.host_map(db, {s.host_id for s in summaries})
|
||||
# Which hosts have a linked Ansible target — gates the prune buttons
|
||||
# (a target is required to route the playbook run to that host).
|
||||
host_ids = {s.host_id for s in summaries}
|
||||
target_by_host: dict[str, str] = {}
|
||||
if host_ids:
|
||||
for hid, tid in (await db.execute(
|
||||
select(AnsibleTarget.host_id, AnsibleTarget.id)
|
||||
.where(AnsibleTarget.host_id.in_(host_ids))
|
||||
)).all():
|
||||
target_by_host[hid] = tid
|
||||
|
||||
images_by_host: dict[str, list] = {}
|
||||
for im in images:
|
||||
images_by_host.setdefault(im.host_id, []).append({
|
||||
"repo_tag": im.repo_tag, "size": human_bytes(im.size),
|
||||
"shared": human_bytes(im.shared_size), "containers": im.containers,
|
||||
"reclaimable": im.containers == 0,
|
||||
})
|
||||
|
||||
host_groups = [
|
||||
{
|
||||
"host_id": s.host_id,
|
||||
"host_name": hosts[s.host_id].name if s.host_id in hosts else s.host_id,
|
||||
"summary": {
|
||||
"images_total": s.images_total, "images_active": s.images_active,
|
||||
"images_size": human_bytes(s.images_size),
|
||||
"images_reclaimable": human_bytes(s.images_reclaimable),
|
||||
"layers_size": human_bytes(s.layers_size),
|
||||
"containers_count": s.containers_count,
|
||||
"containers_size": human_bytes(s.containers_size),
|
||||
"volumes_count": s.volumes_count,
|
||||
"volumes_size": human_bytes(s.volumes_size),
|
||||
"build_cache_size": human_bytes(s.build_cache_size),
|
||||
},
|
||||
"stopped": stopped_by_host.get(s.host_id, 0),
|
||||
"images": images_by_host.get(s.host_id, []),
|
||||
"has_target": s.host_id in target_by_host,
|
||||
}
|
||||
for s in summaries
|
||||
]
|
||||
host_groups.sort(key=lambda g: g["host_name"].lower())
|
||||
return await render_template(
|
||||
"docker/disk.html", host_groups=host_groups,
|
||||
ansible_available=has_capability("ansible.run_playbook"),
|
||||
)
|
||||
|
||||
|
||||
@docker_bp.post("/disk/<host_id>/prune")
|
||||
@require_role(UserRole.admin)
|
||||
async def disk_prune(host_id: str):
|
||||
"""Reclaim Docker disk on a host by firing the bundled prune playbook via
|
||||
Ansible (audited, admin-gated) — the collection agent stays read-only (m78).
|
||||
`target` selects the scope: containers | images | system.
|
||||
"""
|
||||
from steward.core.capabilities import has_capability, invoke_capability
|
||||
from steward.ansible.sources import BUILTIN_SOURCE_NAME
|
||||
from steward.models.ansible_inventory import AnsibleTarget
|
||||
|
||||
if not has_capability("ansible.run_playbook"):
|
||||
return _error(400, "ansible_unavailable", "Ansible is not available")
|
||||
|
||||
form = await request.form
|
||||
prune_target = (form.get("target", "") or "").strip()
|
||||
if prune_target not in docker_svc.PRUNE_TARGETS:
|
||||
return _error(400, "bad_target", "Unknown prune target")
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
target = (await db.execute(
|
||||
select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
|
||||
)).scalar_one_or_none()
|
||||
if target is None:
|
||||
return _error(400, "no_target",
|
||||
"Link an Ansible target to this host before pruning")
|
||||
|
||||
# extra_vars_map outranks the playbook's own `vars:` defaults.
|
||||
extra_vars = docker_svc.prune_extra_vars(prune_target)
|
||||
|
||||
actor_role = UserRole(session.get("user_role", "viewer"))
|
||||
run, _source, err = await invoke_capability(
|
||||
"ansible.run_playbook", actor_role,
|
||||
current_app._get_current_object(), # type: ignore[attr-defined]
|
||||
source_name=BUILTIN_SOURCE_NAME,
|
||||
playbook_path="maintenance/docker_prune.yml",
|
||||
inventory_scope=f"steward:target:{target.id}",
|
||||
params={"extra_vars_map": extra_vars},
|
||||
triggered_by=session.get("user_id"),
|
||||
)
|
||||
if err:
|
||||
return _error(400, "prune_failed", err)
|
||||
return redirect(url_for("ansible.run_detail", run_id=run.id))
|
||||
|
||||
|
||||
@docker_bp.get("/container/<host_id>/<name>/history")
|
||||
@require_role(UserRole.viewer)
|
||||
async def container_history(host_id: str, name: str):
|
||||
"""HTMX fragment: CPU + memory sparklines for the selected time range."""
|
||||
since, range_key = parse_range(request.args.get("range"))
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
cpu_hist, mem_hist = await docker_svc.container_history(db, host_id, name, since)
|
||||
return await render_template(
|
||||
"docker/_container_history.html",
|
||||
sparkline_cpu=_sparkline(cpu_hist, width=320, height=48),
|
||||
sparkline_mem=_sparkline(mem_hist, width=320, height=48),
|
||||
have_data=len(cpu_hist) >= 2,
|
||||
range_key=range_key,
|
||||
)
|
||||
|
||||
|
||||
# Newest lines returned per fetch — the retained window is bounded by the ring,
|
||||
# but a container can still hold thousands of lines; cap what one fetch renders.
|
||||
_LOG_TAIL_DEFAULT = 500
|
||||
|
||||
|
||||
@docker_bp.get("/container/<host_id>/<name>/logs")
|
||||
@require_role(UserRole.viewer)
|
||||
async def container_logs(host_id: str, name: str):
|
||||
"""Full log-viewer page for one container (lines load via an HTMX fragment)."""
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
host = await db.get(Host, host_id)
|
||||
return await render_template(
|
||||
"docker/container_logs.html", host=host, host_id=host_id, name=name,
|
||||
)
|
||||
|
||||
|
||||
@docker_bp.get("/container/<host_id>/<name>/logs/lines")
|
||||
@require_role(UserRole.viewer)
|
||||
async def container_logs_lines(host_id: str, name: str):
|
||||
"""HTMX fragment: the recent retained log lines, stream + text filtered.
|
||||
Polled every few seconds by the viewer for near-live follow."""
|
||||
stream = request.args.get("stream", "all")
|
||||
query = (request.args.get("q") or "").strip()
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
rows = await docker_svc.query_logs(db, host_id, name, stream, query, _LOG_TAIL_DEFAULT)
|
||||
return await render_template(
|
||||
"docker/_container_logs_lines.html",
|
||||
lines=[{"ts": r.ts, "stream": r.stream, "line": r.line} for r in rows],
|
||||
name=name, tail=_LOG_TAIL_DEFAULT,
|
||||
filtered=bool(query) or stream in ("stdout", "stderr"),
|
||||
)
|
||||
@@ -0,0 +1,521 @@
|
||||
"""Docker queries shared by the plugin's pages and its MCP tools.
|
||||
|
||||
Each function takes an open session and returns rows or plain data, with no
|
||||
Quart request context, so the routes and the /mcp tools (which run outside
|
||||
Quart's pipeline) call the same code.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from steward.core.time_range import bucket_seconds
|
||||
from steward.models.hosts import Host
|
||||
|
||||
from .dedup import dedup_by_container_id
|
||||
from .models import (
|
||||
DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly, DockerSwarmService,
|
||||
)
|
||||
from .swarm_view import replaced_tasks
|
||||
|
||||
|
||||
CRASH_LOOP_DEATHS = 3 # default; Settings → Thresholds sets it
|
||||
|
||||
|
||||
def crash_loop_deaths(config) -> int:
|
||||
"""Deaths within an hour that make a container or a swarm service a crash
|
||||
loop, from Settings → Thresholds (app.config["THRESHOLDS"])."""
|
||||
th = (config.get("THRESHOLDS") or {}).get("crash_loop") or {}
|
||||
return int(th.get("warn") or CRASH_LOOP_DEATHS)
|
||||
|
||||
|
||||
# Exit codes a deliberate stop leaves: SIGTERM (143), and SIGKILL (137) when
|
||||
# the process outlives the stop timeout. Without an OOM kill they are someone
|
||||
# stopping the container, not the container failing.
|
||||
STOP_EXITS = (137, 143)
|
||||
|
||||
|
||||
def exit_is_failure(exit_code, oom_killed: bool = False) -> bool:
|
||||
"""Whether a container's exit is a failure: an OOM kill, or a non-zero
|
||||
exit other than a stop's."""
|
||||
return bool(oom_killed) or exit_code not in (None, 0, *STOP_EXITS)
|
||||
|
||||
|
||||
def _event_exit(detail) -> int | None:
|
||||
"""The exit code in a die event's detail ("exit 1"), or None."""
|
||||
text = (detail or "").strip()
|
||||
if text.startswith("exit "):
|
||||
try:
|
||||
return int(text[5:])
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def event_is_failure(e) -> bool:
|
||||
"""Whether a die/oom event is a failure. An oom always is; a die is unless
|
||||
its exit is a stop's (or clean)."""
|
||||
if e.event == "oom":
|
||||
return True
|
||||
return e.event == "die" and exit_is_failure(_event_exit(e.detail))
|
||||
|
||||
|
||||
def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
|
||||
"""What failed in the last 24h, newest first, for the dashboard count and
|
||||
the Docker page's list of it: one row per plain container that died or was
|
||||
OOM-killed, and one per swarm service with failed tasks.
|
||||
|
||||
`events` are die/oom events of the last 24h. A swarm task's container dies
|
||||
whenever Swarm replaces it, so a service with Swarm's task records counts
|
||||
its failed tasks instead of its containers' deaths; a service without them
|
||||
(an agent older than 1.8.0) falls back to its tasks' deaths.
|
||||
|
||||
Each row: {kind: "container"|"service", name, host_id, host, count, at,
|
||||
detail}.
|
||||
"""
|
||||
by_prefix = {s["name"] + ".": s for s in swarm_services}
|
||||
rows: dict[tuple, dict] = {}
|
||||
# Each manager reports every swarm task, so one task's death can arrive
|
||||
# once per manager: a service counts each dead task container once.
|
||||
task_deaths: dict[tuple, set] = {}
|
||||
for e in sorted(events, key=lambda e: e.at, reverse=True):
|
||||
if not event_is_failure(e):
|
||||
continue
|
||||
svc = next((s for p, s in by_prefix.items() if e.container_name.startswith(p)), None)
|
||||
if svc is not None and svc["tasks_known"]:
|
||||
continue
|
||||
kind, name = ("service", svc["name"]) if svc else ("container", e.container_name)
|
||||
key = (kind, name) if svc else (kind, e.host_id, name)
|
||||
if svc is not None:
|
||||
seen = task_deaths.setdefault(key, set())
|
||||
if e.container_name in seen:
|
||||
continue
|
||||
seen.add(e.container_name)
|
||||
row = rows.get(key)
|
||||
if row is None:
|
||||
rows[key] = {"kind": kind, "name": name, "host_id": e.host_id,
|
||||
"host": host_name.get(e.host_id, e.host_id), "count": 1, "at": e.at,
|
||||
"detail": "OOM-killed" if e.event == "oom" else e.detail}
|
||||
else:
|
||||
row["count"] += 1
|
||||
from .swarm_view import task_time
|
||||
for s in swarm_services:
|
||||
if s["tasks_known"] and s["failures_24h"]:
|
||||
err = s["last_error"] or {}
|
||||
rows[("service", s["name"])] = {
|
||||
"kind": "service", "name": s["name"], "host_id": None, "host": err.get("node"),
|
||||
"count": s["failures_24h"], "at": task_time(err) if err else None,
|
||||
"detail": err.get("error") or err.get("message") or err.get("state")}
|
||||
epoch = datetime.min.replace(tzinfo=timezone.utc)
|
||||
return sorted(rows.values(), key=lambda r: r["at"] or epoch, reverse=True)
|
||||
|
||||
|
||||
def task_label(c, node_hostname: dict | None = None) -> str:
|
||||
"""How a container reads to a person. A swarm task is its service and
|
||||
slot: "codex_db · 1" for a replica, "agent · node-3" for a global
|
||||
service's task on a node. Anything else is its own name."""
|
||||
from .swarm_view import _slot_key
|
||||
svc = getattr(c, "service_name", None)
|
||||
if not svc:
|
||||
return c.name
|
||||
slot = _slot_key(c)
|
||||
if slot.isdigit():
|
||||
return f"{svc} · {slot}"
|
||||
host = (node_hostname or {}).get(slot) or (node_hostname or {}).get(getattr(c, "node_id", None))
|
||||
return f"{svc} · {host}" if host else svc
|
||||
|
||||
|
||||
def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None) -> dict:
|
||||
"""A container's CPU and memory in terms a reader can weigh.
|
||||
|
||||
Docker's CPU% is per core (100% = one core busy), so it is shown as cores
|
||||
and graded as a share of the host's CPUs when the host's core count is
|
||||
known. Docker's memory limit is the host's RAM when none is set, so memory
|
||||
is shown in bytes, against a limit only when one is actually set; mem_pct
|
||||
is then a share of that limit, otherwise of the host's memory. `cpu_pct`
|
||||
replaces the container's last sample (an average, say) when given.
|
||||
"""
|
||||
cpu = getattr(c, "cpu_pct", None) if cpu_pct is None else cpu_pct
|
||||
limit = getattr(c, "mem_limit_bytes", None)
|
||||
limited = bool(limit and mem_total and limit < mem_total * 0.95)
|
||||
return {
|
||||
"cores": None if cpu is None else cpu / 100.0,
|
||||
"cpu_share": None if cpu is None or not cpus else cpu / cpus,
|
||||
"cpus": cpus,
|
||||
"mem_bytes": getattr(c, "mem_usage_bytes", None),
|
||||
"mem_limit": limit if limited else None,
|
||||
"mem_pct": getattr(c, "mem_pct", None),
|
||||
"mem_of": "its limit" if limited else ("the host's memory" if mem_total else None),
|
||||
}
|
||||
|
||||
|
||||
async def host_capacity(db, host_ids) -> dict[str, tuple[int | None, int | None]]:
|
||||
"""{host_id: (CPU cores, memory bytes)} from the host agent's latest
|
||||
readings: cores counted from its per-core CPU series, memory from
|
||||
available ÷ (1 − used%). (None, None) for a host without an agent."""
|
||||
from sqlalchemy import or_
|
||||
from steward.models.metrics import PluginMetricLatest
|
||||
hosts = await host_map(db, set(host_ids))
|
||||
if not hosts:
|
||||
return {}
|
||||
names = {h.name: hid for hid, h in hosts.items()}
|
||||
rows = (await db.execute(
|
||||
select(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name,
|
||||
PluginMetricLatest.value)
|
||||
.where(PluginMetricLatest.source_module == "host_agent",
|
||||
or_(PluginMetricLatest.resource_name.in_(list(names)),
|
||||
*(PluginMetricLatest.resource_name.like(escape_like(n) + ":core%", escape="\\")
|
||||
for n in names)))
|
||||
)).all()
|
||||
cores: dict[str, int] = {}
|
||||
mem: dict[str, dict] = {}
|
||||
for res, metric, value in rows:
|
||||
if res in names:
|
||||
mem.setdefault(res, {})[metric] = value
|
||||
elif metric == "cpu_pct":
|
||||
host = res.rsplit(":core", 1)[0]
|
||||
cores[host] = cores.get(host, 0) + 1
|
||||
out = {}
|
||||
for name, hid in names.items():
|
||||
m = mem.get(name, {})
|
||||
avail, used = m.get("mem_available_bytes"), m.get("mem_used_pct")
|
||||
total = int(avail / (1 - used / 100)) if avail and used is not None and used < 99.5 else None
|
||||
out[hid] = (cores.get(name), total)
|
||||
return out
|
||||
|
||||
|
||||
def bucket(values: list[float], target: int = 40) -> list[float]:
|
||||
"""Bucket-average a series down to ~target points (DB-agnostic, in Python).
|
||||
|
||||
Replaces the old SQL strftime() bucketing, which was SQLite-only and broke
|
||||
on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of
|
||||
rows — averaging into a readable point count keeps the sparkline's shape.
|
||||
"""
|
||||
n = len(values)
|
||||
if n <= target:
|
||||
return values
|
||||
size = (n + target - 1) // target
|
||||
out: list[float] = []
|
||||
for i in range(0, n, size):
|
||||
chunk = values[i:i + size]
|
||||
out.append(sum(chunk) / len(chunk))
|
||||
return out
|
||||
|
||||
|
||||
async def container_history(db, host_id: str, name: str, since) -> tuple[list, list]:
|
||||
"""Return (cpu_series, mem_series) sparkline-ready for one host's container."""
|
||||
rows = (await db.execute(
|
||||
select(DockerMetric.cpu_pct, DockerMetric.mem_pct)
|
||||
.where(DockerMetric.host_id == host_id)
|
||||
.where(DockerMetric.container_name == name)
|
||||
.where(DockerMetric.scraped_at >= since)
|
||||
.order_by(DockerMetric.scraped_at)
|
||||
)).all()
|
||||
cpu = bucket([r.cpu_pct or 0.0 for r in rows])
|
||||
mem = bucket([r.mem_pct or 0.0 for r in rows])
|
||||
return cpu, mem
|
||||
|
||||
|
||||
async def host_map(db, host_ids: set[str]) -> dict[str, Host]:
|
||||
if not host_ids:
|
||||
return {}
|
||||
return {
|
||||
h.id: h for h in (await db.execute(
|
||||
select(Host).where(Host.id.in_(host_ids)))).scalars().all()
|
||||
}
|
||||
|
||||
|
||||
def escape_like(text: str) -> str:
|
||||
"""Make `text` match literally inside a LIKE/ILIKE pattern (escape char \\)."""
|
||||
return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
|
||||
async def list_containers(db, host_id: str | None = None) -> list[DockerContainer]:
|
||||
"""Containers, running first then by name; one host's when host_id is given."""
|
||||
stmt = select(DockerContainer)
|
||||
if host_id is not None:
|
||||
stmt = stmt.where(DockerContainer.host_id == host_id)
|
||||
stmt = stmt.order_by((DockerContainer.status == "running").desc(), DockerContainer.name)
|
||||
return list((await db.execute(stmt)).scalars())
|
||||
|
||||
|
||||
async def container_events(db, host_id: str, name: str, limit: int = 50) -> list[DockerEvent]:
|
||||
"""A container's lifecycle events (start/stop/die/oom/health), newest first."""
|
||||
return list((await db.execute(
|
||||
select(DockerEvent)
|
||||
.where(DockerEvent.host_id == host_id)
|
||||
.where(DockerEvent.container_name == name)
|
||||
.order_by(DockerEvent.at.desc())
|
||||
.limit(limit)
|
||||
)).scalars())
|
||||
|
||||
|
||||
async def query_logs(db, host_id: str, name: str, stream: str, query: str,
|
||||
limit: int) -> list:
|
||||
"""The recent retained lines for one container, newest-first, optionally
|
||||
filtered by stream and a case-insensitive substring.
|
||||
|
||||
Newest-first is deliberate: the viewer replaces the list on each poll, which
|
||||
resets scroll to the top — so the latest lines stay visible without any
|
||||
scroll handling, and older lines are a scroll away.
|
||||
"""
|
||||
stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line)
|
||||
.where(DockerLog.host_id == host_id)
|
||||
.where(DockerLog.container_name == name))
|
||||
if stream in ("stdout", "stderr"):
|
||||
stmt = stmt.where(DockerLog.stream == stream)
|
||||
if query:
|
||||
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\"))
|
||||
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
|
||||
return (await db.execute(stmt)).all()
|
||||
|
||||
|
||||
CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes")
|
||||
|
||||
|
||||
async def container_points(db, host_id: str, name: str, metric: str, since,
|
||||
target: int = 120) -> list[tuple]:
|
||||
"""(time, average) points of one container metric since `since`, oldest first.
|
||||
|
||||
Raw samples are bucket-averaged in SQL; the part of the range older than
|
||||
the oldest raw sample (already rolled up and pruned by retention) comes
|
||||
from the hourly table.
|
||||
"""
|
||||
if metric not in CONTAINER_SERIES:
|
||||
raise ValueError(f"metric must be one of {', '.join(CONTAINER_SERIES)}")
|
||||
raw_col = getattr(DockerMetric, metric)
|
||||
width_s = bucket_seconds(since, target)
|
||||
rbucket = func.date_bin(
|
||||
func.make_interval(0, 0, 0, 0, 0, 0, width_s), DockerMetric.scraped_at, func.to_timestamp(0),
|
||||
).label("bucket")
|
||||
raw = (await db.execute(
|
||||
select(rbucket, func.avg(raw_col))
|
||||
.where(DockerMetric.host_id == host_id, DockerMetric.container_name == name,
|
||||
DockerMetric.scraped_at >= since)
|
||||
.group_by(rbucket).order_by(rbucket)
|
||||
)).all()
|
||||
oldest_raw = raw[0][0] if raw else None
|
||||
hourly_stmt = (
|
||||
select(DockerMetricHourly.bucket, getattr(DockerMetricHourly, metric))
|
||||
.where(DockerMetricHourly.host_id == host_id,
|
||||
DockerMetricHourly.container_name == name,
|
||||
DockerMetricHourly.bucket >= since)
|
||||
.order_by(DockerMetricHourly.bucket)
|
||||
)
|
||||
if oldest_raw is not None:
|
||||
hourly_stmt = hourly_stmt.where(DockerMetricHourly.bucket < oldest_raw)
|
||||
hourly = (await db.execute(hourly_stmt)).all()
|
||||
return [(t, float(v)) for t, v in hourly] + [(t, float(v)) for t, v in raw]
|
||||
|
||||
|
||||
async def search_logs(db, *, since, host_id: str | None = None, name: str | None = None,
|
||||
text: str = "", stream: str = "", before=None, limit: int = 50) -> list:
|
||||
"""Retained log lines matching a filter, newest first (limit + 1 fetched by callers
|
||||
that need to know whether more exist)."""
|
||||
stmt = (select(DockerLog.id, DockerLog.host_id, DockerLog.container_name,
|
||||
DockerLog.ts, DockerLog.stream, DockerLog.line)
|
||||
.where(DockerLog.ts >= since))
|
||||
if host_id:
|
||||
stmt = stmt.where(DockerLog.host_id == host_id)
|
||||
if name:
|
||||
stmt = stmt.where(DockerLog.container_name == name)
|
||||
if stream in ("stdout", "stderr"):
|
||||
stmt = stmt.where(DockerLog.stream == stream)
|
||||
if text:
|
||||
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(text)}%", escape="\\"))
|
||||
if before is not None:
|
||||
stmt = stmt.where(DockerLog.ts < before)
|
||||
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
|
||||
return list((await db.execute(stmt)).all())
|
||||
|
||||
|
||||
async def log_context(db, host_id: str, name: str, ts, n: int) -> tuple[list, list]:
|
||||
"""Up to n lines either side of `ts` in one container's log, each oldest first."""
|
||||
cols = (DockerLog.id, DockerLog.ts, DockerLog.stream, DockerLog.line)
|
||||
base = select(*cols).where(DockerLog.host_id == host_id, DockerLog.container_name == name)
|
||||
before = (await db.execute(
|
||||
base.where(DockerLog.ts < ts).order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(n)
|
||||
)).all()
|
||||
after = (await db.execute(
|
||||
base.where(DockerLog.ts > ts).order_by(DockerLog.ts, DockerLog.id).limit(n)
|
||||
)).all()
|
||||
return list(reversed(before)), list(after)
|
||||
|
||||
|
||||
async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]:
|
||||
"""{(host_id, container_name): [die/oom events since `since`, newest first]},
|
||||
leaving out a die that was a stop (event_is_failure)."""
|
||||
rows = (await db.execute(
|
||||
select(DockerEvent)
|
||||
.where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
|
||||
.order_by(DockerEvent.at.desc())
|
||||
)).scalars()
|
||||
out: dict[tuple[str, str], list] = {}
|
||||
for e in rows:
|
||||
if event_is_failure(e):
|
||||
out.setdefault((e.host_id, e.container_name), []).append(e)
|
||||
return out
|
||||
|
||||
|
||||
async def failure_events(db, since) -> list:
|
||||
"""Die and OOM events since `since`, newest first; failures_24h() decides
|
||||
which of them are failures."""
|
||||
return list((await db.execute(
|
||||
select(DockerEvent)
|
||||
.where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
|
||||
.order_by(DockerEvent.at.desc())
|
||||
)).scalars())
|
||||
|
||||
|
||||
async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None,
|
||||
limit: int = 20) -> list[tuple[str, str, datetime]]:
|
||||
"""(host_id, container_name, last seen) of containers Steward still holds
|
||||
logs or events for, newest first — including ones Docker has removed.
|
||||
|
||||
exact: the name itself, from both logs and events. Otherwise names that
|
||||
contain `want` (case-insensitive), from events only: a substring match
|
||||
over every retained log line's name would scan the log table, and a
|
||||
container Steward watched die has a die or stop event.
|
||||
"""
|
||||
seen: dict[tuple[str, str], datetime] = {}
|
||||
sources = ((DockerLog, DockerLog.ts), (DockerEvent, DockerEvent.at)) if exact \
|
||||
else ((DockerEvent, DockerEvent.at),)
|
||||
for model, at in sources:
|
||||
match = (model.container_name == want if exact
|
||||
else model.container_name.ilike(f"%{escape_like(want)}%", escape="\\"))
|
||||
stmt = (select(model.host_id, model.container_name, func.max(at))
|
||||
.where(match).group_by(model.host_id, model.container_name))
|
||||
if host_id:
|
||||
stmt = stmt.where(model.host_id == host_id)
|
||||
for hid, name, last in (await db.execute(stmt)).all():
|
||||
key = (hid, name)
|
||||
if key not in seen or last > seen[key]:
|
||||
seen[key] = last
|
||||
rows = sorted(((h, n, t) for (h, n), t in seen.items()), key=lambda r: r[2], reverse=True)
|
||||
return rows[:limit]
|
||||
|
||||
|
||||
async def unhealthy_since(db, keys) -> dict[tuple[str, str], datetime]:
|
||||
"""{(host_id, container_name): when it last turned unhealthy} for the given
|
||||
containers, from their health_change events; absent when none is retained."""
|
||||
keys = set(keys)
|
||||
if not keys:
|
||||
return {}
|
||||
rows = (await db.execute(
|
||||
select(DockerEvent)
|
||||
.where(DockerEvent.event == "health_change",
|
||||
DockerEvent.detail.like("%→unhealthy"),
|
||||
DockerEvent.container_name.in_({name for _, name in keys}))
|
||||
.order_by(DockerEvent.at.desc())
|
||||
)).scalars()
|
||||
out: dict[tuple[str, str], datetime] = {}
|
||||
for e in rows:
|
||||
key = (e.host_id, e.container_name)
|
||||
if key in keys and key not in out:
|
||||
out[key] = e.at
|
||||
return out
|
||||
|
||||
|
||||
PRUNE_TARGETS = ("containers", "images", "system")
|
||||
|
||||
|
||||
def prune_extra_vars(prune_target: str) -> dict:
|
||||
"""Extra-vars for the bundled docker_prune playbook. "Prune unused images"
|
||||
means ALL unused (docker image prune -a), not just dangling; the full system
|
||||
prune stays conservative (-f, no -a) per the operator's choice (m78)."""
|
||||
extra_vars: dict = {"prune_target": prune_target}
|
||||
if prune_target == "images":
|
||||
extra_vars["prune_all_images"] = True
|
||||
return extra_vars
|
||||
|
||||
|
||||
# ── what is wrong with a container ───────────────────────────────────────────
|
||||
# Shared by the Docker page's "Needs attention" and the MCP tools, so both
|
||||
# rank the same containers the same way.
|
||||
|
||||
PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
|
||||
|
||||
def container_problems(c, failures_1h: int, failed_24h: bool,
|
||||
crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
|
||||
"""What is wrong with one container, most severe first; [] when nothing is."""
|
||||
status = (c.status or "").lower()
|
||||
out = []
|
||||
if status == "restarting":
|
||||
out.append("restarting")
|
||||
if failures_1h >= crash_loop_deaths:
|
||||
out.append("crash_looping")
|
||||
if status == "running" and (c.health or "") == "unhealthy":
|
||||
out.append("unhealthy")
|
||||
if c.oom_killed:
|
||||
out.append("oom_killed")
|
||||
if status in ("exited", "dead") and exit_is_failure(c.exit_code) and failed_24h:
|
||||
out.append("exited_nonzero")
|
||||
return out
|
||||
|
||||
|
||||
def problem_since(problems: list[str], deaths: list, hour_ago: datetime,
|
||||
unhealthy_at: datetime | None = None) -> datetime | None:
|
||||
"""When a container's worst problem began; None when the retained events
|
||||
cannot say. `deaths` are its die/oom events of the last 24h, newest first.
|
||||
|
||||
A restart or crash loop dates from its first death in the last hour, an
|
||||
OOM kill or error exit from the death that ended it, and a failing
|
||||
healthcheck from when its health last turned unhealthy.
|
||||
"""
|
||||
if not problems:
|
||||
return None
|
||||
worst = problems[0]
|
||||
if worst in ("restarting", "crash_looping"):
|
||||
in_hour = [e.at for e in deaths if e.at >= hour_ago]
|
||||
return min(in_hour) if in_hour else (deaths[0].at if deaths else None)
|
||||
if worst == "unhealthy":
|
||||
return unhealthy_at
|
||||
if worst == "oom_killed":
|
||||
ooms = [e.at for e in deaths if e.event == "oom"]
|
||||
return ooms[0] if ooms else (deaths[0].at if deaths else None)
|
||||
return deaths[0].at if deaths else None
|
||||
|
||||
|
||||
PROBLEM_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
|
||||
"oom_killed": "warning", "exited_nonzero": "warning"}
|
||||
PROBLEM_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
|
||||
"unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
|
||||
"exited_nonzero": "exited with an error"}
|
||||
|
||||
|
||||
|
||||
async def ranked_containers(db, now: datetime, crash_loop_deaths: int) -> list[dict]:
|
||||
"""Every container (deduplicated) with what is wrong with it, most severe
|
||||
first: {container, problems, since, replaced}. A swarm task Swarm has
|
||||
replaced has no problems; its failure belongs to its service."""
|
||||
containers = dedup_by_container_id(await list_containers(db))
|
||||
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
||||
replaced = replaced_tasks(containers, services)
|
||||
failures = await recent_failures_by_container(db, now - timedelta(hours=24))
|
||||
hour_ago = now - timedelta(hours=1)
|
||||
sick = await unhealthy_since(
|
||||
db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"])
|
||||
out = []
|
||||
for c in containers:
|
||||
key = (c.host_id, c.name)
|
||||
gone = key in replaced
|
||||
events = failures.get(key, [])
|
||||
recent = sum(1 for e in events if e.at >= hour_ago)
|
||||
problems = [] if gone else container_problems(c, recent, bool(events), crash_loop_deaths)
|
||||
out.append({"container": c, "problems": problems, "replaced": gone,
|
||||
"since": problem_since(problems, events, hour_ago, sick.get(key))})
|
||||
return out
|
||||
|
||||
|
||||
def needs_attention(ranked: list[dict]) -> list[dict]:
|
||||
"""The containers with something wrong, worst problem first, then by host
|
||||
and name."""
|
||||
return sorted((r for r in ranked if r["problems"]),
|
||||
key=lambda r: (PROBLEMS.index(r["problems"][0]), r["container"].host_id, r["container"].name))
|
||||
|
||||
|
||||
def stale_after_seconds(config) -> int:
|
||||
"""How old a container's last report may be before the page calls it
|
||||
stale: the host agent's own staleness window, since the agent reports it."""
|
||||
return int(((config.get("PLUGINS") or {}).get("host_agent") or {}).get("stale_after_seconds", 180))
|
||||
@@ -0,0 +1,382 @@
|
||||
"""Model-free builder for the swarm-aware container view.
|
||||
|
||||
Merges two sources into one service-grouped, cluster-complete picture:
|
||||
• docker_containers — real container rows, collected LOCAL-per-node, so each
|
||||
carries the host (= node) it runs on plus its swarm service/node labels.
|
||||
• DockerSwarmService.placement_json — the managers' cluster-wide task→node
|
||||
counts, which include tasks on nodes that run no Steward agent (and so have
|
||||
no container row of their own).
|
||||
|
||||
For every swarm service we list the real replicas we have detail for, then add
|
||||
"ghost" replicas for the remaining placement count on each node (agent-less
|
||||
nodes). Non-swarm containers pass through grouped by host, unchanged.
|
||||
|
||||
A third source, DockerSwarmService.tasks_json (agent ≥ 1.8.0), is Swarm's own
|
||||
record of each service's newest tasks. It decides which task containers are
|
||||
*superseded* — replaced by Swarm, so neither a replica nor a crash — and gives
|
||||
each service its failures across task retries and the error of its last
|
||||
failed task. Swarm restarts a task by creating a new one (a new container), so
|
||||
failures can only be counted per service, never per container.
|
||||
|
||||
Kept model-free (no ORM imports) so it's unit-testable and safe to import in
|
||||
tests without the plugin-loader "table already defined" gotcha.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
# Task states that mean the task ended in error: `failed` (its container exited
|
||||
# non-zero or never started) and `rejected` (Swarm could not start it at all,
|
||||
# e.g. no such image, a bad mount).
|
||||
FAILED_TASK_STATES = ("failed", "rejected")
|
||||
# Desired states of a task Swarm still means to run. Anything else (shutdown,
|
||||
# remove, complete, orphaned) is a task Swarm has finished with.
|
||||
_CURRENT_DESIRED = ("running", "ready", "")
|
||||
|
||||
|
||||
def _service_state(running: int, desired: int) -> str:
|
||||
if desired > 0 and running >= desired:
|
||||
return "healthy"
|
||||
if running > 0:
|
||||
return "degraded"
|
||||
return "down"
|
||||
|
||||
|
||||
def _placement(service) -> list[dict]:
|
||||
raw = getattr(service, "placement_json", None) or "[]"
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
return []
|
||||
return [p for p in data if isinstance(p, dict)]
|
||||
|
||||
|
||||
def parse_tasks(service) -> list[dict]:
|
||||
"""A service row's task records, newest first; [] when its agent sent none."""
|
||||
raw = getattr(service, "tasks_json", None) or "[]"
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
return []
|
||||
return [t for t in data if isinstance(t, dict)]
|
||||
|
||||
|
||||
def task_time(task: dict) -> datetime | None:
|
||||
"""A task's status timestamp. Docker writes nanoseconds; Python ≥ 3.11 keeps
|
||||
the first six digits."""
|
||||
raw = task.get("at") or ""
|
||||
try:
|
||||
at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return at if at.tzinfo else at.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def task_failures(tasks: list[dict], since: datetime) -> int:
|
||||
"""How many tasks ended in error at or after `since`."""
|
||||
n = 0
|
||||
for t in tasks:
|
||||
at = task_time(t)
|
||||
if t.get("state") in FAILED_TASK_STATES and at is not None and at >= since:
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def last_task_error(tasks: list[dict]) -> dict | None:
|
||||
"""Why the service's newest troubled task is in trouble, from Swarm itself.
|
||||
|
||||
The newest task that failed or was rejected, or that Swarm attached an error
|
||||
to while still trying (a pending task with "no suitable node", say). None
|
||||
when no retained task has a problem.
|
||||
"""
|
||||
for t in tasks: # newest first
|
||||
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
|
||||
return {"state": t.get("state"), "error": t.get("error") or None,
|
||||
"message": t.get("message") or None, "exit_code": t.get("exit_code"),
|
||||
"node_id": t.get("node_id") or None, "at": t.get("at") or None}
|
||||
return None
|
||||
|
||||
|
||||
def task_summary(tasks: list[dict], now: datetime, crash_loop_deaths: int,
|
||||
node_hostname: dict) -> dict:
|
||||
"""What a service's task records say: tasks_known, failures_1h,
|
||||
failures_24h (None without records), crash_looping, and last_error with
|
||||
the node's hostname as `node`."""
|
||||
known = bool(tasks)
|
||||
failures_1h = task_failures(tasks, now - timedelta(hours=1)) if known else None
|
||||
last_error = last_task_error(tasks)
|
||||
if last_error is not None:
|
||||
nid = last_error["node_id"]
|
||||
last_error["node"] = node_hostname.get(nid) or (nid[:12] if nid else None)
|
||||
at = task_time(last_error)
|
||||
last_error["age_s"] = max(0, int((now - at).total_seconds())) if at else None
|
||||
return {
|
||||
"tasks_known": known,
|
||||
"failures_1h": failures_1h,
|
||||
"failures_24h": task_failures(tasks, now - timedelta(hours=24)) if known else None,
|
||||
"crash_looping": bool(known and failures_1h >= crash_loop_deaths),
|
||||
"last_error": last_error,
|
||||
}
|
||||
|
||||
|
||||
def _oldest_failure(tasks: list[dict], since: datetime) -> datetime | None:
|
||||
times = [at for t in tasks if t.get("state") in FAILED_TASK_STATES
|
||||
and (at := task_time(t)) is not None and at >= since]
|
||||
return min(times, default=None)
|
||||
|
||||
|
||||
def trouble_since(tasks: list[dict], state: str, crash_looping: bool,
|
||||
now: datetime) -> datetime | None:
|
||||
"""When a service's current trouble began, as far as its retained tasks
|
||||
tell; None when it has none, or the tasks cannot say.
|
||||
|
||||
A crash loop dates from its first failure in the last hour, and a healthy
|
||||
service that lost tasks from its first failure in the last 24h. A service
|
||||
short of replicas dates from its troubled slots (replicated: the slot;
|
||||
global: the node): in each slot whose newest task is not running, the
|
||||
oldest of its newest troubled tasks (failed, rejected, or carrying an
|
||||
error), walking back until a running task interrupts them. Swarm keeps few
|
||||
old tasks, so a long outage reads as the oldest one retained: a lower
|
||||
bound.
|
||||
"""
|
||||
if crash_looping:
|
||||
return _oldest_failure(tasks, now - timedelta(hours=1))
|
||||
if state == "healthy":
|
||||
return _oldest_failure(tasks, now - timedelta(hours=24))
|
||||
by_slot: dict = {}
|
||||
for t in tasks: # newest first, so each slot's list is too
|
||||
by_slot.setdefault(t.get("slot") or t.get("node_id"), []).append(t)
|
||||
began = []
|
||||
for slot_tasks in by_slot.values():
|
||||
oldest = None
|
||||
for t in slot_tasks:
|
||||
if t.get("state") == "running":
|
||||
break
|
||||
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
|
||||
oldest = task_time(t) or oldest
|
||||
if oldest is not None:
|
||||
began.append(oldest)
|
||||
return min(began, default=None)
|
||||
|
||||
|
||||
def _slot_key(c) -> str:
|
||||
"""A task container's slot within its service: "<service>.<slot>.<task>" for
|
||||
replicated services, "<service>.<node id>.<task>" for global ones."""
|
||||
svc, name = c.service_name, c.name or ""
|
||||
rest = name[len(svc) + 1:] if name.startswith(svc + ".") else name
|
||||
return rest.split(".", 1)[0]
|
||||
|
||||
|
||||
def superseded_containers(swarm_cs, tasks_by_id: dict[str, dict]) -> set[tuple]:
|
||||
"""(host_id, name) of task containers Swarm has replaced.
|
||||
|
||||
With the task on record, its desired state decides: Swarm sets a replaced
|
||||
or failed task's desired state to shutdown. Without it (an older agent, or a
|
||||
task older than the retained history), a container that is not running is
|
||||
superseded when another task in the same slot is running or started later.
|
||||
"""
|
||||
out: set[tuple] = set()
|
||||
by_slot: dict[tuple, list] = {}
|
||||
for c in swarm_cs:
|
||||
by_slot.setdefault((c.service_name, _slot_key(c)), []).append(c)
|
||||
for c in swarm_cs:
|
||||
task = tasks_by_id.get(getattr(c, "task_id", None) or "")
|
||||
if task is not None:
|
||||
if task.get("desired_state") not in _CURRENT_DESIRED:
|
||||
out.add((c.host_id, c.name))
|
||||
continue
|
||||
if (getattr(c, "status", "") or "").lower() == "running":
|
||||
continue
|
||||
mine = getattr(c, "started_at", None)
|
||||
for other in by_slot[(c.service_name, _slot_key(c))]:
|
||||
if other is c:
|
||||
continue
|
||||
theirs = getattr(other, "started_at", None)
|
||||
if ((getattr(other, "status", "") or "").lower() == "running"
|
||||
or (theirs is not None and (mine is None or theirs > mine))):
|
||||
out.add((c.host_id, c.name))
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _newest_per_name(services) -> dict[str, object]:
|
||||
"""Every manager reports the same cluster-global service set; keep the most
|
||||
recently updated row per name, so a manager on an older agent (no tasks)
|
||||
doesn't shadow one that reports them."""
|
||||
by_name: dict[str, object] = {}
|
||||
for s in services:
|
||||
name = getattr(s, "service_name", None)
|
||||
if not name:
|
||||
continue
|
||||
kept = by_name.get(name)
|
||||
if kept is None:
|
||||
by_name[name] = s
|
||||
continue
|
||||
a, b = getattr(s, "updated_at", None), getattr(kept, "updated_at", None)
|
||||
if a is not None and (b is None or a > b):
|
||||
by_name[name] = s
|
||||
return by_name
|
||||
|
||||
|
||||
def replaced_tasks(containers, services) -> set[tuple]:
|
||||
"""(host_id, name) of the task containers Swarm has replaced, from the
|
||||
container rows and the managers' service rows. What the container views use
|
||||
to leave history out of their problem lists."""
|
||||
swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
|
||||
tasks_by_id = {t["task_id"]: t for s in _newest_per_name(services).values()
|
||||
for t in parse_tasks(s) if t.get("task_id")}
|
||||
return superseded_containers(swarm_cs, tasks_by_id)
|
||||
|
||||
|
||||
def build_swarm_services(containers, services, node_hostname: dict, host_name: dict,
|
||||
*, now: datetime | None = None, crash_loop_deaths: int = 3) -> dict:
|
||||
"""Return {"services": [...], "standalone": [...], "superseded": {(host_id, name)}}.
|
||||
|
||||
services[i] = {name, mode, desired, running, state, replicas:[...],
|
||||
previous_tasks, tasks_known, failures_1h, failures_24h,
|
||||
crash_looping, last_error}
|
||||
real replica = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
|
||||
health, restart_count} # current tasks only
|
||||
ghost replica = {ghost:True, host, count} # placement with no local row
|
||||
previous_tasks = task containers Swarm replaced, left out of replicas
|
||||
failures_* = tasks that failed or were rejected in the window (None
|
||||
without task data); crash_looping = failures_1h ≥ the
|
||||
Settings → Thresholds crash-loop count
|
||||
last_error = last_task_error(), plus `node` (the node's hostname)
|
||||
trouble_since = trouble_since(): when the current trouble began, or None
|
||||
standalone[i] = {host_id, host, containers:[...]} # non-swarm, by host
|
||||
superseded = the task containers Swarm replaced, so callers can skip them
|
||||
"""
|
||||
now = now or datetime.now(timezone.utc)
|
||||
containers = list(containers)
|
||||
swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
|
||||
standalone_cs = [c for c in containers if not getattr(c, "service_name", None)]
|
||||
|
||||
svc_by_name = _newest_per_name(services)
|
||||
tasks_by_svc = {name: parse_tasks(s) for name, s in svc_by_name.items()}
|
||||
tasks_by_id = {t["task_id"]: t for ts in tasks_by_svc.values() for t in ts
|
||||
if t.get("task_id")}
|
||||
superseded = superseded_containers(swarm_cs, tasks_by_id)
|
||||
|
||||
# Current task containers grouped by service name; replaced ones are counted.
|
||||
cs_by_service: dict[str, list] = {}
|
||||
previous: dict[str, int] = {}
|
||||
for c in swarm_cs:
|
||||
if (c.host_id, c.name) in superseded:
|
||||
previous[c.service_name] = previous.get(c.service_name, 0) + 1
|
||||
continue
|
||||
cs_by_service.setdefault(c.service_name, []).append(c)
|
||||
|
||||
def _host_of(c):
|
||||
return (host_name.get(getattr(c, "host_id", None))
|
||||
or node_hostname.get(getattr(c, "node_id", None))
|
||||
or getattr(c, "host_id", None) or "?")
|
||||
|
||||
out_services = []
|
||||
for name in sorted(set(svc_by_name) | set(cs_by_service)):
|
||||
svc = svc_by_name.get(name)
|
||||
reals = cs_by_service.get(name, [])
|
||||
|
||||
replicas = []
|
||||
for c in sorted(reals, key=lambda c: (_host_of(c).lower(), c.name)):
|
||||
replicas.append({
|
||||
"ghost": False,
|
||||
"host": _host_of(c),
|
||||
"host_id": getattr(c, "host_id", None),
|
||||
"name": c.name,
|
||||
"status": getattr(c, "status", "unknown"),
|
||||
"cpu_pct": getattr(c, "cpu_pct", None),
|
||||
"mem_pct": getattr(c, "mem_pct", None),
|
||||
"health": getattr(c, "health", None),
|
||||
"restart_count": getattr(c, "restart_count", 0) or 0,
|
||||
})
|
||||
|
||||
# Real running replicas per node, to subtract from placement.
|
||||
real_running_by_node: dict[str, int] = {}
|
||||
for c in reals:
|
||||
if (getattr(c, "status", "") or "").lower() == "running":
|
||||
nid = getattr(c, "node_id", None)
|
||||
if nid:
|
||||
real_running_by_node[nid] = real_running_by_node.get(nid, 0) + 1
|
||||
|
||||
ghosts = []
|
||||
for p in _placement(svc) if svc is not None else []:
|
||||
nid = p.get("node_id", "")
|
||||
placed = int(p.get("running", 0) or 0)
|
||||
ghost = placed - real_running_by_node.get(nid, 0)
|
||||
if ghost > 0:
|
||||
ghosts.append({
|
||||
"ghost": True,
|
||||
"host": node_hostname.get(nid) or (nid[:12] if nid else "?"),
|
||||
"count": ghost,
|
||||
})
|
||||
ghosts.sort(key=lambda g: g["host"].lower())
|
||||
replicas.extend(ghosts)
|
||||
|
||||
running = getattr(svc, "running", None) if svc is not None else None
|
||||
desired = getattr(svc, "desired", None) if svc is not None else None
|
||||
# No service row (manager didn't report it): infer from what we can see.
|
||||
if running is None:
|
||||
running = sum(1 for r in replicas
|
||||
if (not r["ghost"] and (r["status"] or "").lower() == "running"))
|
||||
if desired is None:
|
||||
desired = len(replicas)
|
||||
|
||||
tasks = tasks_by_svc.get(name, [])
|
||||
state = _service_state(int(running or 0), int(desired or 0))
|
||||
summary = task_summary(tasks, now, crash_loop_deaths, node_hostname)
|
||||
out_services.append({
|
||||
"name": name,
|
||||
"mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
|
||||
"running": running,
|
||||
"desired": desired,
|
||||
"state": state,
|
||||
"replicas": replicas,
|
||||
"previous_tasks": previous.get(name, 0),
|
||||
**summary,
|
||||
"trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
|
||||
})
|
||||
|
||||
# Non-swarm containers grouped by host, running-first order preserved.
|
||||
groups: dict[str, dict] = {}
|
||||
for c in standalone_cs:
|
||||
hid = getattr(c, "host_id", None)
|
||||
g = groups.get(hid)
|
||||
if g is None:
|
||||
g = groups[hid] = {"host_id": hid, "host": host_name.get(hid) or hid or "?",
|
||||
"containers": []}
|
||||
g["containers"].append(c)
|
||||
standalone = sorted(groups.values(), key=lambda g: g["host"].lower())
|
||||
|
||||
return {"services": out_services, "standalone": standalone, "superseded": superseded}
|
||||
|
||||
|
||||
def service_trouble(s: dict) -> int | None:
|
||||
"""How bad a built service is, for sorting and for Needs attention: 0 a
|
||||
crash loop, 1 no replica running, 2 short of replicas, 3 healthy but with
|
||||
failed tasks in the last 24h; None when nothing is wrong."""
|
||||
if s.get("crash_looping"):
|
||||
return 0
|
||||
if s.get("state") == "down":
|
||||
return 1
|
||||
if s.get("state") == "degraded":
|
||||
return 2
|
||||
if s.get("failures_24h"):
|
||||
return 3
|
||||
return None
|
||||
|
||||
|
||||
def services_problems_first(services: list[dict]) -> list[dict]:
|
||||
"""Services in trouble first, worst first, then the rest by name."""
|
||||
return sorted(services, key=lambda s: (
|
||||
4 if service_trouble(s) is None else service_trouble(s), s["name"].lower()))
|
||||
|
||||
|
||||
def failing_services(services: list[dict]) -> list[dict]:
|
||||
"""The services that belong in Needs attention: a crash loop, or short of
|
||||
their desired replicas. A healthy service whose failed tasks Swarm has
|
||||
already replaced is history, shown on the service, not a problem now."""
|
||||
# A crash loop ranks 0, so test the rank against None, not for truth.
|
||||
return [s for s in services_problems_first(services)
|
||||
if (t := service_trouble(s)) is not None and t <= 2]
|
||||
@@ -0,0 +1,17 @@
|
||||
{# docker/_container_history.html — CPU/mem sparklines for the selected range #}
|
||||
{% if have_data %}
|
||||
<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:1.5rem;">
|
||||
<div>
|
||||
<div class="section-title" style="margin-bottom:0.4rem;">CPU %</div>
|
||||
{{ sparkline_cpu | safe }}
|
||||
</div>
|
||||
<div>
|
||||
<div class="section-title" style="margin-bottom:0.4rem;">Memory %</div>
|
||||
{{ sparkline_mem | safe }}
|
||||
</div>
|
||||
</div>
|
||||
{% else %}
|
||||
<div style="color:var(--text-muted);font-size:0.85rem;">
|
||||
Not enough samples in this range yet — history fills in as the agent reports.
|
||||
</div>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,22 @@
|
||||
{# docker/_container_logs_lines.html — log-line fragment, HTMX-polled (m79).
|
||||
{{ l.line }} is auto-escaped by Jinja, so log content can't inject markup. #}
|
||||
{% if lines %}
|
||||
{% for l in lines %}
|
||||
<div style="display:flex;gap:0.6rem;white-space:pre-wrap;word-break:break-word;">
|
||||
<span style="color:var(--text-dim);flex-shrink:0;" title="{{ l.ts }}">{{ l.ts.strftime("%m-%d %H:%M:%S") }}</span>
|
||||
<span style="flex-shrink:0;width:3.2rem;color:{% if l.stream == 'stderr' %}var(--red){% else %}var(--text-muted){% endif %};">{{ l.stream }}</span>
|
||||
<span style="flex:1;min-width:0;{% if l.stream == 'stderr' %}color:var(--text);{% endif %}">{{ l.line }}</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
<div style="color:var(--text-muted);padding:1rem 0.25rem;">
|
||||
{% if filtered %}
|
||||
No lines match the current filter.
|
||||
{% else %}
|
||||
No logs collected yet for <code>{{ name }}</code>. Lines appear within a few
|
||||
seconds of the container writing to stdout/stderr — unless it's on the log
|
||||
exclude list or log collection is turned off in
|
||||
<a href="/settings/thresholds/">Settings → Thresholds & Retention</a>.
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,25 @@
|
||||
{#- One line on a swarm service's task failures, from Swarm's own task records
|
||||
(agent ≥ 1.8.0): a crash loop or the failures of the last 24h, and how the
|
||||
newest troubled task ended, in Swarm's words. Expects `s` with state,
|
||||
crash_looping, failures_1h, failures_24h and last_error; `style` is
|
||||
optional extra CSS for the wrapper. Renders nothing for a healthy service
|
||||
with no recent failures. -#}
|
||||
{%- set e = s.last_error -%}
|
||||
{%- if s.crash_looping or s.failures_24h or (e and s.state != 'healthy') %}
|
||||
<div style="display:flex;flex-wrap:wrap;align-items:baseline;gap:0.2rem 0.6rem;font-size:0.78rem;color:var(--text-muted);{{ style or '' }}">
|
||||
{% if s.crash_looping %}
|
||||
<span style="color:var(--red);font-weight:500;">Crash-looping · {{ s.failures_1h }} tasks failed in the last hour</span>
|
||||
{% elif s.failures_24h %}
|
||||
<span style="color:var(--orange);">{{ s.failures_24h }} task{{ 's' if s.failures_24h != 1 }} failed in the last 24h</span>
|
||||
{% endif %}
|
||||
{% if e %}
|
||||
<span title="How Swarm recorded the end of this service's newest failed task">
|
||||
Last task {{ e.state }}:
|
||||
<span style="color:var(--text);font-family:ui-monospace,monospace;overflow-wrap:anywhere;">{{ e.error or e.message or e.state }}</span>
|
||||
{%- if e.exit_code %} · exit {{ e.exit_code }}{% endif %}
|
||||
{%- if e.node %} · {{ e.node }}{% endif %}
|
||||
{%- if e.age_s is not none %} · {{ e.age_s|duration }} ago{% endif %}
|
||||
</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{%- endif %}
|
||||
@@ -0,0 +1,26 @@
|
||||
{#- A container's CPU and memory as service.usage() describes them (#5209).
|
||||
|
||||
Docker counts CPU per core (100% = one core busy), so CPU reads as cores,
|
||||
toned by its share of the host's CPUs when the host's core count is known.
|
||||
Memory reads in bytes, against a limit only when one is actually set;
|
||||
mem_pct is then a share of that limit, otherwise of the host's memory. -#}
|
||||
|
||||
{% macro cores(n) -%}
|
||||
{{ "%.2f"|format(n) if n < 1 else "%.1f"|format(n) }} {{ "core" if n == 1 else "cores" }}
|
||||
{%- endmacro %}
|
||||
|
||||
{% macro cpu(u) -%}
|
||||
{%- if u.cores is none -%}<span class="tone-dim">—</span>
|
||||
{%- else -%}
|
||||
<span class="tone-{{ status_class(u.cpu_share, 'cpu') }}"
|
||||
title="Docker counts CPU per core: {{ '%.0f'|format(u.cores * 100) }}% of one core{% if u.cpus %}, {{ '%.1f'|format(u.cpu_share) }}% of this host's {{ u.cpus }} core{{ 's' if u.cpus != 1 }}{% endif %}">{{ cores(u.cores) }}</span>
|
||||
{%- endif -%}
|
||||
{%- endmacro %}
|
||||
|
||||
{% macro mem(u) -%}
|
||||
{%- if u.mem_bytes is none -%}<span class="tone-dim">—</span>
|
||||
{%- else -%}
|
||||
<span class="tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of else 'dim' }}"
|
||||
title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% else %}No memory limit set{% endif %}">{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %}</span>
|
||||
{%- endif -%}
|
||||
{%- endmacro %}
|
||||
@@ -0,0 +1,130 @@
|
||||
{# docker/container_detail.html — full detail page for one container. A swarm
|
||||
task is titled by its service and slot, with the raw container name under
|
||||
it; CPU reads in cores and memory in bytes (see _usage.html). #}
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import crumbs %}
|
||||
{% from "docker/_usage.html" import cpu, mem %}
|
||||
{% block title %}{{ label or name }} — Docker — Steward{% endblock %}
|
||||
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (label or name, "")]) }}{% endblock %}
|
||||
{% block content %}
|
||||
|
||||
{% if container is none %}
|
||||
<div class="card" style="text-align:center;padding:3rem;">
|
||||
<div style="font-weight:500;margin-bottom:0.4rem;">Container not found</div>
|
||||
<div style="color:var(--text-muted);font-size:0.9rem;">
|
||||
No container named <code>{{ name }}</code> is currently reported for this host.
|
||||
It may have been removed, or the host agent hasn't reported recently.
|
||||
</div>
|
||||
</div>
|
||||
{% else %}
|
||||
|
||||
{# ── Header ──────────────────────────────────────────────────────────────── #}
|
||||
<div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
|
||||
<span class="dot {% if container.status == 'running' %}dot-up{% elif container.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
|
||||
<h1 class="page-title" style="margin-bottom:0;">{{ label }}</h1>
|
||||
{% if container.health == 'healthy' %}<span style="font-size:0.72rem;color:var(--green);border:1px solid var(--green);border-radius:3px;padding:0.05rem 0.4rem;">healthy</span>
|
||||
{% elif container.health == 'unhealthy' %}<span style="font-size:0.72rem;color:var(--red);border:1px solid var(--red);border-radius:3px;padding:0.05rem 0.4rem;">unhealthy</span>
|
||||
{% elif container.health == 'starting' %}<span style="font-size:0.72rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0.05rem 0.4rem;">starting</span>{% endif %}
|
||||
</div>
|
||||
{% if label != container.name %}
|
||||
<div style="font-size:0.75rem;color:var(--text-dim);font-family:ui-monospace,monospace;margin-bottom:0.25rem;overflow-wrap:anywhere;">{{ container.name }}</div>
|
||||
{% endif %}
|
||||
<div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
|
||||
{{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %}
|
||||
{% if host %} · on <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %}
|
||||
· <a href="/plugins/docker/container/{{ host_id }}/{{ name }}/logs">View logs</a>
|
||||
</div>
|
||||
|
||||
{# ── Facts grid ──────────────────────────────────────────────────────────── #}
|
||||
{% macro fact(label, value, colour="") %}
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.3rem;">{{ label }}</div>
|
||||
<div style="font-size:0.95rem;{% if colour %}color:{{ colour }};{% endif %}font-family:ui-monospace,monospace;">{{ value }}</div>
|
||||
</div>
|
||||
{% endmacro %}
|
||||
|
||||
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
|
||||
{{ fact("CPU", cpu(usage) if container.status == 'running' else "—") }}
|
||||
{{ fact("Memory", mem(usage) if container.status == 'running' else "—") }}
|
||||
{#- Swarm never restarts a task's container: it replaces the task with a new
|
||||
one, so restart_count is always 0 there and the service's task failures
|
||||
(Swarm page) are the count that means something. #}
|
||||
{% if not container.service_name %}
|
||||
{{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }}
|
||||
{% endif %}
|
||||
{{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—",
|
||||
"var(--red)" if exit_failed else "") }}
|
||||
{{ fact("Net in since start", net_rx) }}
|
||||
{{ fact("Net out since start", net_tx) }}
|
||||
{{ fact("Disk read since start", blk_read) }}
|
||||
{{ fact("Disk written since start", blk_write) }}
|
||||
</div>
|
||||
{% if container.service_name %}
|
||||
<div style="font-size:0.78rem;color:var(--text-muted);margin:-0.75rem 0 1.5rem;">
|
||||
Swarm replaces a failed task with a new container rather than restarting it; the
|
||||
service's failed tasks are on the <a href="/plugins/docker/swarm">Swarm page</a>.
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{# ── Image / placement ───────────────────────────────────────────────────── #}
|
||||
<div class="card" style="margin-bottom:1.5rem;">
|
||||
<div style="display:grid;grid-template-columns:max-content 1fr;gap:0.4rem 1.25rem;font-size:0.86rem;">
|
||||
<span style="color:var(--text-muted);">Image</span>
|
||||
<span style="font-family:ui-monospace,monospace;word-break:break-all;">{{ container.image or "—" }}</span>
|
||||
{% if container.compose_project %}
|
||||
<span style="color:var(--text-muted);">Compose project</span><span>{{ container.compose_project }}</span>
|
||||
{% endif %}
|
||||
{% if container.service_name %}
|
||||
<span style="color:var(--text-muted);">Swarm service</span><span>{{ container.service_name }}</span>
|
||||
{% endif %}
|
||||
{% if container.node_id %}
|
||||
<span style="color:var(--text-muted);">Node</span>
|
||||
{% if node_name %}<span title="node id {{ container.node_id }}">{{ node_name }}</span>
|
||||
{% else %}<span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>{% endif %}
|
||||
{% endif %}
|
||||
{% if ports %}
|
||||
<span style="color:var(--text-muted);">Ports</span>
|
||||
<span style="font-family:ui-monospace,monospace;">{% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %}</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{# ── Resource history (range-toggled) ────────────────────────────────────── #}
|
||||
<div class="card" style="margin-bottom:1.5rem;">
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
|
||||
<h3 class="section-title" style="margin-bottom:0;">Resource history</h3>
|
||||
{% include "_time_range.html" %}
|
||||
</div>
|
||||
<div id="container-history"
|
||||
hx-get="/plugins/docker/container/{{ host_id }}/{{ name }}/history"
|
||||
hx-trigger="load, rangeChange from:body"
|
||||
hx-include="#time-range"
|
||||
hx-swap="innerHTML">
|
||||
<div style="color:var(--text-muted);font-size:0.85rem;">Loading…</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{# ── Lifecycle timeline ──────────────────────────────────────────────────── #}
|
||||
<div class="card">
|
||||
<h3 class="section-title" style="margin-bottom:0.75rem;">Lifecycle</h3>
|
||||
{% if events %}
|
||||
<div style="display:grid;gap:0.5rem;">
|
||||
{% for e in events %}
|
||||
<div style="display:flex;align-items:baseline;gap:0.6rem;font-size:0.85rem;">
|
||||
<span style="color:{{ e.colour }};width:1rem;text-align:center;flex-shrink:0;">{{ e.glyph }}</span>
|
||||
<span style="font-weight:500;width:6.5rem;flex-shrink:0;">{{ e.event }}</span>
|
||||
<span style="color:var(--text-muted);flex:1;min-width:0;">{{ e.detail or "" }}</span>
|
||||
<span style="color:var(--text-dim);font-size:0.78rem;white-space:nowrap;" title="{{ e.at }}">{{ e.at.strftime("%Y-%m-%d %H:%M") }}</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% else %}
|
||||
<div style="color:var(--text-muted);font-size:0.85rem;">
|
||||
No lifecycle events recorded yet. Start/stop/health changes appear here as the
|
||||
agent reports them over time.
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,51 @@
|
||||
{# docker/container_logs.html — per-container log viewer (m79) #}
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import crumbs %}
|
||||
{% block title %}Logs — {{ name }} — Docker — Steward{% endblock %}
|
||||
{% block breadcrumb %}{{ crumbs([
|
||||
("Docker", "/plugins/docker/"),
|
||||
(name, "/plugins/docker/container/" ~ host_id ~ "/" ~ name),
|
||||
("Logs", "")]) }}{% endblock %}
|
||||
{% block content %}
|
||||
|
||||
<div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
|
||||
<h1 class="page-title" style="margin-bottom:0;">{{ name }}</h1>
|
||||
<span style="font-size:0.9rem;color:var(--text-muted);">logs</span>
|
||||
</div>
|
||||
<div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1rem;">
|
||||
Recent lines (newest first) collected by the host agent{% if host %} on
|
||||
<a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %} —
|
||||
updated every few seconds.
|
||||
<a href="/plugins/docker/container/{{ host_id }}/{{ name }}">← back to container</a>
|
||||
</div>
|
||||
|
||||
{# ── Controls: stream filter + text search (drive the fragment via HTMX) ───── #}
|
||||
<form id="log-controls" onsubmit="return false;"
|
||||
style="display:flex;gap:0.6rem;align-items:center;flex-wrap:wrap;margin-bottom:0.6rem;">
|
||||
<label style="font-size:0.8rem;color:var(--text-muted);display:flex;align-items:center;gap:0.35rem;">
|
||||
Stream
|
||||
<select name="stream" style="padding:0.25rem 0.4rem;">
|
||||
<option value="all">all</option>
|
||||
<option value="stdout">stdout</option>
|
||||
<option value="stderr">stderr</option>
|
||||
</select>
|
||||
</label>
|
||||
<input type="search" name="q" placeholder="Filter lines…" autocomplete="off"
|
||||
aria-label="Filter log lines"
|
||||
style="flex:1;min-width:180px;padding:0.3rem 0.5rem;">
|
||||
</form>
|
||||
|
||||
<div class="card-flush">
|
||||
<div id="log-lines"
|
||||
hx-get="/plugins/docker/container/{{ host_id }}/{{ name }}/logs/lines"
|
||||
hx-trigger="load, every 5s, change from:#log-controls, keyup changed delay:400ms from:#log-controls"
|
||||
hx-include="#log-controls"
|
||||
hx-swap="innerHTML"
|
||||
role="log" aria-live="polite" tabindex="0"
|
||||
style="max-height:70vh;overflow:auto;padding:0.5rem 0.75rem;
|
||||
font-family:ui-monospace,monospace;font-size:0.8rem;line-height:1.5;">
|
||||
<div style="color:var(--text-muted);">Loading…</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,117 @@
|
||||
{# docker/disk.html — image/disk usage: reclaimable space, per-image sizes #}
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import crumbs %}
|
||||
{% block title %}Disk — Docker — Steward{% endblock %}
|
||||
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Image & disk usage", "")]) }}{% endblock %}
|
||||
{% block content %}
|
||||
|
||||
<h1 class="page-title" style="margin-bottom:0.4rem;">Image & disk usage</h1>
|
||||
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
|
||||
Reclaimable = space held by images no container references. Admins can prune
|
||||
per host below; each action runs an audited Ansible playbook on that host.
|
||||
</p>
|
||||
|
||||
{% if host_groups %}
|
||||
{% for g in host_groups %}
|
||||
<div style="margin-bottom:2rem;">
|
||||
<div style="font-weight:500;font-size:0.95rem;margin-bottom:0.75rem;">{{ g.host_name }}</div>
|
||||
|
||||
{# ── Summary stats ────────────────────────────────────────────────────── #}
|
||||
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1rem;">
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.3rem;">Reclaimable</div>
|
||||
<span class="stat-val" style="color:{% if g.summary.images_reclaimable != '0 B' %}var(--orange){% else %}var(--green){% endif %};">{{ g.summary.images_reclaimable }}</span>
|
||||
</div>
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.3rem;">Images</div>
|
||||
<span class="stat-val">{{ g.summary.images_size }}</span>
|
||||
<div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.images_active }}/{{ g.summary.images_total }} in use</div>
|
||||
</div>
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.3rem;">Stopped</div>
|
||||
<span class="stat-val" style="{% if g.stopped %}color:var(--text-muted){% endif %};">{{ g.stopped }}</span>
|
||||
</div>
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.3rem;">Writable layers</div>
|
||||
<span class="stat-val">{{ g.summary.containers_size }}</span>
|
||||
<div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.containers_count }} containers</div>
|
||||
</div>
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.3rem;">Volumes</div>
|
||||
<span class="stat-val">{{ g.summary.volumes_size }}</span>
|
||||
<div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.volumes_count }} volumes</div>
|
||||
</div>
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.3rem;">Build cache</div>
|
||||
<span class="stat-val">{{ g.summary.build_cache_size }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{# ── Cleanup actions (admin only; audited Ansible prune run) ───────────── #}
|
||||
{% if session.user_role == 'admin' %}
|
||||
<div style="margin-bottom:1rem;">
|
||||
{% if ansible_available and g.has_target %}
|
||||
<div style="display:flex;flex-wrap:wrap;gap:0.5rem;align-items:center;">
|
||||
<form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
|
||||
data-confirm="Remove all STOPPED containers on {{ g.host_name }}? This cannot be undone.">{{ csrf_field() }}
|
||||
<input type="hidden" name="target" value="containers">
|
||||
<button type="submit" class="btn btn-sm">Prune stopped containers</button>
|
||||
</form>
|
||||
<form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
|
||||
data-confirm="Remove ALL unused images on {{ g.host_name }} (docker image prune -a)? Any image not used by a container is deleted.">{{ csrf_field() }}
|
||||
<input type="hidden" name="target" value="images">
|
||||
<button type="submit" class="btn btn-sm">Prune unused images</button>
|
||||
</form>
|
||||
<form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
|
||||
data-confirm="Run a full system prune on {{ g.host_name }}? Removes stopped containers, unused networks, dangling images and build cache.">{{ csrf_field() }}
|
||||
<input type="hidden" name="target" value="system">
|
||||
<button type="submit" class="btn btn-sm btn-danger">System prune…</button>
|
||||
</form>
|
||||
</div>
|
||||
<div style="font-size:0.72rem;color:var(--text-muted);margin-top:0.4rem;">
|
||||
Reclaimed space appears on the next agent sample.
|
||||
</div>
|
||||
{% elif not ansible_available %}
|
||||
<div style="font-size:0.74rem;color:var(--text-muted);">Cleanup needs the Ansible runner (currently unavailable).</div>
|
||||
{% else %}
|
||||
<div style="font-size:0.74rem;color:var(--text-muted);">
|
||||
Link an <a href="/hosts/{{ g.host_id }}">Ansible target</a> to this host to enable prune actions.
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{# ── Per-image table ──────────────────────────────────────────────────── #}
|
||||
<div class="card-flush">
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr><th>Image</th><th>Size</th><th>Shared</th><th>Containers</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for im in g.images %}
|
||||
<tr>
|
||||
<td style="font-size:0.84rem;font-family:ui-monospace,monospace;max-width:340px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
|
||||
{{ im.repo_tag }}
|
||||
{% if im.reclaimable %}<span title="not referenced by any container" style="font-size:0.68rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0 0.3rem;margin-left:0.4rem;">reclaimable</span>{% endif %}
|
||||
</td>
|
||||
<td style="font-family:ui-monospace,monospace;font-size:0.86rem;">{{ im.size }}</td>
|
||||
<td style="font-family:ui-monospace,monospace;font-size:0.86rem;color:var(--text-muted);">{{ im.shared }}</td>
|
||||
<td style="font-size:0.86rem;color:{% if im.containers %}var(--text){% else %}var(--text-muted){% endif %};">{{ im.containers }}</td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No images reported.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
<div class="card" style="text-align:center;padding:3rem;">
|
||||
<div style="color:var(--text-muted);font-size:0.9rem;">
|
||||
No disk usage reported yet. The host agent reports image/disk usage from
|
||||
<code>docker system df</code> on hosts running Docker.
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,58 @@
|
||||
{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub:
|
||||
problems first with what is wrong and since when, then running, then
|
||||
stopped. Tasks Swarm has replaced are left out (counted in the header). #}
|
||||
{% from "_macros.html" import status_badge, status_dot %}
|
||||
{% from "docker/_usage.html" import cpu, mem %}
|
||||
<div class="card">
|
||||
<div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;flex-wrap:wrap;">
|
||||
<h3 class="section-title" style="margin-bottom:0;">Docker</h3>
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);">
|
||||
{% if problems %}<span class="tone-crit">{{ problems }} with problems</span> · {% endif %}
|
||||
{{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %}
|
||||
{% if replaced %}<span title="Old swarm tasks Swarm has replaced with new ones: history, not containers that need anything"> · {{ replaced }} replaced task{{ "s" if replaced != 1 }} hidden</span>{% endif %}
|
||||
<a href="/plugins/docker/" style="margin-left:0.6rem;color:var(--text-muted);">All →</a>
|
||||
</span>
|
||||
</div>
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Container</th>
|
||||
<th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
|
||||
<th title="Used, and the limit when one is set">Memory</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for r in rows %}
|
||||
{% set c = r.c %}
|
||||
<tr>
|
||||
<td>
|
||||
<div style="display:flex;align-items:center;gap:0.5rem;min-width:0;">
|
||||
{{ status_dot('crit' if r.problems else ('up' if c.status == 'running' else 'inactive'), c.status|capitalize) }}
|
||||
<div style="min-width:0;">
|
||||
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }} · {{ c.image }}" style="font-weight:500;color:inherit;text-decoration:none;overflow-wrap:anywhere;">{{ r.label }}</a>
|
||||
{% if r.problems %}
|
||||
{% set worst = r.problems[0] %}
|
||||
<div style="font-size:0.75rem;margin-top:0.1rem;">
|
||||
{{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
|
||||
<span style="color:var(--text-muted);">
|
||||
{%- for p in r.problems %} {{ says[p] }}{{ " and" if not loop.last }}{% endfor %}
|
||||
{%- if c.exit_code %} · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}{% endif %}
|
||||
{%- if r.since %} · since {{ r.since|ago }}{% endif -%}
|
||||
</span>
|
||||
</div>
|
||||
{% else %}
|
||||
<div style="font-size:0.72rem;color:var(--text-muted);">
|
||||
{{ c.status }}{% if r.uptime %} · up {{ r.uptime }}{% endif %}
|
||||
{%- if c.status != 'running' and c.exit_code %} · exit {{ c.exit_code }}{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
<td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
|
||||
<td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
@@ -0,0 +1,29 @@
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import page_header %}
|
||||
{% block title %}Docker — Steward{% endblock %}
|
||||
{% block content %}
|
||||
{% call page_header("Docker", "What needs attention first, then every service and container.") %}
|
||||
{% if has_swarm %}<a class="btn btn-sm btn-ghost" href="/plugins/docker/swarm">Swarm</a>{% endif %}
|
||||
{% if has_disk %}<a class="btn btn-sm btn-ghost" href="/plugins/docker/disk">Disk</a>{% endif %}
|
||||
{% endcall %}
|
||||
|
||||
<div id="docker-rows"
|
||||
hx-get="/plugins/docker/rows"
|
||||
hx-trigger="load, every {{ poll_interval }}s"
|
||||
hx-swap="innerHTML">
|
||||
<div class="card"><p class="field-hint">Loading…</p></div>
|
||||
</div>
|
||||
{# A link to a section (the dashboard's "failed (24h)" → #failures) lands
|
||||
before the rows exist; scroll to it once, when they first arrive. #}
|
||||
<script>
|
||||
(function () {
|
||||
var rows = document.getElementById('docker-rows');
|
||||
if (!location.hash || !rows) return;
|
||||
rows.addEventListener('htmx:afterSwap', function once() {
|
||||
rows.removeEventListener('htmx:afterSwap', once);
|
||||
var target = document.getElementById(location.hash.slice(1));
|
||||
if (target) target.scrollIntoView({ block: 'start' });
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,229 @@
|
||||
{# docker/rows.html — HTMX fragment for the Docker page: what needs attention
|
||||
first (service.needs_attention, the ranking the MCP tools use), the counts,
|
||||
then swarm services and each host's containers. #}
|
||||
{% from "_macros.html" import stat_tile, status_badge, status_dot %}
|
||||
{% from "docker/_usage.html" import cpu, mem, cores %}
|
||||
|
||||
{# ── Needs attention ──────────────────────────────────────────────────────── #}
|
||||
{% if attention or failing_services %}
|
||||
<section class="card problem-card" aria-labelledby="docker-problems">
|
||||
<h2 id="docker-problems" class="section-title" style="margin-bottom:0.6rem;">Needs attention</h2>
|
||||
<ul class="problem-list">
|
||||
{# A swarm service in trouble: Swarm replaces its failed tasks, so the
|
||||
service, not any one container, is what is failing. #}
|
||||
{% for s in failing_services %}
|
||||
{% set e = s.last_error %}
|
||||
<li>
|
||||
{% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
|
||||
{% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}
|
||||
{% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
|
||||
<a href="/plugins/docker/swarm">{{ s.name }}</a>
|
||||
<span class="problem-detail">
|
||||
{{- s.running }} of {{ s.desired }} running
|
||||
{%- if s.crash_looping %} · {{ s.failures_1h }} tasks failed in the last hour{% endif %}
|
||||
{%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.exit_code %} (exit {{ e.exit_code }}){% endif %}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
|
||||
{%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
|
||||
</span>
|
||||
</li>
|
||||
{% endfor %}
|
||||
{% for r in attention %}
|
||||
{% set c = r.container %}
|
||||
{% set worst = r.problems[0] %}
|
||||
<li>
|
||||
{{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
|
||||
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }}">{{ r.label }}</a>
|
||||
<span class="problem-detail">
|
||||
{%- for p in r.problems %}{{ says[p] }}{{ " and " if not loop.last }}{% endfor %}
|
||||
{%- if c.exit_code and worst in ('exited_nonzero', 'crash_looping', 'restarting') %} · exit {{ c.exit_code }}{% endif %}
|
||||
{%- if r.since %} · since {{ r.since|ago }}{% endif %}
|
||||
{%- if r.stale %} · {{ status_badge('stale') }} last report {{ c.scraped_at|ago if c.scraped_at else "never" }}{% endif -%}
|
||||
</span>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{# ── Hosts whose Docker is not answering (agent ≥ 1.9.0) ──────────────────── #}
|
||||
{% if docker_down %}
|
||||
<section class="card problem-card" aria-labelledby="docker-unreachable">
|
||||
<h2 id="docker-unreachable" class="section-title" style="margin-bottom:0.6rem;">Docker not answering</h2>
|
||||
<ul class="problem-list">
|
||||
{% for d in docker_down %}
|
||||
<li>
|
||||
{{ status_badge('crit', 'Error') if d.status == 'error' else status_badge('stale', 'No Docker') }}
|
||||
<a href="/hosts/{{ d.host_id }}">{{ d.host_name }}</a>
|
||||
<span class="problem-detail">{{ d.detail or "no reason given" }} · since {{ d.since|ago }} · its containers below are what Docker last reported</span>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{# ── Counts ──────────────────────────────────────────────────────────────
|
||||
Swarm services are counted as services (a task's container comes and goes);
|
||||
the container counts are the containers outside any swarm service. ────── #}
|
||||
<div class="stat-strip">
|
||||
{{ stat_tile("Needs attention", attention|length + failing_services|length, state="crit" if attention or failing_services else "inactive") }}
|
||||
{{ stat_tile("Unhealthy", unhealthy, state="crit" if unhealthy else "inactive") }}
|
||||
{{ stat_tile("Restarting", restarting, state="crit" if restarting else "inactive", hint="restarting or crash-looping") }}
|
||||
{{ stat_tile("Failed (24h)", failures|length, state="warn" if failures else "inactive", hint="containers and services; listed below", href="#failures" if failures else none) }}
|
||||
{% if swarm_services %}{{ stat_tile("Swarm services", services_healthy ~ "/" ~ swarm_services|length, state="up" if services_healthy == swarm_services|length else "warn", hint="healthy", href="/plugins/docker/swarm") }}{% endif %}
|
||||
{{ stat_tile("Containers running", running, state="up" if running else "inactive", hint="outside swarm services" if swarm_services else none) }}
|
||||
{{ stat_tile("Containers stopped", stopped, state="inactive") }}
|
||||
{% if stale %}{{ stat_tile("Stale", stale, state="stale", hint="no recent report") }}{% endif %}
|
||||
{{ stat_tile("Docker hosts", docker_hosts) }}
|
||||
</div>
|
||||
|
||||
{# ── What failed in the last 24h: what the dashboard's "failed (24h)" counts.
|
||||
A plain container's death, or a swarm service's failed tasks; a container
|
||||
someone stopped (exit 137/143) is not a failure. ───────────────────────── #}
|
||||
{% if failures %}
|
||||
<section id="failures" class="card" aria-labelledby="docker-failures" style="margin-bottom:1.5rem;">
|
||||
<h2 id="docker-failures" class="section-title" style="margin-bottom:0.6rem;">Failed in the last 24h</h2>
|
||||
<ul class="problem-list">
|
||||
{% for f in failures %}
|
||||
<li>
|
||||
{{ status_badge('warn', 'Service' if f.kind == 'service' else 'Container') }}
|
||||
{% if f.kind == 'service' %}<a href="/plugins/docker/swarm">{{ f.name }}</a>
|
||||
{% else %}<a href="/plugins/docker/container/{{ f.host_id }}/{{ f.name }}">{{ f.name }}</a>{% endif %}
|
||||
<span class="problem-detail">
|
||||
{{- f.count }} {{ "failure" if f.count == 1 else "failures" }}
|
||||
{%- if f.detail %} · last: {{ f.detail }}{% endif %}
|
||||
{%- if f.host %} · {{ f.host }}{% endif %}
|
||||
{%- if f.at %} · {{ f.at|ago }}{% endif -%}
|
||||
</span>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{# ── Swarm services: collapsed per-service, each current replica labelled with
|
||||
its host (tasks Swarm replaced are left out). Ghost replicas (tasks on nodes
|
||||
with no Steward agent) come from the managers' placement data so the picture
|
||||
is cluster-complete. A failing service names Swarm's last task error. ───── #}
|
||||
{% if swarm_services %}
|
||||
<div class="card-flush" style="margin-bottom:1.5rem;">
|
||||
<div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
|
||||
<span style="font-weight:500;font-size:0.95rem;">Swarm services</span>
|
||||
<a href="/plugins/docker/swarm" style="font-size:0.78rem;color:var(--text-muted);">Topology →</a>
|
||||
</div>
|
||||
<div style="padding:0.5rem 0.75rem;display:grid;gap:0.85rem;">
|
||||
{% for s in swarm_services %}
|
||||
<div>
|
||||
<div style="display:flex;align-items:center;gap:0.55rem;flex-wrap:wrap;margin-bottom:0.35rem;">
|
||||
<span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
|
||||
<span style="font-weight:500;font-size:0.9rem;">{{ s.name }}</span>
|
||||
<span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
|
||||
<span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-size:0.78rem;font-variant-numeric:tabular-nums;">
|
||||
{{ s.running }}/{{ s.desired }} running{% if s.state != 'healthy' and s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif %}
|
||||
</span>
|
||||
</div>
|
||||
{% with style="padding-left:1.1rem;margin-bottom:0.35rem;" %}{% include "docker/_swarm_failures.html" %}{% endwith %}
|
||||
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(210px,1fr));gap:0.4rem;padding-left:1.1rem;">
|
||||
{% for r in s.replicas %}
|
||||
{% if r.ghost %}
|
||||
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:5px;padding:0.3rem 0.55rem;" title="Running on a node with no Steward agent — detail unavailable">
|
||||
<span class="dot dot-warn" style="opacity:0.6;"></span>
|
||||
<span style="font-weight:500;">{{ r.host }}</span>
|
||||
<span style="color:var(--text-dim);margin-left:auto;">{{ r.count }} · no agent</span>
|
||||
</div>
|
||||
{% else %}
|
||||
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px solid var(--border);border-radius:5px;padding:0.3rem 0.55rem;">
|
||||
<span class="dot {% if r.status == 'running' %}dot-up{% elif r.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
|
||||
<a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" style="font-weight:500;color:inherit;text-decoration:none;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" title="{{ r.name }}">{{ r.host }}</a>
|
||||
{% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;" title="Docker counts CPU per core">{{ cores(r.cpu_pct / 100) }}</span>{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{# ── Container tables, one per host ───────────────────────────────────────── #}
|
||||
{% if host_groups %}
|
||||
{% for g in host_groups %}
|
||||
<div class="card-flush" style="margin-bottom:1.5rem;">
|
||||
<div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
|
||||
{% if g.host %}
|
||||
<a href="/hosts/{{ g.host.id }}" style="font-weight:500;font-size:0.95rem;color:inherit;text-decoration:none;">{{ g.host.name }}</a>
|
||||
{% else %}
|
||||
<span style="font-weight:500;font-size:0.95rem;">{{ g.host_name }}</span>
|
||||
{% endif %}
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);">
|
||||
{{ g.running }} running{% if g.stopped %} · {{ g.stopped }} stopped{% endif %}
|
||||
</span>
|
||||
</div>
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Container</th>
|
||||
<th>Image</th>
|
||||
<th>Ports</th>
|
||||
<th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
|
||||
<th title="Used, and the limit when one is set">Memory</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for sub in g.subgroups %}
|
||||
{% if g.grouped and sub.label %}
|
||||
<tr><td colspan="5" style="padding:0.4rem 0.75rem 0.2rem;font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;font-weight:500;">{{ sub.label }}</td></tr>
|
||||
{% endif %}
|
||||
{% for item in sub.containers %}
|
||||
{% set c = item.container %}
|
||||
<tr>
|
||||
<td>
|
||||
<div style="display:flex;align-items:center;gap:0.5rem;">
|
||||
{{ status_dot('stale' if item.stale else ('up' if c.status == 'running' else ('pending' if c.status == 'paused' else 'down')), 'Stale' if item.stale else c.status|capitalize) }}
|
||||
<div>
|
||||
<div style="font-weight:500;font-size:0.9rem;">
|
||||
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="color:inherit;text-decoration:none;border-bottom:1px dotted var(--border-mid);">{{ c.name }}</a>
|
||||
{% if c.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}
|
||||
{% elif c.health == 'starting' %} {{ status_badge('pending', 'Starting') }}{% endif %}
|
||||
{% if item.stale %} {{ status_badge('stale', title="Last report " ~ (c.scraped_at.strftime('%Y-%m-%d %H:%M UTC') if c.scraped_at else 'never') ~ "; what is shown may no longer be true") }}{% endif %}
|
||||
</div>
|
||||
<div style="font-size:0.73rem;color:var(--text-muted);">
|
||||
{{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
|
||||
{% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
|
||||
· <span class="tone-{{ 'crit' if item.exit_failed else 'dim' }}" title="{{ 'last exit code' if item.exit_failed else 'stopped by a signal; not a failure' }}">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
|
||||
{% endif %}
|
||||
{% if c.restart_count %} · <span class="tone-warn" title="Restarted {{ c.restart_count }} times">⟳{{ c.restart_count }}</span>{% endif %}
|
||||
</div>
|
||||
{% if c.service_name or c.compose_project %}
|
||||
<div style="font-size:0.68rem;color:var(--text-dim);margin-top:0.1rem;">
|
||||
{{ c.service_name or c.compose_project }}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
<td style="font-size:0.82rem;color:var(--text-muted);max-width:200px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
|
||||
{{ c.image }}
|
||||
</td>
|
||||
<td style="font-size:0.78rem;font-family:ui-monospace,monospace;">
|
||||
{% for p in item.ports %}
|
||||
<div style="color:var(--text-muted);">{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}</div>
|
||||
{% endfor %}
|
||||
</td>
|
||||
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;white-space:nowrap;">{{ cpu(item.usage) }}</td>
|
||||
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;white-space:nowrap;">{{ mem(item.usage) }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% elif not swarm_services %}
|
||||
<div class="card" style="text-align:center;padding:3rem;">
|
||||
<div style="color:var(--text-muted);font-size:0.9rem;">
|
||||
No containers reported yet. Containers are collected by the Steward host agent —
|
||||
deploy the agent to a host running Docker (Hosts → the host → agent panel) and
|
||||
its containers will appear here under that host.
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,95 @@
|
||||
{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement.
|
||||
Services in trouble come first (crash loop, down, short), with since-times;
|
||||
nodes that are down or not taking work come first. #}
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import crumbs %}
|
||||
{% block title %}Swarm — Docker — Steward{% endblock %}
|
||||
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Swarm", "")]) }}{% endblock %}
|
||||
{% block content %}
|
||||
|
||||
<h1 class="page-title" style="margin-bottom:1.5rem;">Swarm</h1>
|
||||
|
||||
{% if swarms %}
|
||||
{% for g in swarms %}
|
||||
<div style="margin-bottom:2rem;">
|
||||
<div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.75rem;flex-wrap:wrap;">
|
||||
<span style="font-weight:500;font-size:0.95rem;">Swarm</span>
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);">
|
||||
reported by {{ g.managers | length }} manager{{ 's' if g.managers | length != 1 }} · {{ g.managers | join(", ") }}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{# ── Services ─────────────────────────────────────────────────────────── #}
|
||||
<div class="card-flush" style="margin-bottom:1rem;">
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Service</th><th>Mode</th><th>Replicas</th><th>Image</th><th>Placement</th><th>Task failures</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for s in g.services %}
|
||||
<tr>
|
||||
<td style="font-weight:500;font-size:0.9rem;">{{ s.name }}</td>
|
||||
<td style="font-size:0.82rem;color:var(--text-muted);">{{ s.mode }}</td>
|
||||
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
|
||||
<span style="color:{% if s.healthy %}var(--green){% elif s.down %}var(--red){% elif s.degraded %}var(--orange){% else %}var(--text-muted){% endif %};">
|
||||
{{ s.running }}/{{ s.desired }}
|
||||
</span>
|
||||
{% if s.state != 'healthy' and s.trouble_since %}<div style="font-family:inherit;font-size:0.72rem;color:var(--text-dim);" title="{{ s.trouble_since }}">since {{ s.trouble_since|ago }}</div>{% endif %}
|
||||
</td>
|
||||
<td style="font-size:0.8rem;color:var(--text-muted);max-width:220px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.image or "—" }}</td>
|
||||
<td style="font-size:0.78rem;color:var(--text-muted);">
|
||||
{% for p in s.placement %}{{ p.node }} ({{ p.running }}){% if not loop.last %}, {% endif %}{% else %}—{% endfor %}
|
||||
</td>
|
||||
<td style="max-width:360px;">
|
||||
{% include "docker/_swarm_failures.html" %}
|
||||
{%- if not s.tasks_known %}<span style="font-size:0.78rem;color:var(--text-dim);" title="The reporting manager's agent predates task reporting (1.8.0); update it to see task failures.">—</span>
|
||||
{%- elif not (s.crash_looping or s.failures_24h or (s.last_error and s.state != 'healthy')) %}<span style="font-size:0.78rem;color:var(--text-dim);">none in 24h</span>{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr><td colspan="6" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No services in this swarm.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{# ── Nodes ────────────────────────────────────────────────────────────── #}
|
||||
<div class="card-flush">
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr><th>Node</th><th>Role</th><th>Availability</th><th>Status</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for n in g.nodes %}
|
||||
<tr>
|
||||
<td style="font-weight:500;font-size:0.9rem;">
|
||||
{{ n.hostname or n.node_id }}
|
||||
{% if n.leader %}<span title="cluster leader" style="font-size:0.68rem;color:var(--accent);border:1px solid var(--accent);border-radius:3px;padding:0 0.3rem;margin-left:0.3rem;">leader</span>{% endif %}
|
||||
</td>
|
||||
<td style="font-size:0.82rem;color:var(--text-muted);">{{ n.role }}</td>
|
||||
<td style="font-size:0.82rem;color:{% if n.availability == 'active' %}var(--text){% else %}var(--orange){% endif %};">{{ n.availability }}</td>
|
||||
<td style="font-size:0.82rem;">
|
||||
<span class="dot {% if n.status == 'ready' %}dot-up{% else %}dot-down{% endif %}"></span>
|
||||
<span style="color:{% if n.status == 'ready' %}var(--green){% else %}var(--red){% endif %};">{{ n.status }}</span>
|
||||
</td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No nodes reported.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
<div class="card" style="text-align:center;padding:3rem;">
|
||||
<div style="color:var(--text-muted);font-size:0.9rem;">
|
||||
No Swarm topology reported. A Steward host agent running on a Swarm
|
||||
<strong>manager</strong> reports services, nodes, and task placement here —
|
||||
workers and non-swarm hosts contribute only their local containers.
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,80 @@
|
||||
{# docker/widget.html — dashboard widget: what Docker is running and what has
|
||||
failed. Swarm services first, troubled ones at the top with since-times; then
|
||||
the containers outside any service, by host, problems first. The failed count
|
||||
links to the Docker page's list of exactly what it counts. #}
|
||||
{% from "docker/_usage.html" import cpu %}
|
||||
{% if total_count == 0 %}
|
||||
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No containers reported yet.</p>
|
||||
{% else %}
|
||||
|
||||
<div style="display:flex;flex-wrap:wrap;gap:0.4rem 1.25rem;margin-bottom:0.65rem;">
|
||||
{% if swarm_services %}
|
||||
<div title="Swarm services with every replica running">
|
||||
<span class="tone-{{ 'ok' if services_healthy == swarm_services|length else 'warn' }}" style="font-size:1.4rem;font-weight:500;line-height:1;">{{ services_healthy }}/{{ swarm_services|length }}</span>
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">services healthy</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
<div title="Containers running outside any swarm service">
|
||||
<span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--green);">{{ running_count }}</span>
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">containers running</span>
|
||||
</div>
|
||||
<a href="/plugins/docker/#failures" style="color:inherit;text-decoration:none;"
|
||||
title="Containers that died or were OOM-killed in the last 24h, and swarm services with failed tasks. A container someone stopped is not counted. Click for the list.">
|
||||
<span class="tone-{{ 'crit' if failures else 'dim' }}" style="font-size:1.4rem;font-weight:500;line-height:1;">{{ failures|length }}</span>
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;border-bottom:1px dotted var(--border-mid);">failed in 24h</span>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{# Swarm services — troubled first, each with its replicas' host chips
|
||||
(dashed = a node with no Steward agent, count-only from placement). #}
|
||||
{% if swarm_services %}
|
||||
<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">Swarm services</div>
|
||||
<div style="display:grid;gap:0.4rem;margin-bottom:0.4rem;">
|
||||
{% for s in swarm_services %}
|
||||
<div>
|
||||
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
|
||||
<span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
|
||||
<span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
|
||||
{% if s.crash_looping %}<span class="tone-crit" style="font-size:0.7rem;font-weight:500;flex-shrink:0;" title="{{ s.failures_1h }} tasks failed in the last hour{% if s.last_error %}: {{ s.last_error.error or s.last_error.message or s.last_error.state }}{% endif %}">crash-looping</span>{% endif %}
|
||||
{% if s.state != 'healthy' and s.trouble_since %}<span class="tone-dim" style="font-size:0.7rem;flex-shrink:0;">since {{ s.trouble_since|ago }}</span>{% endif %}
|
||||
<span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
|
||||
</div>
|
||||
<div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">
|
||||
{% for r in s.replicas %}
|
||||
{% if r.ghost %}
|
||||
<span title="no Steward agent on this node — count from swarm placement" style="font-size:0.68rem;color:var(--text-dim);background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }} ×{{ r.count }}</span>
|
||||
{% else %}
|
||||
<span style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }}</span>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if host_groups and swarm_services %}
|
||||
<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.6rem 0 0.2rem;">Other containers</div>
|
||||
{% endif %}
|
||||
{% for g in host_groups %}
|
||||
{% if multi_host %}
|
||||
<div style="font-size:0.7rem;color:var(--text-dim);margin:0.4rem 0 0.15rem;">{{ g.host_name }}</div>
|
||||
{% endif %}
|
||||
<div style="display:grid;gap:2px;">
|
||||
{% for item in g.containers %}
|
||||
{% set c = item.c %}
|
||||
<div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
|
||||
<span class="dot {% if c.status == 'running' and c.health != 'unhealthy' %}dot-up{% elif c.status in ('paused', 'restarting') %}dot-warn{% else %}dot-down{% endif %}"></span>
|
||||
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;">
|
||||
{{ c.name }}{% if c.health == 'unhealthy' %} <span class="tone-crit" style="font-size:0.7rem;">unhealthy</span>{% endif %}{% if c.status == 'restarting' %} <span class="tone-warn" style="font-size:0.7rem;">restarting</span>{% endif %}
|
||||
</a>
|
||||
{% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
|
||||
<span title="{{ 'last exit code' if item.exit_failed else 'stopped by a signal; not a failure' }}" class="tone-{{ 'crit' if item.exit_failed else 'dim' }}" style="font-size:0.7rem;flex-shrink:0;">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
|
||||
{% endif %}
|
||||
{% if c.status == 'running' %}<span style="font-size:0.72rem;font-family:ui-monospace,monospace;flex-shrink:0;">{{ cpu(item.usage) }}</span>{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
{% endif %}
|
||||
@@ -0,0 +1,42 @@
|
||||
{# docker/widget_resources.html — dashboard widget: the containers using the
|
||||
most CPU (averaged over the last few minutes, so the order holds still),
|
||||
each with its CPU in cores and memory in bytes. The CPU bar is a share of
|
||||
the host's CPUs; the memory bar a share of the container's limit, or of
|
||||
the host's memory when no limit is set. #}
|
||||
{% from "docker/_usage.html" import cpu, mem %}
|
||||
{% if not rows %}
|
||||
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No running containers.</div>
|
||||
{% else %}
|
||||
{% macro bar(label, pct, tone, text, title) %}
|
||||
<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;" title="{{ title }}">
|
||||
<span style="font-size:0.62rem;color:var(--text-dim);width:1.9rem;flex-shrink:0;letter-spacing:0.03em;">{{ label }}</span>
|
||||
<div style="flex:1;height:6px;background:var(--bg-elevated);border-radius:3px;overflow:hidden;">
|
||||
{% if pct is not none %}
|
||||
<div style="height:100%;border-radius:3px;width:{{ [[pct, 0]|max, 100]|min }}%;
|
||||
background:{% if tone == 'crit' %}var(--red){% elif tone == 'warn' %}var(--orange){% else %}var(--accent){% endif %};
|
||||
transition:width 0.4s;"></div>
|
||||
{% endif %}
|
||||
</div>
|
||||
<span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;min-width:4.5rem;text-align:right;flex-shrink:0;white-space:nowrap;">{{ text }}</span>
|
||||
</div>
|
||||
{% endmacro %}
|
||||
<div style="font-size:0.7rem;color:var(--text-dim);margin-bottom:0.45rem;">Busiest by CPU, {{ minutes }}-minute average</div>
|
||||
<div style="display:grid;gap:0.6rem;">
|
||||
{% for r in rows %}
|
||||
{% set u = r.usage %}
|
||||
<div>
|
||||
<div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;" title="{{ r.c.name }}">
|
||||
<a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}" style="color:inherit;text-decoration:none;">{{ r.label }}</a>{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
|
||||
</div>
|
||||
{% if u.cores is not none %}
|
||||
{{ bar("CPU", u.cpu_share, status_class(u.cpu_share, 'cpu'), cpu(u),
|
||||
("%.1f%% of the host's %d core%s" % (u.cpu_share, u.cpus, "" if u.cpus == 1 else "s")) if u.cpus else "Host core count unknown") }}
|
||||
{% endif %}
|
||||
{% if u.mem_bytes is not none %}
|
||||
{{ bar("MEM", u.mem_pct if u.mem_of else none, status_class(u.mem_pct, 'mem') if u.mem_of else 'unknown', mem(u),
|
||||
("%.1f%% of %s" % (u.mem_pct, u.mem_of)) if u.mem_of and u.mem_pct is not none else "") }}
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,44 @@
|
||||
# plugins/host_agent/__init__.py
|
||||
from __future__ import annotations
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from quart import Quart
|
||||
|
||||
_app: "Quart | None" = None
|
||||
|
||||
|
||||
def setup(app: "Quart") -> None:
|
||||
global _app
|
||||
_app = app
|
||||
|
||||
|
||||
def get_scheduled_tasks() -> list:
|
||||
from .scheduler import make_task
|
||||
return [make_task(_app)]
|
||||
|
||||
|
||||
def get_blueprint():
|
||||
from .routes import host_agent_bp
|
||||
return host_agent_bp
|
||||
|
||||
|
||||
def register_mcp_tools(register) -> None:
|
||||
from .mcp_tools import register_tools
|
||||
register_tools(register)
|
||||
|
||||
|
||||
async def mcp_overview(db) -> list[dict]:
|
||||
from .mcp_tools import mcp_overview as findings
|
||||
return await findings(db)
|
||||
|
||||
|
||||
def get_alert_metrics() -> dict[str, list[str]]:
|
||||
"""Host metrics ingest records for alert rules (resource = the host name;
|
||||
disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
|
||||
comes from the scheduler, not the agent: "agent offline" is > 180."""
|
||||
return {"host_agent": [
|
||||
"cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
|
||||
"disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m",
|
||||
"uptime_secs", "seconds_since_seen",
|
||||
]}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,81 @@
|
||||
"""The host_agent plugin's MCP tools and its get_overview findings."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from steward.hosts import service as host_svc
|
||||
from steward.mcp._context import mcp_app
|
||||
from steward.mcp.tools._shape import age_s, db_session, iso, now_utc
|
||||
from steward.mcp.tools.overview import finding
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.users import UserRole
|
||||
|
||||
from .models import HostAgentRegistration
|
||||
|
||||
# A registration this old that has never checked in is a failed install, not one in progress.
|
||||
PENDING_GRACE = timedelta(minutes=30)
|
||||
|
||||
|
||||
def agent_state(reg, now, stale_after_s: int) -> str:
|
||||
"""The agent's state: reporting, stale (stopped checking in) or pending (never has)."""
|
||||
if reg.last_seen_at is None:
|
||||
return "pending"
|
||||
return "reporting" if (now - reg.last_seen_at).total_seconds() <= stale_after_s else "stale"
|
||||
|
||||
|
||||
async def _agents(db):
|
||||
return (await db.execute(
|
||||
select(HostAgentRegistration, Host)
|
||||
.join(Host, Host.id == HostAgentRegistration.host_id)
|
||||
.order_by(Host.name)
|
||||
)).all()
|
||||
|
||||
|
||||
async def list_agents() -> dict:
|
||||
"""Every host agent Steward has registered, with its version and whether it is reporting.
|
||||
|
||||
`state` is "reporting", "stale" (it checked in before but its last
|
||||
check-in is older than the stale window, `stale_after_s`) or "pending"
|
||||
(registered, never checked in, usually an install that is still running
|
||||
or that failed). Each row carries the agent version, kernel, distro, arch
|
||||
and the IP the agent reports. get_host shows an agent's latest metrics.
|
||||
"""
|
||||
now = now_utc()
|
||||
stale_after = host_svc.agent_stale_after_seconds(mcp_app().config)
|
||||
async with db_session() as db:
|
||||
rows = await _agents(db)
|
||||
agents = [
|
||||
{
|
||||
"host_id": h.id, "host_name": h.name,
|
||||
"state": agent_state(reg, now, stale_after),
|
||||
"last_seen_at": iso(reg.last_seen_at), "last_seen_age_s": age_s(reg.last_seen_at, now),
|
||||
"agent_version": reg.agent_version, "kernel": reg.kernel, "distro": reg.distro,
|
||||
"arch": reg.arch, "reported_ip": reg.host_ip,
|
||||
"registered_at": iso(reg.created_at),
|
||||
}
|
||||
for reg, h in rows
|
||||
]
|
||||
counts = {s: sum(1 for a in agents if a["state"] == s) for s in ("reporting", "stale", "pending")}
|
||||
return {"stale_after_s": stale_after, "counts": counts, "agents": agents, "total": len(agents)}
|
||||
|
||||
|
||||
async def mcp_overview(db) -> list[dict]:
|
||||
"""Agents registered more than PENDING_GRACE ago that have never checked in.
|
||||
|
||||
Agents that stopped reporting are already flagged by the core overview
|
||||
from their metrics, so they are not repeated here.
|
||||
"""
|
||||
now = now_utc()
|
||||
return [
|
||||
finding("warning", "agent_never_reported", h.name,
|
||||
"host agent was registered but has never checked in; the install may have failed",
|
||||
since=reg.created_at, ids={"host_id": h.id}, next="list_agents()")
|
||||
for reg, h in await _agents(db)
|
||||
if reg.last_seen_at is None and now - reg.created_at > PENDING_GRACE
|
||||
]
|
||||
|
||||
|
||||
def register_tools(register) -> None:
|
||||
register(list_agents, role=UserRole.viewer, read_only=True)
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Host-metric read helpers (latest snapshot + bucketed history).
|
||||
|
||||
Kept separate from routes.py so it imports only the core PluginMetric model — not
|
||||
the host_agent ORM models — which lets integration tests import these helpers
|
||||
without tripping the plugin-loader's double-registration ("Table already
|
||||
defined") guard.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from steward.core.time_range import bucket_seconds
|
||||
from steward.models.metrics import PluginMetric, PluginMetricHourly
|
||||
|
||||
SOURCE_MODULE = "host_agent"
|
||||
|
||||
# Host-level metrics charted on the detail page (sub-resources are shown as
|
||||
# current-value lists, not time series, to keep the page readable).
|
||||
HISTORY_METRICS = (
|
||||
"cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m",
|
||||
"net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
|
||||
"temp_c_max", "psi_mem_some_avg10",
|
||||
)
|
||||
|
||||
|
||||
async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]:
|
||||
"""{resource_name: {metric: value}} — latest sample per (resource, metric) for
|
||||
a host + its sub-resources, from plugin_metric_latest (#5208). A host whose
|
||||
agent has gone quiet keeps its last values until they age out of the raw
|
||||
window; the caller decides staleness from the registration's last_seen_at.
|
||||
"""
|
||||
from steward.core.metric_series import latest_values
|
||||
|
||||
out: dict[str, dict[str, float]] = {}
|
||||
for r in await latest_values(session, SOURCE_MODULE, host_name, include_children=True):
|
||||
out.setdefault(r.resource_name, {})[r.metric_name] = r.value
|
||||
return out
|
||||
|
||||
|
||||
async def _history_for_host(session, host_name: str, since, *, raw_days: int = 7) -> dict[str, list]:
|
||||
"""{metric: [[epoch_ms, avg_value], …]} host-level series since `since`.
|
||||
|
||||
Retention rolls raw plugin_metrics older than `raw_days` into hourly averages
|
||||
(plugin_metrics_hourly) and deletes the raw rows. So we read the rollup for the
|
||||
part of the range older than that boundary and raw (bucket-averaged in SQL,
|
||||
capped to ≤1h to match the rollup) for the recent part — never shipping raw
|
||||
samples to Python. The two windows don't overlap, so appending hourly-then-raw
|
||||
keeps each metric's series time-ordered. Epoch-ms x feeds a linear chart axis.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
raw_cutoff = (now - timedelta(days=raw_days)).replace(minute=0, second=0, microsecond=0)
|
||||
series: dict[str, list] = {m: [] for m in HISTORY_METRICS}
|
||||
|
||||
# ── Older than the raw window: the hourly rollup ──
|
||||
if since < raw_cutoff:
|
||||
hourly = (await session.execute(
|
||||
select(PluginMetricHourly.metric_name, PluginMetricHourly.bucket,
|
||||
PluginMetricHourly.value_avg)
|
||||
.where(
|
||||
PluginMetricHourly.source_module == SOURCE_MODULE,
|
||||
PluginMetricHourly.resource_name == host_name,
|
||||
PluginMetricHourly.metric_name.in_(HISTORY_METRICS),
|
||||
PluginMetricHourly.bucket >= since,
|
||||
PluginMetricHourly.bucket < raw_cutoff,
|
||||
)
|
||||
.order_by(PluginMetricHourly.bucket)
|
||||
)).all()
|
||||
for metric_name, bucket, avg in hourly:
|
||||
series[metric_name].append([int(bucket.timestamp() * 1000), round(float(avg), 2)])
|
||||
|
||||
# ── Recent part: raw, bucket-averaged in SQL ──
|
||||
raw_since = since if since >= raw_cutoff else raw_cutoff
|
||||
width_s = bucket_seconds(raw_since, 120)
|
||||
if since < raw_cutoff:
|
||||
width_s = min(width_s, 3600) # ≤ 1h so it matches the hourly portion
|
||||
rbucket = func.date_bin(
|
||||
func.make_interval(0, 0, 0, 0, 0, 0, width_s),
|
||||
PluginMetric.recorded_at,
|
||||
func.to_timestamp(0), # epoch origin
|
||||
).label("bucket")
|
||||
raw = (await session.execute(
|
||||
select(PluginMetric.metric_name, rbucket, func.avg(PluginMetric.value))
|
||||
.where(
|
||||
PluginMetric.source_module == SOURCE_MODULE,
|
||||
PluginMetric.resource_name == host_name,
|
||||
PluginMetric.metric_name.in_(HISTORY_METRICS),
|
||||
PluginMetric.recorded_at >= raw_since,
|
||||
)
|
||||
.group_by(PluginMetric.metric_name, rbucket)
|
||||
.order_by(rbucket)
|
||||
)).all()
|
||||
for metric_name, b, avg in raw:
|
||||
series[metric_name].append([int(b.timestamp() * 1000), round(float(avg), 2)])
|
||||
return series
|
||||
@@ -0,0 +1,70 @@
|
||||
# plugins/host_agent/migrations/env.py
|
||||
"""Alembic env.py for the host_agent plugin."""
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||
from alembic import context
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
|
||||
|
||||
from steward.models.base import Base
|
||||
import steward.models # noqa: F401
|
||||
from plugins.host_agent.models import HostAgentRegistration # noqa: F401
|
||||
|
||||
config = context.config
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def _get_url() -> str:
|
||||
import yaml
|
||||
cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml")
|
||||
try:
|
||||
with open(cfg_path) as f:
|
||||
cfg = yaml.safe_load(f) or {}
|
||||
url = cfg.get("database", {}).get("url", "")
|
||||
except FileNotFoundError:
|
||||
url = ""
|
||||
return os.environ.get("STEWARD_DATABASE__URL", url)
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
context.configure(
|
||||
url=_get_url(), target_metadata=target_metadata,
|
||||
literal_binds=True, dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def do_run_migrations(connection: Connection) -> None:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
async def run_async_migrations() -> None:
|
||||
cfg = config.get_section(config.config_ini_section, {})
|
||||
cfg["sqlalchemy.url"] = _get_url()
|
||||
connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool)
|
||||
async with connectable.connect() as connection:
|
||||
await connection.run_sync(do_run_migrations)
|
||||
await connectable.dispose()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
asyncio.run(run_async_migrations())
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,36 @@
|
||||
# plugins/host_agent/migrations/versions/host_agent_001_initial.py
|
||||
"""host_agent plugin initial tables
|
||||
|
||||
Revision ID: host_agent_001_initial
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "host_agent_001_initial"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = "host_agent"
|
||||
depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"host_agent_registrations",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("host_id", sa.String(36),
|
||||
sa.ForeignKey("hosts.id", ondelete="CASCADE"),
|
||||
nullable=False, unique=True),
|
||||
sa.Column("token_hash", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("token_created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("agent_version", sa.String(32), nullable=True),
|
||||
sa.Column("kernel", sa.String(128), nullable=True),
|
||||
sa.Column("distro", sa.String(128), nullable=True),
|
||||
sa.Column("arch", sa.String(32), nullable=True),
|
||||
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("host_agent_registrations")
|
||||
@@ -0,0 +1,24 @@
|
||||
# plugins/host_agent/migrations/versions/host_agent_002_host_ip.py
|
||||
"""host_agent: add agent-reported host_ip column
|
||||
|
||||
Revision ID: host_agent_002_host_ip
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "host_agent_002_host_ip"
|
||||
down_revision: Union[str, None] = "host_agent_001_initial"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"host_agent_registrations",
|
||||
sa.Column("host_ip", sa.String(45), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("host_agent_registrations", "host_ip")
|
||||
@@ -0,0 +1,34 @@
|
||||
# plugins/host_agent/models.py
|
||||
from __future__ import annotations
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy import Column, String, DateTime, ForeignKey
|
||||
from steward.models.base import Base
|
||||
|
||||
|
||||
def _uuid() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class HostAgentRegistration(Base):
|
||||
__tablename__ = "host_agent_registrations"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=_uuid)
|
||||
host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
|
||||
unique=True, nullable=False)
|
||||
token_hash = Column(String(64), nullable=False, unique=True, index=True)
|
||||
token_created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
||||
agent_version = Column(String(32), nullable=True)
|
||||
kernel = Column(String(128), nullable=True)
|
||||
distro = Column(String(128), nullable=True)
|
||||
arch = Column(String(32), nullable=True)
|
||||
# Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix).
|
||||
host_ip = Column(String(45), nullable=True)
|
||||
last_seen_at = Column(DateTime(timezone=True), nullable=True)
|
||||
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
||||
updated_at = Column(DateTime(timezone=True), nullable=False,
|
||||
default=_utcnow, onupdate=_utcnow)
|
||||
@@ -0,0 +1,19 @@
|
||||
# plugins/host_agent/plugin.yaml
|
||||
name: host_agent
|
||||
version: "1.2.0"
|
||||
description: "Remote Linux host resource monitoring via a lightweight Python push agent (CPU incl. per-core, memory + PSI, storage, disk I/O, network throughput, load, temperatures, uptime)"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/host_agent"
|
||||
tags:
|
||||
- host
|
||||
- monitoring
|
||||
- cpu
|
||||
- memory
|
||||
- storage
|
||||
|
||||
config:
|
||||
stale_after_seconds: 180
|
||||
default_interval_seconds: 30
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,65 @@
|
||||
# plugins/host_agent/scheduler.py
|
||||
from __future__ import annotations
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Iterable
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from steward.core.alerts import MetricBatch
|
||||
from steward.core.scheduler import ScheduledTask
|
||||
from steward.models.hosts import Host
|
||||
from .models import HostAgentRegistration
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
|
||||
now: datetime) -> list[tuple[str, float]]:
|
||||
"""(host name, seconds since its agent last reported) per registration.
|
||||
|
||||
An agent that has never reported counts from when it was registered. A
|
||||
registration whose host is gone is skipped.
|
||||
"""
|
||||
out = []
|
||||
for r in regs:
|
||||
name = host_names.get(r.host_id)
|
||||
seen = r.last_seen_at or r.created_at
|
||||
if name is None or seen is None:
|
||||
continue
|
||||
out.append((name, max(0.0, (now - seen).total_seconds())))
|
||||
return out
|
||||
|
||||
|
||||
async def record_seconds_since_seen(app) -> None:
|
||||
"""Record host_agent/<host>/seconds_since_seen for every registered agent,
|
||||
so "agent offline" is an ordinary alert rule (seconds_since_seen > 180)."""
|
||||
now = datetime.now(timezone.utc)
|
||||
async with app.db_sessionmaker() as session:
|
||||
async with session.begin():
|
||||
regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
|
||||
if not regs:
|
||||
return
|
||||
host_names = dict((await session.execute(
|
||||
select(Host.id, Host.name).where(Host.id.in_([r.host_id for r in regs]))
|
||||
)).all())
|
||||
batch = MetricBatch()
|
||||
for name, seconds in seconds_since_seen(regs, host_names, now):
|
||||
batch.add(source_module="host_agent", resource_name=name,
|
||||
metric_name="seconds_since_seen", value=seconds, recorded_at=now)
|
||||
await batch.record(session)
|
||||
|
||||
|
||||
def make_task(app) -> ScheduledTask:
|
||||
async def _record():
|
||||
try:
|
||||
await record_seconds_since_seen(app)
|
||||
except Exception:
|
||||
logger.exception("host_agent: recording seconds_since_seen failed")
|
||||
|
||||
return ScheduledTask(
|
||||
name="host_agent_seconds_since_seen",
|
||||
coro_factory=_record,
|
||||
interval_seconds=60,
|
||||
run_on_startup=False,
|
||||
)
|
||||
@@ -0,0 +1,6 @@
|
||||
{# History data-only fragment. Swapped into the hidden #hm-charts-data poller;
|
||||
the script runs and feeds the persistent charts created by host_detail.html,
|
||||
updating them in place (no canvas swap → no flicker / no re-animation). #}
|
||||
<script>
|
||||
if (window.applyHostSeries) window.applyHostSeries({{ series|tojson }}, {{ range_key|tojson }});
|
||||
</script>
|
||||
@@ -0,0 +1,165 @@
|
||||
{# Current-state fragment for the full-metrics page — polled live via HTMX.
|
||||
Each poll re-renders the latest snapshot the server has. #}
|
||||
{% from "_macros.html" import status_badge %}
|
||||
{# A percentage bar graded by Settings → Thresholds for `kind` (cpu, mem, disk). #}
|
||||
{% macro bar(pct, kind) %}
|
||||
<div class="meter"><div class="meter-fill meter-{{ status_class(pct, kind) }}" style="width:{{ (pct or 0)|round(1) }}%;"></div></div>
|
||||
{% endmacro %}
|
||||
{% macro gauge(value, kind, text) %}<span class="stat-val tone-{{ status_class(value, kind) }}" style="font-size:1.6rem;">{{ text }}</span>{% endmacro %}
|
||||
|
||||
{# ── Identity / metadata ─────────────────────────────────────────────────── #}
|
||||
<div class="card" style="display:flex;flex-wrap:wrap;gap:0.5rem 1.25rem;align-items:center;font-size:0.82rem;color:var(--text-muted);">
|
||||
<span>{{ status_badge('stale' if stale else 'up', 'Stale' if stale else 'Live') }}</span>
|
||||
<span><span style="color:var(--text-dim);">Address</span> {{ host.address or "—" }}</span>
|
||||
{% if reg %}
|
||||
{% if reg.distro %}<span><span style="color:var(--text-dim);">OS</span> {{ reg.distro }}</span>{% endif %}
|
||||
{% if reg.kernel %}<span><span style="color:var(--text-dim);">Kernel</span> {{ reg.kernel }}</span>{% endif %}
|
||||
{% if reg.arch %}<span><span style="color:var(--text-dim);">Arch</span> {{ reg.arch }}</span>{% endif %}
|
||||
{% if reg.agent_version %}<span><span style="color:var(--text-dim);">Agent</span> v{{ reg.agent_version }}</span>{% endif %}
|
||||
{% set up = hostlvl.get('uptime_secs') %}
|
||||
{% if up %}<span><span style="color:var(--text-dim);">Uptime</span> {{ up|duration }}</span>{% endif %}
|
||||
<span><span style="color:var(--text-dim);">Last seen</span> {{ reg.last_seen_at|ago if reg.last_seen_at else "never" }}</span>
|
||||
{% else %}
|
||||
<span>No agent registration found for this host.</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% if not hostlvl %}
|
||||
<div class="card empty">No metrics reported yet. Once the agent checks in, CPU, memory, network, disk I/O, temperatures, and pressure will appear here.</div>
|
||||
{% else %}
|
||||
|
||||
{# ── Current headline gauges ─────────────────────────────────────────────── #}
|
||||
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1rem;">
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.35rem;">CPU</div>
|
||||
{% set c = hostlvl.get('cpu_pct') %}
|
||||
{{ gauge(c, 'cpu', ("%.0f%%"|format(c)) if c is not none else "—") }}
|
||||
<div style="margin-top:0.5rem;">{{ bar(c, 'cpu') }}</div>
|
||||
</div>
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.35rem;">Memory</div>
|
||||
{% set mp = hostlvl.get('mem_used_pct') %}
|
||||
{{ gauge(mp, 'mem', ("%.0f%%"|format(mp)) if mp is not none else "—") }}
|
||||
<div style="margin-top:0.5rem;">{{ bar(mp, 'mem') }}</div>
|
||||
<div style="margin-top:0.45rem;font-size:0.72rem;color:var(--text-dim);">
|
||||
avail {{ hostlvl.get('mem_available_bytes')|bytes }} · cache {{ hostlvl.get('mem_cached_bytes')|bytes }} · swap {{ hostlvl.get('swap_used_bytes')|bytes }}
|
||||
</div>
|
||||
</div>
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.35rem;">Load</div>
|
||||
<span class="stat-val" style="font-size:1.6rem;">{% if hostlvl.get('load_1m') is not none %}{{ "%.2f"|format(hostlvl.get('load_1m')) }}{% else %}—{% endif %}</span>
|
||||
<div style="margin-top:0.45rem;font-size:0.72rem;color:var(--text-dim);">
|
||||
5m {% if hostlvl.get('load_5m') is not none %}{{ "%.2f"|format(hostlvl.get('load_5m')) }}{% else %}—{% endif %} ·
|
||||
15m {% if hostlvl.get('load_15m') is not none %}{{ "%.2f"|format(hostlvl.get('load_15m')) }}{% else %}—{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.35rem;">Network</div>
|
||||
<div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
|
||||
<div><span style="color:var(--green);">↓</span> {{ hostlvl.get('net_rx_bps')|rate }}</div>
|
||||
<div><span style="color:var(--red);">↑</span> {{ hostlvl.get('net_tx_bps')|rate }}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.35rem;">Disk I/O</div>
|
||||
<div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
|
||||
<div><span style="color:var(--text-dim);">rd</span> {{ hostlvl.get('disk_read_bps')|rate }}</div>
|
||||
<div><span style="color:var(--text-dim);">wr</span> {{ hostlvl.get('disk_write_bps')|rate }}</div>
|
||||
</div>
|
||||
</div>
|
||||
{% if hostlvl.get('temp_c_max') is not none %}
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.35rem;">Temp (max)</div>
|
||||
{% set t = hostlvl.get('temp_c_max') %}
|
||||
{{ gauge(t, 'temp', "%.0f°C"|format(t)) }}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if hostlvl.get('psi_mem_some_avg10') is not none %}
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.35rem;" title="Pressure stall — % of time stalled, 10s avg">Pressure (10s)</div>
|
||||
<div style="font-size:0.78rem;font-variant-numeric:tabular-nums;color:var(--text-muted);">
|
||||
<div>mem {{ "%.1f"|format(hostlvl.get('psi_mem_some_avg10')) }}%</div>
|
||||
{% if hostlvl.get('psi_cpu_some_avg10') is not none %}<div>cpu {{ "%.1f"|format(hostlvl.get('psi_cpu_some_avg10')) }}%</div>{% endif %}
|
||||
{% if hostlvl.get('psi_io_some_avg10') is not none %}<div>io {{ "%.1f"|format(hostlvl.get('psi_io_some_avg10')) }}%</div>{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{# ── Per-core CPU ─────────────────────────────────────────────────────────── #}
|
||||
{% if cores %}
|
||||
<div class="card">
|
||||
<div class="section-title" style="margin-bottom:0.6rem;">Per-core CPU</div>
|
||||
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(120px,1fr));gap:0.5rem 0.9rem;">
|
||||
{% for idx, pct in cores %}
|
||||
<div>
|
||||
<div style="display:flex;justify-content:space-between;font-size:0.72rem;color:var(--text-dim);margin-bottom:0.2rem;">
|
||||
<span>core {{ idx }}</span><span>{% if pct is not none %}{{ "%.0f"|format(pct) }}%{% else %}—{% endif %}</span>
|
||||
</div>
|
||||
{{ bar(pct, 'cpu') }}
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{# ── Filesystems ──────────────────────────────────────────────────────────── #}
|
||||
{% if mounts %}
|
||||
<div class="card">
|
||||
<div class="section-title" style="margin-bottom:0.6rem;">Filesystems</div>
|
||||
{% for mount, m in mounts.items() %}
|
||||
<div style="margin-bottom:0.6rem;">
|
||||
<div style="display:flex;justify-content:space-between;font-size:0.8rem;margin-bottom:0.2rem;">
|
||||
<span style="font-family:ui-monospace,monospace;">{{ mount }}</span>
|
||||
<span style="color:var(--text-muted);">{{ m.get('disk_used_bytes')|bytes }} / {{ m.get('disk_total_bytes')|bytes }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %}</span>
|
||||
</div>
|
||||
{{ bar(m.get('disk_used_pct'), 'disk') }}
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{# ── Interfaces / disks (short panels, side by side at natural height) ─────── #}
|
||||
{% if nets or disks_io %}
|
||||
<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(260px,1fr));gap:1rem;align-items:start;">
|
||||
{% if nets %}
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.5rem;">Interfaces</div>
|
||||
{% for iface, m in nets.items() %}
|
||||
<div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
|
||||
<span style="font-family:ui-monospace,monospace;">{{ iface }}</span>
|
||||
<span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">↓ {{ m.get('net_rx_bps')|rate }} · ↑ {{ m.get('net_tx_bps')|rate }}</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if disks_io %}
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.5rem;">Disks</div>
|
||||
{% for dev, m in disks_io.items() %}
|
||||
<div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
|
||||
<span style="font-family:ui-monospace,monospace;">{{ dev }}</span>
|
||||
<span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">rd {{ m.get('disk_read_bps')|rate }} · wr {{ m.get('disk_write_bps')|rate }}</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{# ── Temperatures (full width; cores flow into a compact multi-column grid) ── #}
|
||||
{% if temps %}
|
||||
<div class="card" style="margin-top:1rem;margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.5rem;">Temperatures</div>
|
||||
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(130px,1fr));gap:0.1rem 1.25rem;">
|
||||
{% for label, c in temps.items() %}
|
||||
<div style="display:flex;justify-content:space-between;gap:0.5rem;font-size:0.8rem;padding:0.15rem 0;">
|
||||
<span style="white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ label }}</span>
|
||||
<span class="tone-{{ status_class(c, 'temp') }}" style="font-variant-numeric:tabular-nums;">{% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %}</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% endif %}
|
||||
@@ -0,0 +1,33 @@
|
||||
{# Host health header — live polled into the host hub: live or stale, the worst
|
||||
reading against Settings → Thresholds, last seen, then CPU / memory / disk /
|
||||
load tiles graded by the same cutoffs. Renders nothing until the agent
|
||||
reports (the Agent panel below then shows provisioning). #}
|
||||
{% from "_macros.html" import status_badge, stat_tile %}
|
||||
{% macro pct(v) %}{{ '%.0f%%'|format(v) if v is not none else '—' }}{% endmacro %}
|
||||
{% if reporting %}
|
||||
<section class="card host-health" aria-label="Host health">
|
||||
<div class="host-health-head">
|
||||
{{ status_badge('stale' if stale else 'up', 'Stale' if stale else 'Live') }}
|
||||
<span class="host-health-summary">
|
||||
{%- if stale %}No report since {{ reg.last_seen_at|ago }}; the readings below are from then.
|
||||
{%- elif worst %}<span class="tone-{{ worst[2] }}">{{ worst[0] }} {{ pct(worst[1]) }}</span> is past its {{ "critical" if worst[2] == "crit" else "warning" }} threshold.
|
||||
{%- else %}Every reading is within its thresholds.{% endif -%}
|
||||
</span>
|
||||
<span class="problem-detail">
|
||||
{%- if not stale %}seen {{ reg.last_seen_at|ago }}{% endif %}
|
||||
{%- if reg.agent_version %}{{ " · " if not stale }}agent v{{ reg.agent_version }}{% endif -%}
|
||||
</span>
|
||||
</div>
|
||||
<div class="stat-strip" style="margin-bottom:0;">
|
||||
{{ stat_tile("CPU", pct(cpu), state=status_class(cpu, 'cpu'), hint=sparks.cpu|safe) }}
|
||||
{{ stat_tile("Memory", pct(mem), state=status_class(mem, 'mem'), hint=sparks.mem|safe) }}
|
||||
{{ stat_tile("Disk /", pct(disk_root), state=status_class(disk_root, 'disk'), hint=sparks.disk|safe) }}
|
||||
{{ stat_tile("Load per core", pct(load_per_core), state=status_class(load_per_core, 'load'), hint=sparks.load|safe) }}
|
||||
</div>
|
||||
{% if psi.cpu is not none or psi.mem is not none or psi.io is not none %}
|
||||
<p class="field-hint" style="margin-top:0.6rem;" title="Pressure stall: the share of the last 10 s that tasks waited for the resource">
|
||||
Pressure (10 s): cpu {{ pct(psi.cpu) }} · mem {{ pct(psi.mem) }} · io {{ pct(psi.io) }}
|
||||
</p>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,108 @@
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import crumbs %}
|
||||
{% block title %}{{ host.name }} — Metrics — Steward{% endblock %}
|
||||
{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), (host.name, "/hosts/" ~ host.id), ("Metrics", "")]) }}{% endblock %}
|
||||
{% block content %}
|
||||
{# Shell only: current-state + history data stream in via HTMX so the heavy
|
||||
history query never blocks first paint and the data refreshes live (≈ the
|
||||
agent's push cadence). The chart CANVASES live here permanently — only their
|
||||
data is polled and applied in place, so refreshes never re-create/flicker. #}
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;flex-wrap:wrap;margin-bottom:0.75rem;">
|
||||
<h1 class="page-title" style="margin-bottom:0;">{{ host.name }}</h1>
|
||||
{% include "_time_range.html" %}
|
||||
</div>
|
||||
|
||||
{# Current state — lazy + polled live. Atomic innerHTML swap of fixed-height
|
||||
cards, so values update without a layout jump. #}
|
||||
<div id="hm-current"
|
||||
hx-get="/plugins/host_agent/{{ host.id }}/metrics"
|
||||
hx-trigger="load, every {{ poll_seconds }}s"
|
||||
hx-swap="innerHTML">
|
||||
<div class="card empty">Loading metrics…</div>
|
||||
</div>
|
||||
|
||||
{# History charts — persistent canvases (created once below); only data is polled. #}
|
||||
<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(340px,1fr));gap:1rem;margin-top:1rem;">
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.5rem;">Utilization % — last <span class="hm-chart-range">{{ current_range }}</span></div>
|
||||
<div style="height:220px;"><canvas id="chart-util"></canvas></div>
|
||||
</div>
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.5rem;">Throughput (B/s) — last <span class="hm-chart-range">{{ current_range }}</span></div>
|
||||
<div style="height:220px;"><canvas id="chart-net"></canvas></div>
|
||||
</div>
|
||||
<div class="card" style="margin-bottom:0;">
|
||||
<div class="section-title" style="margin-bottom:0.5rem;">Load & pressure — last <span class="hm-chart-range">{{ current_range }}</span></div>
|
||||
<div style="height:220px;"><canvas id="chart-pressure"></canvas></div>
|
||||
</div>
|
||||
</div>
|
||||
{# Hidden poller: fetches the history series and applies it to the charts above
|
||||
(no visible swap). Lazy on load → never blocks paint; refreshed slowly + on
|
||||
range change. #}
|
||||
<div id="hm-charts-data" style="display:none;"
|
||||
hx-get="/plugins/host_agent/{{ host.id }}/charts"
|
||||
hx-trigger="load, every {{ chart_poll_seconds }}s, rangeChange from:body"
|
||||
hx-include="#time-range"
|
||||
hx-swap="innerHTML"></div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
const fmtTime = (v) => { const d = new Date(v); return ("0" + d.getHours()).slice(-2) + ":" + ("0" + d.getMinutes()).slice(-2); };
|
||||
const baseOpts = (ymax) => ({
|
||||
responsive: true, maintainAspectRatio: false,
|
||||
animation: false, // no grow-in / re-animate on refresh
|
||||
interaction: { mode: "index", intersect: false },
|
||||
elements: { point: { radius: 0 } },
|
||||
scales: {
|
||||
x: { type: "linear", ticks: { callback: (v) => fmtTime(v), maxTicksLimit: 8, font: { size: 10 } } },
|
||||
y: { beginAtZero: true, max: ymax, ticks: { font: { size: 10 } } },
|
||||
},
|
||||
plugins: {
|
||||
legend: { labels: { boxWidth: 10, font: { size: 11 } } },
|
||||
tooltip: { callbacks: { title: (items) => items.length ? fmtTime(items[0].parsed.x) : "" } },
|
||||
},
|
||||
});
|
||||
const defs = {
|
||||
"chart-util": { ymax: 100, ds: [
|
||||
{ key: "cpu_pct", label: "CPU %", color: stewardColor("--gold") },
|
||||
{ key: "mem_used_pct", label: "Mem %", color: stewardColor("--green") },
|
||||
{ key: "disk_used_pct_worst", label: "Disk % (worst)", color: stewardColor("--yellow") },
|
||||
] },
|
||||
"chart-net": { ymax: undefined, ds: [
|
||||
{ key: "net_rx_bps", label: "Net RX", color: stewardColor("--green") },
|
||||
{ key: "net_tx_bps", label: "Net TX", color: stewardColor("--red") },
|
||||
{ key: "disk_read_bps", label: "Disk read", color: stewardColor("--blue") },
|
||||
{ key: "disk_write_bps", label: "Disk write", color: stewardColor("--gold") },
|
||||
] },
|
||||
"chart-pressure": { ymax: undefined, ds: [
|
||||
{ key: "load_1m", label: "Load 1m", color: stewardColor("--gold") },
|
||||
{ key: "psi_mem_some_avg10", label: "Mem PSI %", color: stewardColor("--red") },
|
||||
{ key: "temp_c_max", label: "Temp °C", color: stewardColor("--yellow") },
|
||||
] },
|
||||
};
|
||||
const charts = {};
|
||||
for (const id of Object.keys(defs)) {
|
||||
const el = document.getElementById(id);
|
||||
if (!el) continue;
|
||||
charts[id] = new Chart(el.getContext("2d"), {
|
||||
type: "line",
|
||||
data: { datasets: defs[id].ds.map((d) => ({ label: d.label, data: [], borderColor: d.color, backgroundColor: d.color, borderWidth: 1.5, tension: 0.25 })) },
|
||||
options: baseOpts(defs[id].ymax),
|
||||
});
|
||||
}
|
||||
// Called by the polled /charts data fragment. Updates data in place (no
|
||||
// re-create, no animation), so refreshes never flicker or shift layout.
|
||||
window.applyHostSeries = function (series, rangeKey) {
|
||||
for (const id of Object.keys(defs)) {
|
||||
const ch = charts[id];
|
||||
if (!ch) continue;
|
||||
defs[id].ds.forEach((d, i) => {
|
||||
ch.data.datasets[i].data = (series[d.key] || []).map((p) => ({ x: p[0], y: p[1] }));
|
||||
});
|
||||
ch.update("none");
|
||||
}
|
||||
if (rangeKey) document.querySelectorAll(".hm-chart-range").forEach((s) => { s.textContent = rangeKey; });
|
||||
};
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/bin/sh
|
||||
# Steward host agent installer
|
||||
# Generated for: {{ host_name }} ({{ host_address }})
|
||||
# Steward URL: {{ url }}
|
||||
set -e
|
||||
|
||||
STEWARD_URL="{{ url }}"
|
||||
AGENT_TOKEN="{{ token }}"
|
||||
AGENT_VERSION="{{ agent_version }}"
|
||||
|
||||
AGENT_USER="steward-agent"
|
||||
AGENT_DIR="/usr/local/lib/steward-agent"
|
||||
CONF_FILE="/etc/steward-agent.conf"
|
||||
UNIT_FILE="/etc/systemd/system/steward-agent.service"
|
||||
|
||||
# ── preflight ────────────────────────────────────────────────────────────────
|
||||
[ "$(id -u)" = "0" ] || { echo "must run as root (use sudo)"; exit 1; }
|
||||
command -v systemctl >/dev/null 2>&1 || { echo "systemd not found — unsupported init system"; exit 1; }
|
||||
command -v python3 >/dev/null 2>&1 || { echo "python3 not found — install python3 first"; exit 1; }
|
||||
|
||||
# Handle --uninstall
|
||||
if [ "${1:-}" = "--uninstall" ]; then
|
||||
systemctl disable --now steward-agent.service 2>/dev/null || true
|
||||
rm -f "$UNIT_FILE" "$CONF_FILE"
|
||||
rm -rf "$AGENT_DIR"
|
||||
systemctl daemon-reload
|
||||
# keep the system user — removing it risks orphaned files elsewhere
|
||||
echo "uninstalled"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── create system user ───────────────────────────────────────────────────────
|
||||
if ! id "$AGENT_USER" >/dev/null 2>&1; then
|
||||
useradd --system --no-create-home --shell /usr/sbin/nologin "$AGENT_USER"
|
||||
fi
|
||||
|
||||
# ── drop the agent file ──────────────────────────────────────────────────────
|
||||
mkdir -p "$AGENT_DIR"
|
||||
curl -sSL "${STEWARD_URL}/plugins/host_agent/agent.py" -o "$AGENT_DIR/agent.py"
|
||||
chmod 0755 "$AGENT_DIR/agent.py"
|
||||
chown root:root "$AGENT_DIR/agent.py"
|
||||
|
||||
# ── write config ─────────────────────────────────────────────────────────────
|
||||
cat > "$CONF_FILE" <<EOF
|
||||
url = $STEWARD_URL
|
||||
token = $AGENT_TOKEN
|
||||
interval_seconds = 30
|
||||
# Container logs are collected by default. To opt this host out entirely:
|
||||
# docker_logs_enabled = false
|
||||
# To skip specific noisy containers (comma-separated names):
|
||||
# docker_log_exclude = watchtower, some-chatty-service
|
||||
EOF
|
||||
chown "root:$AGENT_USER" "$CONF_FILE"
|
||||
chmod 0640 "$CONF_FILE"
|
||||
|
||||
# ── write systemd unit ───────────────────────────────────────────────────────
|
||||
cat > "$UNIT_FILE" <<EOF
|
||||
[Unit]
|
||||
Description=Steward host agent
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=$AGENT_USER
|
||||
ExecStart=/usr/bin/python3 $AGENT_DIR/agent.py
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
ReadOnlyPaths=/proc /sys
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now steward-agent.service
|
||||
|
||||
echo
|
||||
echo "Steward host agent $AGENT_VERSION installed and running."
|
||||
echo "Check status: systemctl status steward-agent"
|
||||
echo "Logs: journalctl -u steward-agent -f"
|
||||
@@ -0,0 +1,146 @@
|
||||
{# Per-host agent panel — embedded into /hosts/<id> via HTMX. Self-contained. #}
|
||||
{% set scope = "steward:target:" ~ target.id if target else "" %}
|
||||
<div class="card">
|
||||
<div style="display:flex;align-items:baseline;justify-content:space-between;margin-bottom:0.75rem;gap:1rem;flex-wrap:wrap;">
|
||||
<h3 class="section-title" style="margin-bottom:0;">Agent</h3>
|
||||
{% if reporting %}
|
||||
<span style="font-size:0.78rem;" class="tone-{{ 'warn' if stale else 'ok' }}">
|
||||
{{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %}
|
||||
· last seen {{ reg.last_seen_at|ago }}
|
||||
</span>
|
||||
{% elif reg %}
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);">pending — no check-in yet</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% if reporting %}
|
||||
{# Reporting: live vitals are shown by the vitals strip above; this panel is
|
||||
lifecycle/management only. #}
|
||||
<div style="display:flex;gap:0.5rem;flex-wrap:wrap;align-items:center;">
|
||||
<a href="/plugins/host_agent/{{ host.id }}/" class="btn btn-sm btn-ghost">Full metrics →</a>
|
||||
{% if session.user_role == 'admin' %}
|
||||
{% if ansible_available and target %}
|
||||
<form method="post" action="/plugins/host_agent/update" style="margin:0;">{{ csrf_field() }}
|
||||
<input type="hidden" name="inventory_scope" value="{{ scope }}">
|
||||
<button type="submit" class="btn btn-sm" title="Refresh agent.py + restart (token preserved)">Update agent</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/rotate-token" style="margin:0;"
|
||||
data-confirm="Rotate token? The agent stops reporting until reinstalled/updated with the new token.">{{ csrf_field() }}
|
||||
<button type="submit" class="btn btn-sm btn-ghost">Rotate token</button>
|
||||
</form>
|
||||
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0;"
|
||||
data-confirm="Remove this agent registration? Metrics stop until re-registered.">{{ csrf_field() }}
|
||||
<button type="submit" class="btn btn-sm btn-danger">Remove</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if ansible_available and not target %}
|
||||
<p style="color:var(--text-dim);font-size:0.8rem;margin:0.6rem 0 0;">
|
||||
Link an Ansible target (above) to enable one-click agent updates.
|
||||
</p>
|
||||
{% endif %}
|
||||
|
||||
{% if session.user_role == 'admin' and ansible_available and target and managed_key_set %}
|
||||
<details style="margin-top:0.7rem;">
|
||||
<summary style="cursor:pointer;font-size:0.8rem;color:var(--text-muted);">Re-provision (reinstall the steward account + managed key, then the agent)</summary>
|
||||
<p style="font-size:0.78rem;color:var(--text-dim);margin:0.5rem 0;">
|
||||
Use after regenerating the managed key, or if SSH auth as <code>steward</code> breaks.
|
||||
Connects over a one-time bootstrap user + password (not stored).
|
||||
</p>
|
||||
<form method="post" action="/plugins/host_agent/provision"
|
||||
style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
|
||||
<input type="hidden" name="inventory_scope" value="{{ scope }}">
|
||||
<div class="form-group" style="margin:0;">
|
||||
<label>Bootstrap user</label>
|
||||
<input type="text" name="bootstrap_user" required placeholder="root" style="width:8rem;" autocomplete="off">
|
||||
</div>
|
||||
<div class="form-group" style="margin:0;">
|
||||
<label>Bootstrap password</label>
|
||||
<input type="password" name="bootstrap_password" required style="width:10rem;" autocomplete="new-password">
|
||||
</div>
|
||||
<button type="submit" class="btn btn-sm">Re-provision</button>
|
||||
</form>
|
||||
</details>
|
||||
{% endif %}
|
||||
|
||||
{% else %}
|
||||
{# ── Not reporting: pending (token minted, no check-in) or never installed ── #}
|
||||
{% if reg %}
|
||||
<div style="background:color-mix(in srgb,var(--yellow) 10%,var(--bg-elevated));
|
||||
border:1px solid color-mix(in srgb,var(--yellow) 30%,var(--border));
|
||||
border-radius:6px;padding:0.7rem 0.9rem;margin-bottom:0.75rem;font-size:0.84rem;">
|
||||
A token was minted for this host but <strong>no metrics have arrived yet</strong> — the
|
||||
deploy may still be running, or it failed. Open the latest
|
||||
<a href="/ansible/">Ansible run</a> to check, then retry below. Live metrics appear here
|
||||
once the agent checks in.
|
||||
</div>
|
||||
{% else %}
|
||||
<p style="color:var(--text-muted);font-size:0.88rem;margin-bottom:0.75rem;">
|
||||
No agent installed. The agent reports CPU, memory, disk, network and more back to Steward.
|
||||
</p>
|
||||
{% endif %}
|
||||
|
||||
{% if session.user_role != 'admin' %}
|
||||
<p style="color:var(--text-dim);font-size:0.85rem;">An admin can install the agent here.</p>
|
||||
|
||||
{% elif not ansible_available %}
|
||||
<p style="color:var(--text-dim);font-size:0.85rem;">
|
||||
Ansible isn't available, so the agent can't be deployed from here. Install it manually with the
|
||||
<a href="/plugins/host_agent/fleet/">curl install command</a>.
|
||||
</p>
|
||||
|
||||
{% elif not managed_key_set %}
|
||||
<div style="background:color-mix(in srgb,var(--yellow) 12%,var(--bg-elevated));
|
||||
border:1px solid color-mix(in srgb,var(--yellow) 35%,var(--border));
|
||||
border-radius:6px;padding:0.7rem 0.9rem;display:flex;align-items:center;gap:0.8rem;flex-wrap:wrap;">
|
||||
<span style="color:var(--yellow);">⚠</span>
|
||||
<span style="font-size:0.84rem;flex:1;min-width:13rem;">No managed SSH key yet — Steward needs one to log into hosts.</span>
|
||||
<form method="post" action="/settings/ansible/generate-key" style="margin:0;">{{ csrf_field() }}
|
||||
<input type="hidden" name="next" value="/hosts/{{ host.id }}">
|
||||
<button type="submit" class="btn btn-sm">Generate managed key</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
{% elif not target %}
|
||||
<p style="color:var(--text-dim);font-size:0.85rem;">
|
||||
Link or create an Ansible target for this host (in the Ansible section below) first — provisioning
|
||||
needs an SSH connection.
|
||||
</p>
|
||||
|
||||
{% else %}
|
||||
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.75rem;">
|
||||
Deploys the agent to <code>{{ target.name }}</code> by running an Ansible playbook. Steward mints
|
||||
the host's API token automatically; you'll watch the run live.
|
||||
</p>
|
||||
{# Provision: brand-new host (creates steward account + key over a one-time password) #}
|
||||
<form method="post" action="/plugins/host_agent/provision"
|
||||
style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;margin-bottom:0.75rem;">{{ csrf_field() }}
|
||||
<input type="hidden" name="inventory_scope" value="{{ scope }}">
|
||||
<div style="font-size:0.78rem;color:var(--text-muted);width:100%;">Provision (first contact — fresh host):</div>
|
||||
<div class="form-group" style="margin:0;">
|
||||
<label>Bootstrap user</label>
|
||||
<input type="text" name="bootstrap_user" required placeholder="root" style="width:8rem;" autocomplete="off">
|
||||
</div>
|
||||
<div class="form-group" style="margin:0;">
|
||||
<label>Bootstrap password</label>
|
||||
<input type="password" name="bootstrap_password" required style="width:10rem;" autocomplete="new-password">
|
||||
</div>
|
||||
<button type="submit" class="btn btn-sm">Provision</button>
|
||||
</form>
|
||||
{# Install: host already has the steward account (managed key works) #}
|
||||
<form method="post" action="/plugins/host_agent/deploy" style="display:flex;gap:0.6rem;align-items:center;">{{ csrf_field() }}
|
||||
<input type="hidden" name="inventory_scope" value="{{ scope }}">
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);">Already provisioned?</span>
|
||||
<button type="submit" class="btn btn-sm btn-ghost">Install agent (managed key)</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
|
||||
{% if reg and session.user_role == 'admin' %}
|
||||
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0.75rem 0 0;"
|
||||
data-confirm="Clear this pending registration? Its token is discarded.">{{ csrf_field() }}
|
||||
<button type="submit" class="btn btn-sm btn-ghost">Clear pending registration</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</div>
|
||||
@@ -0,0 +1,179 @@
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import crumbs, status_badge %}
|
||||
{% block title %}Agent fleet — Steward{% endblock %}
|
||||
{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), ("Agent fleet", "")]) }}{% endblock %}
|
||||
{% block content %}
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1rem;gap:1rem;flex-wrap:wrap;">
|
||||
<h1 class="page-title" style="margin-bottom:0;">Agent fleet</h1>
|
||||
</div>
|
||||
<p style="color:var(--text-muted);font-size:0.84rem;margin:-0.4rem 0 1.25rem;">
|
||||
Bulk agent operations across your inventory. For a single host, use its
|
||||
<a href="/hosts/">host page</a> — provisioning and metrics live there.
|
||||
</p>
|
||||
|
||||
{% if new_token %}
|
||||
<div class="card" style="background:var(--fs-accent-faint);">
|
||||
<h3>New token — copy this install command now</h3>
|
||||
<p style="font-size:0.82rem;color:var(--text-muted);">
|
||||
This is the only time the raw token is shown. Rotate if you lose it.
|
||||
</p>
|
||||
<pre style="user-select:all;word-break:break-all;">curl -sSL '{{ install_url }}' | sudo sh</pre>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="card-flush" style="margin-bottom:1.5rem;">
|
||||
<div class="table-wrap">
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Status</th><th>Host</th><th>Last seen</th><th>Agent version</th><th>Reported IP</th><th>Distro</th>
|
||||
<th class="td-actions">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for item in registrations %}
|
||||
<tr>
|
||||
<td>{{ status_badge(item.state, {'up': 'Reporting', 'stale': 'Silent', 'pending': 'Never reported'}[item.state]) }}</td>
|
||||
<td>{% if item.host %}<a href="/hosts/{{ item.host.id }}">{{ item.host.name }}</a>{% else %}<span class="mono">{{ item.reg.host_id }}</span>{% endif %}</td>
|
||||
<td>{% if item.reg.last_seen_at %}<span title="{{ item.reg.last_seen_at.strftime('%Y-%m-%d %H:%M:%S UTC') }}">{{ item.reg.last_seen_at|ago }}</span>{% else %}never{% endif %}</td>
|
||||
<td>{{ item.reg.agent_version or "—" }}</td>
|
||||
<td class="mono">{{ item.reg.host_ip or "—" }}</td>
|
||||
<td>{{ item.reg.distro or "—" }}</td>
|
||||
<td class="td-actions">
|
||||
<form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/rotate-token"
|
||||
style="display:inline;"
|
||||
data-confirm="Rotate the token for {{ item.name }}? The existing agent will stop working until reinstalled with the new token.">{{ csrf_field() }}
|
||||
<button type="submit" class="btn btn-ghost btn-sm">Rotate token</button>
|
||||
</form>
|
||||
<form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/delete"
|
||||
style="display:inline;"
|
||||
data-confirm="Delete the agent registration for {{ item.name }}? The agent will be unable to report metrics until re-registered.">{{ csrf_field() }}
|
||||
<button type="submit" class="btn btn-danger btn-sm">Delete</button>
|
||||
</form>
|
||||
</td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr><td colspan="7" class="empty">No hosts registered. Add one below.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<form method="post" action="/plugins/host_agent/fleet/add-host"
|
||||
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
|
||||
<div class="form-group" style="margin-bottom:0;">
|
||||
<label>Host name</label>
|
||||
<input type="text" name="name" required>
|
||||
</div>
|
||||
<div class="form-group" style="margin-bottom:0;">
|
||||
<label>Address (optional)</label>
|
||||
<input type="text" name="address">
|
||||
</div>
|
||||
<button type="submit" class="btn">Add host</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
{% if ansible_available %}
|
||||
{% set scope_select %}
|
||||
<label>Target</label>
|
||||
<select name="inventory_scope" required>
|
||||
{% for g in deploy_groups %}<option value="steward:group:{{ g.id }}">Group: {{ g.name }}</option>{% endfor %}
|
||||
{% for t in deploy_targets %}<option value="steward:target:{{ t.id }}">Target: {{ t.name }}</option>{% endfor %}
|
||||
</select>
|
||||
{% endset %}
|
||||
|
||||
<div class="card">
|
||||
<h3 style="margin-bottom:0.4rem;">Agent lifecycle via Ansible</h3>
|
||||
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.6rem;">
|
||||
These actions run bundled <strong>Ansible playbooks</strong> to deploy and maintain the
|
||||
Steward monitoring agent on your <a href="/ansible/inventory/targets">inventory hosts</a>.
|
||||
Steward generates each host's API token automatically and connects over SSH as the managed
|
||||
<code>steward</code> account — you'll be dropped into the live Ansible run to watch it.
|
||||
</p>
|
||||
|
||||
{% if not managed_key_set %}
|
||||
<div style="background:color-mix(in srgb,var(--yellow) 12%,var(--bg-elevated));
|
||||
border:1px solid color-mix(in srgb,var(--yellow) 35%,var(--border));
|
||||
border-radius:6px;padding:0.7rem 0.9rem;display:flex;align-items:center;
|
||||
gap:0.8rem;flex-wrap:wrap;">
|
||||
<span style="color:var(--yellow);">⚠</span>
|
||||
<span style="font-size:0.84rem;flex:1;min-width:14rem;">
|
||||
No managed SSH key yet — Steward needs one to log into hosts as <code>steward</code>.
|
||||
</span>
|
||||
<form method="post" action="/settings/ansible/generate-key" style="margin:0;">{{ csrf_field() }}
|
||||
<input type="hidden" name="next" value="/plugins/host_agent/fleet/">
|
||||
<button type="submit" class="btn btn-sm">Generate managed key</button>
|
||||
</form>
|
||||
</div>
|
||||
{% elif not (deploy_targets or deploy_groups) %}
|
||||
<p style="font-size:0.85rem;color:var(--text-dim);margin:0;">
|
||||
No Ansible inventory targets yet. Add some under <a href="/ansible/inventory/targets">Ansible → Inventory</a>.
|
||||
</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% if managed_key_set and (deploy_targets or deploy_groups) %}
|
||||
<div class="card">
|
||||
<h3 style="margin-bottom:0.4rem;">1 · Provision a fresh host</h3>
|
||||
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.75rem;">
|
||||
First contact for a brand-new host. Connects with a one-time <strong>bootstrap</strong>
|
||||
user + password (used for this run only, never stored), creates the <code>steward</code>
|
||||
login account with passwordless sudo, installs the managed SSH key, then installs the
|
||||
agent. Run this once per host.
|
||||
</p>
|
||||
<form method="post" action="/plugins/host_agent/provision"
|
||||
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
|
||||
<div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
|
||||
<div class="form-group" style="margin-bottom:0;">
|
||||
<label>Bootstrap user</label>
|
||||
<input type="text" name="bootstrap_user" required placeholder="root" style="width:9rem;" autocomplete="off">
|
||||
</div>
|
||||
<div class="form-group" style="margin-bottom:0;">
|
||||
<label>Bootstrap password</label>
|
||||
<input type="password" name="bootstrap_password" required style="width:11rem;" autocomplete="new-password">
|
||||
</div>
|
||||
<div class="form-group" style="margin-bottom:0;">
|
||||
<label>Report interval (s)</label>
|
||||
<input type="number" name="agent_interval" value="30" min="5" style="width:7rem;">
|
||||
</div>
|
||||
<button type="submit" class="btn">Provision</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h3 style="margin-bottom:0.4rem;">2 · Install / enroll agent</h3>
|
||||
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.75rem;">
|
||||
For a host that's already provisioned (has the <code>steward</code> account) but has no
|
||||
agent yet — or to re-enroll one. Connects as <code>steward</code> with the managed key,
|
||||
mints a fresh token, and installs the agent. No <code>curl | sh</code> needed.
|
||||
</p>
|
||||
<form method="post" action="/plugins/host_agent/deploy"
|
||||
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
|
||||
<div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
|
||||
<div class="form-group" style="margin-bottom:0;">
|
||||
<label>Report interval (s)</label>
|
||||
<input type="number" name="agent_interval" value="30" min="5" style="width:7rem;">
|
||||
</div>
|
||||
<button type="submit" class="btn">Install</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h3 style="margin-bottom:0.4rem;">3 · Update agents</h3>
|
||||
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.75rem;">
|
||||
Refresh the agent binary on hosts already running it. Connects as <code>steward</code>
|
||||
with the managed key and restarts the service — the token and config are left untouched,
|
||||
so identity is preserved.
|
||||
</p>
|
||||
<form method="post" action="/plugins/host_agent/update"
|
||||
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
|
||||
<div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
|
||||
<button type="submit" class="btn">Update</button>
|
||||
</form>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,90 @@
|
||||
{# host_agent history-graph widget — the host-view utilization chart, embedded
|
||||
on a dashboard. Fills the (resizable) panel; epoch-ms linear axis so no
|
||||
Chart.js date adapter is needed. #}
|
||||
{% set has_data = host and (series.cpu_pct or series.mem_used_pct or series.disk_root) %}
|
||||
{% if not host %}
|
||||
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.75rem 0;">
|
||||
No host selected. <strong>Edit</strong> this dashboard and pick a host for this graph.
|
||||
</div>
|
||||
{% elif not has_data %}
|
||||
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.75rem 0;">
|
||||
No agent metrics for <strong>{{ host.name }}</strong> in the last {{ hours }}h yet.
|
||||
</div>
|
||||
{% else %}
|
||||
<div style="display:flex;flex-direction:column;height:100%;min-height:200px;">
|
||||
{# Name the host the graph represents (panel title is generic) + a live range
|
||||
toggle that re-requests this widget without entering edit mode. #}
|
||||
<div style="flex:0 0 auto;display:flex;align-items:center;gap:0.5rem;flex-wrap:wrap;font-size:0.75rem;color:var(--text-muted);margin-bottom:0.3rem;">
|
||||
{% if session.user_id %}
|
||||
<a href="/hosts/{{ host.id }}" style="font-weight:500;">{{ host.name }}</a>
|
||||
{% else %}
|
||||
<strong style="color:var(--text);">{{ host.name }}</strong>
|
||||
{% endif %}
|
||||
<span class="hist-range" style="margin-left:auto;display:inline-flex;gap:0.25rem;">
|
||||
{% for opt in [1, 6, 24] %}
|
||||
<button type="button" onclick="histRange({{ wid|int }}, {{ opt|int }})"
|
||||
class="range-btn{% if opt == hours %} active{% endif %}">{{ opt }}h</button>
|
||||
{% endfor %}
|
||||
</span>
|
||||
</div>
|
||||
<div style="flex:1 1 auto;min-height:150px;position:relative;">
|
||||
<canvas id="host-hist-{{ wid }}"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
// Live time-range switch. Rewrites the parent cell's hx-get so the choice sticks
|
||||
// across polls (htmx re-reads the attribute each poll), then fetches immediately.
|
||||
// Only `hours` changes: the rest of the query (host, widget id, a share link's
|
||||
// token) is kept, so the switch also works on a shared dashboard.
|
||||
window.histRange = function (wid, hours) {
|
||||
var cell = document.getElementById("widget-" + wid);
|
||||
if (!cell) return;
|
||||
var parts = (cell.getAttribute("hx-get") || "").split("?");
|
||||
var query = new URLSearchParams(parts[1] || "");
|
||||
query.set("hours", hours);
|
||||
var url = parts[0] + "?" + query.toString();
|
||||
cell.setAttribute("hx-get", url);
|
||||
if (window.htmx) window.htmx.ajax("GET", url, { target: cell, swap: "innerHTML" });
|
||||
};
|
||||
(function () {
|
||||
var el = document.getElementById("host-hist-{{ wid }}");
|
||||
if (!el) return;
|
||||
var series = {{ series|tojson }};
|
||||
var fmtTime = function (v) {
|
||||
var d = new Date(v);
|
||||
return ("0" + d.getHours()).slice(-2) + ":" + ("0" + d.getMinutes()).slice(-2);
|
||||
};
|
||||
var ds = [
|
||||
{ key: "cpu_pct", label: "CPU %", color: stewardColor("--gold") },
|
||||
{ key: "mem_used_pct", label: "Mem %", color: stewardColor("--green") },
|
||||
{ key: "disk_root", label: "Disk / %", color: stewardColor("--yellow") },
|
||||
];
|
||||
new Chart(el.getContext("2d"), {
|
||||
type: "line",
|
||||
data: { datasets: ds.map(function (d) {
|
||||
return {
|
||||
label: d.label,
|
||||
data: (series[d.key] || []).map(function (p) { return { x: p[0], y: p[1] }; }),
|
||||
borderColor: d.color, backgroundColor: d.color, borderWidth: 1.5, tension: 0.25,
|
||||
};
|
||||
}) },
|
||||
options: {
|
||||
responsive: true, maintainAspectRatio: false,
|
||||
interaction: { mode: "index", intersect: false },
|
||||
elements: { point: { radius: 0 } },
|
||||
scales: {
|
||||
x: { type: "linear", ticks: { callback: function (v) { return fmtTime(v); }, maxTicksLimit: 6, font: { size: 10 } } },
|
||||
// Fixed 0–{{ y_max }} ceiling (server snaps the data peak up to the next
|
||||
// 10% band). Steady across refreshes — auto-scaling made the zoom jump on
|
||||
// every poll — while a banded ceiling still keeps the lines filling the panel.
|
||||
y: { min: 0, max: {{ y_max }}, ticks: { font: { size: 10 } } },
|
||||
},
|
||||
plugins: {
|
||||
legend: { labels: { boxWidth: 10, font: { size: 11 } } },
|
||||
tooltip: { callbacks: { title: function (items) { return items.length ? fmtTime(items[0].parsed.x) : ""; } } },
|
||||
},
|
||||
},
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,69 @@
|
||||
{# host_agent widget — fleet glance. One horizontal row per host, zebra-striped
|
||||
so adjacent hosts read as distinct groupings: health dot + host name on the
|
||||
left, metric cells (cpu/mem/disk/load) each with a trend sparkline laid out
|
||||
horizontally on the right. The name wraps (never hard-truncated) but the row
|
||||
stays horizontal. #}
|
||||
{% if rows %}
|
||||
{% macro metric_cell(label, value, fmt, svg, style="", title="") %}
|
||||
<div style="min-width:62px;flex:0 0 auto;" title="{{ title }}">
|
||||
<div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
|
||||
<div style="font-weight:500;font-size:0.8rem;font-variant-numeric:tabular-nums;{{ style }}">
|
||||
{% if value is not none %}{{ fmt|format(value) }}{% else %}—{% endif %}
|
||||
</div>
|
||||
<div style="line-height:0;height:16px;">{{ svg | safe }}</div>
|
||||
</div>
|
||||
{% endmacro %}
|
||||
{# Two-line down/up (or read/write) throughput cell. #}
|
||||
{% macro io_cell(label, down_sym, down_val, up_sym, up_val, title="") %}
|
||||
<div style="min-width:80px;flex:0 0 auto;" title="{{ title }}">
|
||||
<div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
|
||||
<div style="font-size:0.72rem;font-variant-numeric:tabular-nums;line-height:1.3;">
|
||||
<div>{{ down_sym }} {{ down_val|rate }}</div>
|
||||
<div>{{ up_sym }} {{ up_val|rate }}</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endmacro %}
|
||||
<div class="host-fleet">
|
||||
{% for r in rows %}
|
||||
{% set stale = r.stale %}
|
||||
<div style="display:flex;align-items:center;gap:0.6rem 1rem;flex-wrap:wrap;padding:0.5rem 0.6rem;border-radius:4px;{% if loop.index0 % 2 %}background:var(--bg-elevated);{% endif %}">
|
||||
{# Left — health dot + full host name (wraps, never truncated) + last seen #}
|
||||
<div style="display:flex;align-items:center;gap:0.5rem;flex:1 1 150px;min-width:120px;">
|
||||
<span class="dot {% if stale %}dot-dim{% elif (r.cpu_pct or 0) >= 90 or (r.mem_used_pct or 0) >= 90 or (r.disk_worst or 0) >= 90 %}dot-warn{% else %}dot-up{% endif %}"
|
||||
style="flex-shrink:0;"
|
||||
title="{% if stale %}stale / no recent data{% else %}warns at CPU/memory ≥90% or any disk mount ≥90% (the 'disk /' figure shows root only){% endif %}"></span>
|
||||
<span style="min-width:0;">
|
||||
{% if session.user_id %}
|
||||
<a href="/hosts/{{ r.host.id }}" style="font-weight:500;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</a>
|
||||
{% else %}
|
||||
<span style="font-weight:500;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</span>
|
||||
{% endif %}
|
||||
<span style="display:block;font-size:0.7rem;color:var(--text-dim);">
|
||||
{% if r.reg.last_seen_at %}seen {{ r.reg.last_seen_at|ago }}{% else %}never seen{% endif %}
|
||||
</span>
|
||||
</span>
|
||||
</div>
|
||||
{# Right — metric cells with value + trend, laid out horizontally #}
|
||||
<div style="display:flex;flex-wrap:wrap;gap:0.4rem 1rem;justify-content:flex-end;">
|
||||
{{ metric_cell("cpu", r.cpu_pct, "%.0f%%", r.sparks.cpu, threshold_style(r.cpu_pct, 'cpu'), "Average CPU utilization") }}
|
||||
{{ metric_cell("mem", r.mem_used_pct, "%.0f%%", r.sparks.mem, threshold_style(r.mem_used_pct, 'mem'), "Memory in use") }}
|
||||
{{ metric_cell("disk /", r.disk_root, "%.0f%%", r.sparks.disk, threshold_style(r.disk_root, 'disk'), "Root filesystem (/) usage") }}
|
||||
{{ metric_cell("load", r.load_1m, "%.2f", r.sparks.load, "", "Load average (1m)") }}
|
||||
{# Extra agent metrics — shown only when the host reports them (VMs/containers
|
||||
often have no temp sensor, etc.) so absent data doesn't clutter the row. #}
|
||||
{% if r.net_rx_bps is not none or r.net_tx_bps is not none %}
|
||||
{{ io_cell("net", "↓", r.net_rx_bps, "↑", r.net_tx_bps, "Network throughput (down / up)") }}
|
||||
{% endif %}
|
||||
{% if r.disk_read_bps is not none or r.disk_write_bps is not none %}
|
||||
{{ io_cell("disk i/o", "rd", r.disk_read_bps, "wr", r.disk_write_bps, "Disk I/O (read / write)") }}
|
||||
{% endif %}
|
||||
{% if r.temp_max is not none %}
|
||||
{{ metric_cell("temp", r.temp_max, "%.0f°C", "", threshold_style(r.temp_max, 'temp'), "Hottest sensor") }}
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% else %}
|
||||
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No hosts with agent data yet.</p>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,74 @@
|
||||
# steward-plugins / index.yaml
|
||||
#
|
||||
# Plugin catalog for Steward.
|
||||
# Fetched by the app at: Settings → Plugins → Plugin Catalog
|
||||
#
|
||||
# After updating an entry, commit and push — changes are live within 5 minutes
|
||||
# (the app caches this file in-process for 300 seconds).
|
||||
#
|
||||
# See docs/plugins/index.yaml.example in the main app repo for the full schema.
|
||||
|
||||
version: 1
|
||||
updated: "2026-03-22"
|
||||
|
||||
plugins:
|
||||
|
||||
- name: docker
|
||||
version: "2.0.0"
|
||||
description: "Per-host Docker container status + resource usage, collected by the Steward host agent"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v2.0.0/docker.zip"
|
||||
checksum_sha256: ""
|
||||
tags:
|
||||
- containers
|
||||
- docker
|
||||
- infrastructure
|
||||
|
||||
- name: traefik
|
||||
version: "1.0.0"
|
||||
description: "Traefik reverse proxy metrics and access log integration"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip"
|
||||
checksum_sha256: ""
|
||||
tags:
|
||||
- proxy
|
||||
- metrics
|
||||
- access-log
|
||||
|
||||
- name: unifi
|
||||
version: "1.0.0"
|
||||
description: "UniFi Network controller integration — WAN health, devices, clients, DPI"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/unifi-v1.0.0/unifi.zip"
|
||||
checksum_sha256: ""
|
||||
tags:
|
||||
- network
|
||||
- unifi
|
||||
- ubiquiti
|
||||
|
||||
- name: ups
|
||||
version: "1.0.0"
|
||||
description: "UPS monitoring via NUT (Network UPS Tools) with Ansible shutdown automation"
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/ups"
|
||||
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/ups-v1.0.0/ups.zip"
|
||||
checksum_sha256: ""
|
||||
tags:
|
||||
- ups
|
||||
- power
|
||||
- nut
|
||||
@@ -0,0 +1,34 @@
|
||||
# plugins/snmp/__init__.py
|
||||
from __future__ import annotations
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from quart import Quart
|
||||
|
||||
_app: "Quart | None" = None
|
||||
|
||||
|
||||
def setup(app: "Quart") -> None:
|
||||
global _app
|
||||
_app = app
|
||||
|
||||
|
||||
def get_scheduled_tasks() -> list:
|
||||
from .scheduler import make_poll_task
|
||||
return [make_poll_task(_app)]
|
||||
|
||||
|
||||
def get_blueprint():
|
||||
from .routes import snmp_bp
|
||||
return snmp_bp
|
||||
|
||||
|
||||
def get_nav() -> list[dict]:
|
||||
# Surfaces in the sidebar's Infrastructure group (SNMP device readings).
|
||||
return [{"label": "SNMP", "href": "/plugins/snmp/"}]
|
||||
|
||||
|
||||
def get_alert_metrics() -> dict[str, list[str]]:
|
||||
"""SNMP metric names are the OID labels each device is configured with, so
|
||||
the list is empty here and the rule form offers the labels recorded so far."""
|
||||
return {"snmp": []}
|
||||
@@ -0,0 +1,59 @@
|
||||
# plugins/snmp/plugin.yaml
|
||||
name: snmp
|
||||
version: "1.0.0"
|
||||
description: "SNMP polling for network devices — switches, routers, printers, UPSes, etc."
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
|
||||
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/snmp"
|
||||
tags:
|
||||
- snmp
|
||||
- network
|
||||
- monitoring
|
||||
|
||||
config:
|
||||
poll_interval_seconds: 60
|
||||
# Each device is polled independently. OIDs must resolve to numeric values.
|
||||
#
|
||||
# `host` is the SNMP poll target (IP/hostname we query). A device also shows
|
||||
# up on a Steward Host's detail page when it maps to that host. By default
|
||||
# that mapping is implicit: `host` is matched against the Host's address or
|
||||
# name. To bind explicitly instead — e.g. when you poll by IP but the Host is
|
||||
# recorded by DNS name, or for an SNMP-only switch/PDU/UPS — add ONE of:
|
||||
# host_id: "<steward-host-uuid>" # exact Host.id match
|
||||
# steward_host: "switch01" # Host name OR address (case-insensitive)
|
||||
# An explicit binding is exclusive: the device then maps ONLY to that host,
|
||||
# never implicitly to another by a coincidental address string.
|
||||
devices:
|
||||
- name: "core-switch"
|
||||
host: "192.168.1.1"
|
||||
port: 161
|
||||
community: "public"
|
||||
version: "2c" # "1", "2c", or "3"
|
||||
oids:
|
||||
- oid: "1.3.6.1.2.1.1.3.0"
|
||||
label: "uptime_centisecs"
|
||||
- oid: "1.3.6.1.2.1.2.2.1.10.1"
|
||||
label: "if1_in_octets"
|
||||
- oid: "1.3.6.1.2.1.2.2.1.16.1"
|
||||
label: "if1_out_octets"
|
||||
# Example: network UPS with SNMP management card (RFC 1628 UPS-MIB).
|
||||
# upsBatteryStatus values: 1=unknown, 2=batteryNormal, 3=batteryLow, 4=batteryDepleted.
|
||||
# Uncomment and point at your UPS to enable.
|
||||
# - name: "ups"
|
||||
# host: "192.168.1.10"
|
||||
# port: 161
|
||||
# community: "public"
|
||||
# version: "2c"
|
||||
# oids:
|
||||
# - oid: "1.3.6.1.2.1.33.1.2.1.0"
|
||||
# label: "battery_status"
|
||||
# - oid: "1.3.6.1.2.1.33.1.2.2.0"
|
||||
# label: "seconds_on_battery"
|
||||
# - oid: "1.3.6.1.2.1.33.1.2.3.0"
|
||||
# label: "runtime_minutes_remaining"
|
||||
# - oid: "1.3.6.1.2.1.33.1.2.4.0"
|
||||
# label: "battery_charge_pct"
|
||||
# - oid: "1.3.6.1.2.1.33.1.3.3.1.3.1"
|
||||
# label: "input_voltage_dV"
|
||||
@@ -0,0 +1,171 @@
|
||||
# plugins/snmp/poller.py
|
||||
"""
|
||||
Asynchronous SNMP GET helper.
|
||||
|
||||
Requires pysnmp-lextudio (the maintained pysnmp fork), bundled into the Docker
|
||||
image via the `snmp` extra (`pip install .[snmp]`):
|
||||
pip install 'steward[snmp]'
|
||||
|
||||
If pysnmp is not installed, poll_device() returns an empty dict and logs a
|
||||
warning — SNMP polling is then simply disabled, nothing else breaks.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _pysnmp_available() -> bool:
|
||||
try:
|
||||
import pysnmp # noqa: F401
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
def _mp_model(version: str) -> int:
|
||||
"""Map an SNMP version string to pysnmp's mpModel int (0 = v1, 1 = v2c)."""
|
||||
return 0 if version == "1" else 1
|
||||
|
||||
|
||||
def _close_engine(engine) -> None:
|
||||
"""Release the engine's UDP transport socket.
|
||||
|
||||
pysnmp opens a UDP socket per ``SnmpEngine`` and never closes it on its own.
|
||||
Since we build a fresh engine for every poll, an unclosed engine leaks one
|
||||
file descriptor each scheduler tick; over hours of polling that exhausts the
|
||||
process fd limit (``OSError: [Errno 24] Too many open files``), which also
|
||||
takes down the app's listening socket. So close it explicitly here.
|
||||
|
||||
The close method/attribute names differ across pysnmp majors (6.2.x lextudio
|
||||
is camelCase, canonical 7.x is snake_case), so probe for whatever exists.
|
||||
All paths are best-effort — a failed close must never break the poll loop.
|
||||
"""
|
||||
# 7.x exposes a convenience close directly on the engine.
|
||||
for meth in ("close_dispatcher", "closeDispatcher"):
|
||||
fn = getattr(engine, meth, None)
|
||||
if callable(fn):
|
||||
try:
|
||||
fn()
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
# 6.2.x: go through the transport dispatcher.
|
||||
for attr in ("transport_dispatcher", "transportDispatcher"):
|
||||
dispatcher = getattr(engine, attr, None)
|
||||
if dispatcher is None:
|
||||
continue
|
||||
for meth in ("close_dispatcher", "closeDispatcher"):
|
||||
fn = getattr(dispatcher, meth, None)
|
||||
if callable(fn):
|
||||
try:
|
||||
fn()
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
return
|
||||
|
||||
|
||||
async def poll_device(
|
||||
host: str,
|
||||
port: int,
|
||||
community: str,
|
||||
version: str,
|
||||
oids: list[dict],
|
||||
) -> dict[str, float]:
|
||||
"""Perform an SNMP GET for each OID and return ``{label: float_value}``.
|
||||
|
||||
Non-numeric OIDs (strings, etc.) are skipped. Returns an empty dict on any
|
||||
error (unreachable host, wrong community, …) so a flaky device never breaks
|
||||
the poll loop.
|
||||
|
||||
pysnmp's HLAPI is asyncio-only as of v6. The import path moved between major
|
||||
versions, so we support both rather than let a dependency bump silently
|
||||
re-break polling:
|
||||
• pysnmp-lextudio 6.2.x → ``pysnmp.hlapi.asyncio`` (``getCmd`` + a directly
|
||||
constructed ``UdpTransportTarget``).
|
||||
• canonical pysnmp 7.x → ``pysnmp.hlapi.v3arch.asyncio`` (``get_cmd`` + the
|
||||
async ``UdpTransportTarget.create``).
|
||||
"""
|
||||
if not _pysnmp_available():
|
||||
logger.warning("pysnmp not installed — SNMP polling disabled. "
|
||||
"Install with: pip install 'steward[snmp]'")
|
||||
return {}
|
||||
|
||||
try:
|
||||
# canonical pysnmp 7.x
|
||||
from pysnmp.hlapi.v3arch.asyncio import (
|
||||
CommunityData,
|
||||
ContextData,
|
||||
ObjectIdentity,
|
||||
ObjectType,
|
||||
SnmpEngine,
|
||||
UdpTransportTarget,
|
||||
get_cmd as _get_cmd,
|
||||
)
|
||||
_transport_is_async = True
|
||||
except ImportError:
|
||||
# pysnmp-lextudio 6.2.x
|
||||
from pysnmp.hlapi.asyncio import (
|
||||
CommunityData,
|
||||
ContextData,
|
||||
ObjectIdentity,
|
||||
ObjectType,
|
||||
SnmpEngine,
|
||||
UdpTransportTarget,
|
||||
getCmd as _get_cmd,
|
||||
)
|
||||
_transport_is_async = False
|
||||
|
||||
engine = SnmpEngine()
|
||||
|
||||
# Always release the engine's UDP socket — see _close_engine for why.
|
||||
try:
|
||||
# Same host/port for every OID on this device, so build the transport once.
|
||||
try:
|
||||
if _transport_is_async:
|
||||
transport = await UdpTransportTarget.create((host, port), timeout=5, retries=1)
|
||||
else:
|
||||
transport = UdpTransportTarget((host, port), timeout=5, retries=1)
|
||||
except Exception as exc:
|
||||
logger.debug("SNMP transport setup failed for %s:%s: %s", host, port, exc)
|
||||
return {}
|
||||
|
||||
results: dict[str, float] = {}
|
||||
|
||||
for oid_cfg in oids:
|
||||
oid = oid_cfg["oid"]
|
||||
label = oid_cfg.get("label") or oid
|
||||
scale = float(oid_cfg.get("scale", 1.0))
|
||||
|
||||
try:
|
||||
error_indication, error_status, error_index, var_binds = await _get_cmd(
|
||||
engine,
|
||||
CommunityData(community, mpModel=_mp_model(version)),
|
||||
transport,
|
||||
ContextData(),
|
||||
ObjectType(ObjectIdentity(oid)),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("SNMP GET %s@%s OID %s failed: %s", host, port, oid, exc)
|
||||
continue
|
||||
|
||||
if error_indication:
|
||||
logger.debug("SNMP error %s@%s OID %s: %s", host, port, oid, error_indication)
|
||||
continue
|
||||
if error_status:
|
||||
logger.debug("SNMP status %s@%s OID %s: %s at %s",
|
||||
host, port, oid, error_status.prettyPrint(),
|
||||
error_index and var_binds[int(error_index) - 1][0] or "?")
|
||||
continue
|
||||
|
||||
for _, val in var_binds:
|
||||
try:
|
||||
results[label] = float(val) * scale
|
||||
except (TypeError, ValueError):
|
||||
# Non-numeric type (e.g. OctetString description) — skip.
|
||||
logger.debug("SNMP non-numeric value for %s label=%s: %r", oid, label, val)
|
||||
|
||||
return results
|
||||
finally:
|
||||
_close_engine(engine)
|
||||
@@ -0,0 +1,214 @@
|
||||
# plugins/snmp/routes.py
|
||||
from __future__ import annotations
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from quart import Blueprint, current_app, render_template, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.models.users import UserRole
|
||||
from steward.core.metric_series import latest_by_resource
|
||||
from steward.models.metrics import PluginMetric
|
||||
|
||||
snmp_bp = Blueprint("snmp", __name__, template_folder="templates")
|
||||
|
||||
|
||||
async def _latest_readings(db, device_names: list[str]) -> dict[str, dict]:
|
||||
"""{device_name: {"readings": {label: latest_value}, "at": newest reading's
|
||||
time}} for the devices that have any readings."""
|
||||
latest = await latest_by_resource(db, "snmp", device_names)
|
||||
return {name: {"readings": {m: r.value for m, r in metrics.items()},
|
||||
"at": max(r.recorded_at for r in metrics.values())}
|
||||
for name, metrics in latest.items() if metrics}
|
||||
|
||||
|
||||
# A device whose newest reading is older than this many poll intervals is not
|
||||
# shown as reachable: the poller has stopped getting answers from it.
|
||||
STALE_POLLS = 3
|
||||
|
||||
|
||||
def _device_state(entry: dict | None, poll_seconds: int, now: datetime) -> dict:
|
||||
"""What the page says about a device's readings: {"readings", "at",
|
||||
"reach"}, reach being "up" (answered within STALE_POLLS polls), "stale"
|
||||
(has readings, but none recent) or "pending" (never answered)."""
|
||||
if not entry:
|
||||
return {"readings": {}, "at": None, "reach": "pending"}
|
||||
fresh = (now - entry["at"]).total_seconds() <= STALE_POLLS * max(int(poll_seconds), 1)
|
||||
return {**entry, "reach": "up" if fresh else "stale"}
|
||||
|
||||
|
||||
def _poll_seconds() -> int:
|
||||
return int(current_app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60))
|
||||
|
||||
|
||||
async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]:
|
||||
"""Return {label: [(recorded_at, value), ...]} for a single device."""
|
||||
since = datetime.now(timezone.utc) - timedelta(hours=hours)
|
||||
result = await db.execute(
|
||||
select(PluginMetric)
|
||||
.where(
|
||||
PluginMetric.source_module == "snmp",
|
||||
PluginMetric.resource_name == device_name,
|
||||
PluginMetric.recorded_at >= since,
|
||||
)
|
||||
.order_by(PluginMetric.recorded_at)
|
||||
)
|
||||
rows = result.scalars().all()
|
||||
|
||||
out: dict[str, list] = {}
|
||||
for row in rows:
|
||||
out.setdefault(row.metric_name, []).append(
|
||||
(row.recorded_at.isoformat(), row.value)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@snmp_bp.get("/")
|
||||
@require_role(UserRole.viewer)
|
||||
async def index():
|
||||
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
|
||||
device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
latest = await _latest_readings(db, device_names)
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# Merge config + latest readings for the template
|
||||
devices = []
|
||||
for cfg in devices_cfg:
|
||||
name = cfg.get("name") or cfg.get("host", "?")
|
||||
devices.append({
|
||||
"name": name,
|
||||
"host": cfg.get("host", ""),
|
||||
"oids": cfg.get("oids", []),
|
||||
**_device_state(latest.get(name), _poll_seconds(), now),
|
||||
})
|
||||
|
||||
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
|
||||
return await render_template("snmp/index.html",
|
||||
devices=devices,
|
||||
poll_interval=poll_interval)
|
||||
|
||||
|
||||
def _device_binds_to_host(d: dict, keys: set[str], host_id: str) -> bool:
|
||||
"""Does config device `d` belong on the host identified by `keys`
|
||||
(lowercased {address, name}) / `host_id`?
|
||||
|
||||
A device may declare an **explicit** binding that decouples "which Steward
|
||||
host this belongs to" from "where to poll" (the `host` field):
|
||||
• `host_id` — the Steward Host UUID (exact match), the explicit link.
|
||||
• `steward_host` — a friendly bind by Host name or address (case-insensitive).
|
||||
An explicit binding is **exclusive**: a device bound to host A must not also
|
||||
match host B by a coincidental poll-target string. Only when no explicit
|
||||
binding is present do we fall back to the implicit `host`-string match
|
||||
(backward compatible with configs that only set `host`).
|
||||
"""
|
||||
explicit_id = str(d.get("host_id", "")).strip()
|
||||
explicit_name = str(d.get("steward_host", "")).strip().lower()
|
||||
if explicit_id or explicit_name:
|
||||
if explicit_id and host_id and explicit_id == host_id:
|
||||
return True
|
||||
return bool(explicit_name and explicit_name in keys)
|
||||
return str(d.get("host", "")).strip().lower() in keys
|
||||
|
||||
|
||||
def _devices_for_host(devices_cfg: list, address: str | None, name: str | None,
|
||||
host_id: str | None = None) -> list[dict]:
|
||||
"""SNMP devices that map onto a Steward host's page. Prefers an explicit
|
||||
per-device binding (`host_id` / `steward_host`); otherwise falls back to
|
||||
matching the device's `host` (poll target) against the host's address or
|
||||
name (case-insensitive). See `_device_binds_to_host` for the precedence."""
|
||||
keys = {(address or "").strip().lower(), (name or "").strip().lower()}
|
||||
keys.discard("")
|
||||
hid = (host_id or "").strip()
|
||||
return [
|
||||
d for d in devices_cfg
|
||||
if isinstance(d, dict) and _device_binds_to_host(d, keys, hid)
|
||||
]
|
||||
|
||||
|
||||
@snmp_bp.get("/host/<host_id>")
|
||||
@require_role(UserRole.viewer)
|
||||
async def host_panel(host_id: str):
|
||||
"""Per-host SNMP fragment for the Hosts hub. Surfaces any configured SNMP
|
||||
device that maps to this host (by address or name) with its latest readings.
|
||||
Renders nothing when no device maps, so hosts without SNMP carry no empty card.
|
||||
"""
|
||||
from steward.models.hosts import Host
|
||||
|
||||
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
host = (await db.execute(
|
||||
select(Host).where(Host.id == host_id))).scalar_one_or_none()
|
||||
if host is None:
|
||||
return ""
|
||||
matched = _devices_for_host(devices_cfg, host.address, host.name, host.id)
|
||||
if not matched:
|
||||
return ""
|
||||
names = [d.get("name") or d.get("host", "?") for d in matched]
|
||||
latest = await _latest_readings(db, names)
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
devices = [{
|
||||
"name": d.get("name") or d.get("host", "?"),
|
||||
"host": d.get("host", ""),
|
||||
"oids": d.get("oids", []),
|
||||
**_device_state(latest.get(d.get("name") or d.get("host", "?")), _poll_seconds(), now),
|
||||
"bound": bool(str(d.get("host_id", "")).strip()
|
||||
or str(d.get("steward_host", "")).strip()),
|
||||
} for d in matched]
|
||||
return await render_template("snmp/host_panel.html", devices=devices)
|
||||
|
||||
|
||||
@snmp_bp.get("/widget")
|
||||
@require_role(UserRole.viewer)
|
||||
async def widget():
|
||||
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
|
||||
device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
latest = await _latest_readings(db, device_names[:10])
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
devices = []
|
||||
for cfg in devices_cfg[:10]:
|
||||
name = cfg.get("name") or cfg.get("host", "?")
|
||||
devices.append({
|
||||
"name": name,
|
||||
"oids": cfg.get("oids", []),
|
||||
**_device_state(latest.get(name), _poll_seconds(), now),
|
||||
})
|
||||
|
||||
return await render_template("snmp/widget.html",
|
||||
devices=devices,
|
||||
total=len(devices_cfg))
|
||||
|
||||
|
||||
@snmp_bp.get("/device/<device_name>")
|
||||
@require_role(UserRole.viewer)
|
||||
async def device_detail(device_name: str):
|
||||
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
|
||||
device_cfg = next(
|
||||
(d for d in devices_cfg if (d.get("name") or d.get("host")) == device_name),
|
||||
None,
|
||||
)
|
||||
if device_cfg is None:
|
||||
from quart import abort
|
||||
abort(404)
|
||||
|
||||
hours = int(request.args.get("hours", 24))
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
hist = await _history(db, device_name, hours=hours)
|
||||
|
||||
import json
|
||||
history_json = json.dumps(hist)
|
||||
oid_labels = [o.get("label") or o.get("oid") for o in device_cfg.get("oids", [])]
|
||||
|
||||
return await render_template(
|
||||
"snmp/device.html",
|
||||
device=device_cfg,
|
||||
device_name=device_name,
|
||||
oid_labels=oid_labels,
|
||||
history_json=history_json,
|
||||
hours=hours,
|
||||
)
|
||||
@@ -0,0 +1,70 @@
|
||||
# plugins/snmp/scheduler.py
|
||||
from __future__ import annotations
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from quart import Quart
|
||||
|
||||
from steward.core.scheduler import ScheduledTask
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def make_poll_task(app: "Quart") -> ScheduledTask:
|
||||
interval = app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60)
|
||||
|
||||
async def poll() -> None:
|
||||
await _do_poll(app)
|
||||
|
||||
return ScheduledTask(
|
||||
name="snmp_poll",
|
||||
coro_factory=poll,
|
||||
interval_seconds=int(interval),
|
||||
run_on_startup=True,
|
||||
)
|
||||
|
||||
|
||||
async def _do_poll(app: "Quart") -> None:
|
||||
from .poller import poll_device
|
||||
from steward.core.alerts import MetricBatch
|
||||
|
||||
devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", [])
|
||||
if not devices:
|
||||
return
|
||||
|
||||
async with app.db_sessionmaker() as session:
|
||||
async with session.begin():
|
||||
batch = MetricBatch()
|
||||
for device in devices:
|
||||
if not isinstance(device, dict):
|
||||
continue
|
||||
name = device.get("name") or device.get("host", "unknown")
|
||||
host = device.get("host", "")
|
||||
port = int(device.get("port", 161))
|
||||
community = device.get("community", "public")
|
||||
version = str(device.get("version", "2c"))
|
||||
oids = device.get("oids", [])
|
||||
|
||||
if not host or not oids:
|
||||
continue
|
||||
|
||||
try:
|
||||
readings = await poll_device(host, port, community, version, oids)
|
||||
except Exception:
|
||||
logger.exception("SNMP poll failed for device %s (%s)", name, host)
|
||||
continue
|
||||
|
||||
for label, value in readings.items():
|
||||
batch.add(
|
||||
source_module="snmp",
|
||||
resource_name=name,
|
||||
metric_name=label,
|
||||
value=value,
|
||||
)
|
||||
|
||||
if readings:
|
||||
logger.debug("SNMP polled %s (%s): %d OID(s)", name, host, len(readings))
|
||||
else:
|
||||
logger.debug("SNMP polled %s (%s): no readings", name, host)
|
||||
await batch.record(session)
|
||||
@@ -0,0 +1,82 @@
|
||||
{# plugins/snmp/templates/snmp/device.html #}
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import crumbs %}
|
||||
{% block title %}SNMP — {{ device_name }} — Steward{% endblock %}
|
||||
{% block breadcrumb %}{{ crumbs([("SNMP", "/plugins/snmp/"), (device_name, "")]) }}{% endblock %}
|
||||
{% block content %}
|
||||
<div style="display:flex;align-items:center;gap:1rem;margin-bottom:1.5rem;flex-wrap:wrap;">
|
||||
<h1 class="page-title" style="margin:0;">{{ device_name }}</h1>
|
||||
<span style="font-size:0.8rem;color:var(--text-dim);">{{ device.host }}</span>
|
||||
<div style="margin-left:auto;display:flex;gap:0.5rem;">
|
||||
{% for h in [1, 6, 24, 168] %}
|
||||
<a href="?hours={{ h }}" class="range-btn{% if hours == h %} active{% endif %}"{% if hours == h %} aria-current="true"{% endif %}>
|
||||
{% if h < 24 %}{{ h }}h{% elif h == 24 %}24h{% else %}7d{% endif %}
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if oid_labels %}
|
||||
<div class="card">
|
||||
<canvas id="snmp-chart" style="width:100%;max-height:320px;"></canvas>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function(){
|
||||
const raw = {{ history_json | safe }};
|
||||
const labels = {{ oid_labels | tojson }};
|
||||
// Series colours from the tokens, in an order that keeps neighbours apart.
|
||||
const palette = ["--blue", "--yellow", "--green", "--red", "--gold", "--orange", "--text-muted"].map(stewardColor);
|
||||
|
||||
// Collect all unique timestamps, sorted
|
||||
const tsSet = new Set();
|
||||
for (const pts of Object.values(raw)) {
|
||||
for (const [ts] of pts) tsSet.add(ts);
|
||||
}
|
||||
const allTs = Array.from(tsSet).sort();
|
||||
|
||||
if (allTs.length < 2) {
|
||||
document.getElementById("snmp-chart").parentElement.innerHTML =
|
||||
'<p style="color:var(--text-muted);padding:1rem;">Not enough data for a chart yet.</p>';
|
||||
return;
|
||||
}
|
||||
|
||||
const datasets = [];
|
||||
labels.forEach((label, i) => {
|
||||
const pts = raw[label] || [];
|
||||
const map = Object.fromEntries(pts.map(([ts, v]) => [ts, v]));
|
||||
datasets.push({
|
||||
label,
|
||||
data: allTs.map(ts => map[ts] ?? null),
|
||||
borderColor: palette[i % palette.length],
|
||||
backgroundColor: "transparent",
|
||||
borderWidth: 1.5,
|
||||
pointRadius: 0,
|
||||
tension: 0.2,
|
||||
spanGaps: true,
|
||||
});
|
||||
});
|
||||
|
||||
const ctx = document.getElementById("snmp-chart").getContext("2d");
|
||||
new Chart(ctx, {
|
||||
type: "line",
|
||||
data: { labels: allTs.map(ts => ts.substring(11, 16)), datasets },
|
||||
options: {
|
||||
responsive: true,
|
||||
interaction: { mode: "index", intersect: false },
|
||||
plugins: {
|
||||
legend: { position: "top", labels: { boxWidth: 12 } },
|
||||
},
|
||||
scales: {
|
||||
x: { ticks: { maxTicksLimit: 12 } },
|
||||
},
|
||||
},
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
{% else %}
|
||||
<div class="card" style="color:var(--text-muted);text-align:center;padding:2rem;">
|
||||
No OIDs configured for this device.
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,53 @@
|
||||
{# Per-host SNMP fragment — embedded on the Hosts hub via HTMX. Self-contained.
|
||||
Shows the SNMP device(s) whose address maps to this host + latest readings. #}
|
||||
<div class="card">
|
||||
<div style="display:flex;align-items:baseline;justify-content:space-between;margin-bottom:0.6rem;gap:1rem;flex-wrap:wrap;">
|
||||
<h3 class="section-title" style="margin-bottom:0;">SNMP</h3>
|
||||
<a href="/plugins/snmp/" style="font-size:0.8rem;color:var(--text-muted);">All devices →</a>
|
||||
</div>
|
||||
|
||||
{% for device in devices %}
|
||||
<div style="{% if not loop.last %}margin-bottom:1rem;{% endif %}">
|
||||
<div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.5rem;flex-wrap:wrap;">
|
||||
<span style="font-weight:500;font-size:0.92rem;">{{ device.name }}</span>
|
||||
<span style="font-size:0.78rem;color:var(--text-dim);font-family:ui-monospace,monospace;">{{ device.host }}</span>
|
||||
{% if device.bound %}
|
||||
<span title="Explicitly bound to this host (host_id / steward_host)"
|
||||
style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:3px;padding:0.05rem 0.4rem;">bound</span>
|
||||
{% endif %}
|
||||
{% if device.reach == 'up' %}
|
||||
<span style="font-size:0.74rem;" class="tone-ok">● reachable · read {{ device.at|ago }}</span>
|
||||
{% elif device.reach == 'stale' %}
|
||||
<span style="font-size:0.74rem;" class="tone-warn" title="The values below are from then">● no answer since {{ device.at|ago }}</span>
|
||||
{% else %}
|
||||
<span style="font-size:0.74rem;color:var(--text-dim);">○ no data yet</span>
|
||||
{% endif %}
|
||||
<a href="/plugins/snmp/device/{{ device.name }}" class="btn btn-ghost btn-sm"
|
||||
style="margin-left:auto;font-size:0.76rem;padding:0.15rem 0.55rem;">History</a>
|
||||
</div>
|
||||
|
||||
{% if device.readings %}
|
||||
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.5rem;">
|
||||
{% for oid_cfg in device.oids %}
|
||||
{% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
|
||||
{% set val = device.readings.get(label) %}
|
||||
<div style="background:var(--bg-elevated);border-radius:5px;padding:0.5rem 0.7rem;border:1px solid var(--border);">
|
||||
<div style="font-size:0.68rem;color:var(--text-dim);margin-bottom:0.2rem;font-family:ui-monospace,monospace;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ label }}</div>
|
||||
{% if val is not none %}
|
||||
<div style="font-size:1rem;font-weight:500;font-variant-numeric:tabular-nums;">
|
||||
{% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}<span style="font-size:0.72rem;color:var(--text-muted);">M</span>
|
||||
{% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}<span style="font-size:0.72rem;color:var(--text-muted);">K</span>
|
||||
{% else %}{{ "%.4g"|format(val) }}{% endif %}
|
||||
</div>
|
||||
{% else %}
|
||||
<div style="font-size:0.85rem;color:var(--text-dim);">—</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% else %}
|
||||
<p style="font-size:0.8rem;color:var(--text-dim);margin:0;">No readings yet — waiting for the next poll.</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
@@ -0,0 +1,45 @@
|
||||
{# plugins/snmp/templates/snmp/index.html #}
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import page_header, status_badge, stat_tile, empty_state %}
|
||||
{% block title %}SNMP — Steward{% endblock %}
|
||||
{% block content %}
|
||||
{{ page_header("SNMP devices", devices|length ~ " device" ~ ("s" if devices|length != 1 else "") ~ " configured") }}
|
||||
|
||||
{% if not devices %}
|
||||
<div class="card">{{ empty_state("No SNMP devices configured. Add them to the snmp plugin's plugin.yaml.") }}</div>
|
||||
{% else %}
|
||||
|
||||
{% for device in devices %}
|
||||
<section class="card" aria-labelledby="snmp-{{ loop.index }}">
|
||||
<div class="host-health-head">
|
||||
{#- reach: answered within the last few polls (up), only long ago (stale),
|
||||
or never (pending); routes._device_state. #}
|
||||
{% if device.reach == 'up' %}{{ status_badge('up', 'Reachable') }}
|
||||
{% elif device.reach == 'stale' %}{{ status_badge('stale', 'No answer', title="The last reading is from " ~ device.at.strftime('%Y-%m-%d %H:%M UTC') ~ "; the values below are from then") }}
|
||||
{% else %}{{ status_badge('pending', 'No data yet') }}{% endif %}
|
||||
<h2 id="snmp-{{ loop.index }}" class="section-title" style="margin:0;">{{ device.name }}</h2>
|
||||
<span class="field-hint mono">{{ device.host }}</span>
|
||||
{% if device.at %}<span class="field-hint">read {{ device.at|ago }}</span>{% endif %}
|
||||
<a href="/plugins/snmp/device/{{ device.name }}" class="btn btn-sm btn-ghost" style="margin-left:auto;">History</a>
|
||||
</div>
|
||||
|
||||
{% if device.readings %}
|
||||
<div class="stat-strip" style="margin-bottom:0;">
|
||||
{% for oid_cfg in device.oids %}
|
||||
{% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
|
||||
{% set val = device.readings.get(label) %}
|
||||
{{ stat_tile(label,
|
||||
"—" if val is none else
|
||||
("%.2fM"|format(val / 1000000)) if val >= 1000000 else
|
||||
("%.1fK"|format(val / 1000)) if val >= 1000 else
|
||||
"%.4g"|format(val)) }}
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% else %}
|
||||
<p class="field-hint">No readings yet; waiting for the next poll.</p>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endfor %}
|
||||
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,51 @@
|
||||
{# plugins/snmp/templates/snmp/widget.html #}
|
||||
{% if devices %}
|
||||
{% for device in devices %}
|
||||
<div class="ping-row" style="align-items:flex-start;flex-direction:column;gap:0.3rem;padding:0.4rem 0;">
|
||||
<div style="display:flex;align-items:center;gap:0.6rem;width:100%;">
|
||||
<span style="font-size:0.82rem;font-weight:500;flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">
|
||||
{{ device.name }}
|
||||
</span>
|
||||
{% if device.reach == 'up' %}
|
||||
<span style="font-size:0.7rem;" class="tone-ok" title="Reachable · read {{ device.at.strftime('%H:%M UTC') }}">●</span>
|
||||
{% elif device.reach == 'stale' %}
|
||||
<span style="font-size:0.7rem;" class="tone-warn">no answer since {{ device.at|ago }}</span>
|
||||
{% else %}
|
||||
<span style="font-size:0.7rem;color:var(--text-dim);">○</span>
|
||||
{% endif %}
|
||||
<a href="/plugins/snmp/device/{{ device.name }}" style="font-size:0.72rem;color:var(--text-muted);">detail →</a>
|
||||
</div>
|
||||
{% if device.readings %}
|
||||
<div style="display:flex;flex-wrap:wrap;gap:0.5rem 1rem;padding-left:0.1rem;">
|
||||
{% for oid_cfg in device.oids[:4] %}
|
||||
{% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
|
||||
{% set val = device.readings.get(label) %}
|
||||
{% if val is not none %}
|
||||
<span style="font-size:0.78rem;font-variant-numeric:tabular-nums;">
|
||||
<span style="color:var(--text-muted);">{{ label }}</span>
|
||||
<span style="margin-left:0.25rem;">
|
||||
{% if val >= 1000000 %}{{ "%.1f"|format(val / 1000000) }}M
|
||||
{% elif val >= 1000 %}{{ "%.0f"|format(val / 1000) }}K
|
||||
{% else %}{{ "%.4g"|format(val) }}{% endif %}
|
||||
</span>
|
||||
</span>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if device.oids|length > 4 %}
|
||||
<span style="font-size:0.72rem;color:var(--text-dim);">+{{ device.oids|length - 4 }} more</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if not loop.last %}
|
||||
<div style="height:1px;background:var(--border);margin:0.1rem 0;"></div>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if total > devices|length %}
|
||||
<div style="color:var(--text-dim);font-size:0.75rem;padding:0.25rem 0;">
|
||||
+{{ total - devices|length }} more — <a href="/plugins/snmp/">view all →</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% else %}
|
||||
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No SNMP devices configured.</p>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,48 @@
|
||||
# plugins/traefik/__init__.py
|
||||
from __future__ import annotations
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from quart import Quart
|
||||
|
||||
_app: "Quart | None" = None
|
||||
|
||||
|
||||
def setup(app: "Quart") -> None:
|
||||
global _app
|
||||
_app = app
|
||||
from .models import TraefikMetric, TraefikCert, TraefikGlobalStat, TraefikAccessSummary # noqa: F401
|
||||
from steward.core.cleanup import prune_by_age
|
||||
# Samples older than data.retention_days are pruned by the cleanup (#5155).
|
||||
prune_by_age(TraefikMetric.scraped_at)
|
||||
prune_by_age(TraefikGlobalStat.scraped_at)
|
||||
prune_by_age(TraefikAccessSummary.period_end)
|
||||
|
||||
|
||||
def get_scheduled_tasks() -> list:
|
||||
from .scheduler import make_scrape_task, make_access_log_task
|
||||
tasks = [make_scrape_task(_app)]
|
||||
access_log_cfg = _app.config["PLUGINS"]["traefik"].get("access_log", {})
|
||||
if not isinstance(access_log_cfg, dict):
|
||||
access_log_cfg = {}
|
||||
if access_log_cfg.get("enabled", False):
|
||||
tasks.append(make_access_log_task(_app))
|
||||
return tasks
|
||||
|
||||
|
||||
def get_blueprint():
|
||||
from .routes import traefik_bp
|
||||
return traefik_bp
|
||||
|
||||
|
||||
def get_nav() -> list[dict]:
|
||||
# Surfaces in the sidebar's Infrastructure group (Traefik services view).
|
||||
return [{"label": "Traefik", "href": "/plugins/traefik/"}]
|
||||
|
||||
|
||||
def get_alert_metrics() -> dict[str, list[str]]:
|
||||
"""What scheduler.py records for alert rules: per router (resource = the
|
||||
router name) and cert_expiry_days per certificate (resource = its CN)."""
|
||||
return {"traefik": ["request_rate", "error_rate_4xx_pct", "error_rate_5xx_pct",
|
||||
"latency_p50_ms", "latency_p95_ms", "latency_p99_ms",
|
||||
"response_bytes_rate", "cert_expiry_days"]}
|
||||
@@ -0,0 +1,199 @@
|
||||
# plugins/traefik/access_log.py
|
||||
"""Traefik JSON access log tailer and traffic-origin aggregator.
|
||||
|
||||
Reads new lines from Traefik's JSON access log on each scheduler tick,
|
||||
classifies source IPs as internal (RFC1918) or external, optionally
|
||||
looks up country codes via MaxMind GeoLite2, and returns an aggregated
|
||||
summary ready to persist in TraefikAccessSummary.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
# ── Module-level file-position state ─────────────────────────────────────────
|
||||
_log_position: int = 0 # byte offset of last read
|
||||
_log_inode: int = 0 # detect rotation by inode change
|
||||
|
||||
# ── Private IP ranges (RFC1918 + loopback + link-local) ──────────────────────
|
||||
_PRIVATE_NETS = [
|
||||
ipaddress.ip_network("10.0.0.0/8"),
|
||||
ipaddress.ip_network("172.16.0.0/12"),
|
||||
ipaddress.ip_network("192.168.0.0/16"),
|
||||
ipaddress.ip_network("127.0.0.0/8"),
|
||||
ipaddress.ip_network("169.254.0.0/16"),
|
||||
ipaddress.ip_network("::1/128"),
|
||||
ipaddress.ip_network("fc00::/7"),
|
||||
ipaddress.ip_network("fe80::/10"),
|
||||
]
|
||||
|
||||
|
||||
def _strip_port(raw: str) -> str:
|
||||
"""Strip trailing :port from an IPv4 address string. Leaves IPv6 untouched."""
|
||||
if not raw:
|
||||
return raw
|
||||
# Bracketed IPv6 with port: [::1]:12345
|
||||
if raw.startswith("["):
|
||||
return raw[1:raw.index("]")] if "]" in raw else raw
|
||||
# IPv4:port has exactly one colon
|
||||
if raw.count(":") == 1:
|
||||
return raw.rsplit(":", 1)[0]
|
||||
return raw
|
||||
|
||||
|
||||
def is_internal(raw_ip: str) -> bool:
|
||||
"""Return True if the address falls within a private/loopback range."""
|
||||
ip_str = _strip_port(raw_ip)
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip_str)
|
||||
return any(addr in net for net in _PRIVATE_NETS)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _country(ip_str: str, geoip_db) -> str | None:
|
||||
"""Return ISO country code via an open maxminddb reader, or None."""
|
||||
if geoip_db is None:
|
||||
return None
|
||||
clean = _strip_port(ip_str)
|
||||
try:
|
||||
record = geoip_db.get(clean)
|
||||
if record:
|
||||
return record.get("country", {}).get("iso_code")
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def read_new_lines(log_path: str) -> list[dict]:
|
||||
"""Return all new JSON-parsed access log entries since the last call.
|
||||
|
||||
Handles log rotation: if the file shrinks or its inode changes, we
|
||||
reset to the beginning of the new file.
|
||||
"""
|
||||
global _log_position, _log_inode
|
||||
|
||||
path = Path(log_path)
|
||||
if not path.exists():
|
||||
return []
|
||||
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
current_inode = stat.st_ino
|
||||
current_size = stat.st_size
|
||||
|
||||
# Rotation detected
|
||||
if current_inode != _log_inode or current_size < _log_position:
|
||||
_log_position = 0
|
||||
_log_inode = current_inode
|
||||
|
||||
if current_size == _log_position:
|
||||
return []
|
||||
|
||||
entries: list[dict] = []
|
||||
try:
|
||||
with open(log_path, "rb") as f:
|
||||
f.seek(_log_position)
|
||||
raw = f.read(current_size - _log_position)
|
||||
_log_position = current_size
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
for raw_line in raw.split(b"\n"):
|
||||
raw_line = raw_line.strip()
|
||||
if not raw_line:
|
||||
continue
|
||||
try:
|
||||
entries.append(json.loads(raw_line))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def aggregate(entries: list[dict], geoip_db=None, top_n: int = 10) -> dict | None:
|
||||
"""Aggregate a batch of access log entries into a summary dict.
|
||||
|
||||
Returns None if there are no entries.
|
||||
Result keys map directly to TraefikAccessSummary columns (excluding
|
||||
id and period_start/period_end, which the caller provides).
|
||||
"""
|
||||
if not entries:
|
||||
return None
|
||||
|
||||
total = 0
|
||||
internal = 0
|
||||
external = 0
|
||||
total_bytes = 0.0
|
||||
|
||||
by_router: dict[str, int] = defaultdict(int)
|
||||
ip_counts: dict[str, int] = defaultdict(int)
|
||||
ip_internal: dict[str, bool] = {}
|
||||
ip_country: dict[str, str | None] = {}
|
||||
country_counts: dict[str, int] = defaultdict(int)
|
||||
|
||||
for entry in entries:
|
||||
total += 1
|
||||
raw_ip = entry.get("ClientHost", "")
|
||||
router = entry.get("RouterName") or entry.get("ServiceName") or "unknown"
|
||||
total_bytes += entry.get("DownstreamContentSize") or 0
|
||||
|
||||
by_router[router] += 1
|
||||
|
||||
internal_flag = is_internal(raw_ip)
|
||||
if internal_flag:
|
||||
internal += 1
|
||||
else:
|
||||
external += 1
|
||||
if raw_ip not in ip_country:
|
||||
ip_country[raw_ip] = _country(raw_ip, geoip_db)
|
||||
c = ip_country[raw_ip]
|
||||
if c:
|
||||
country_counts[c] += 1
|
||||
|
||||
ip_counts[raw_ip] += 1
|
||||
ip_internal[raw_ip] = internal_flag
|
||||
|
||||
top_ips = [
|
||||
{
|
||||
"ip": ip,
|
||||
"count": cnt,
|
||||
"internal": ip_internal[ip],
|
||||
"country": ip_country.get(ip),
|
||||
}
|
||||
for ip, cnt in sorted(ip_counts.items(), key=lambda x: x[1], reverse=True)[:top_n]
|
||||
]
|
||||
top_countries = [
|
||||
{"country": c, "count": n}
|
||||
for c, n in sorted(country_counts.items(), key=lambda x: x[1], reverse=True)[:top_n]
|
||||
]
|
||||
top_routers = [
|
||||
{"router": r, "count": n}
|
||||
for r, n in sorted(by_router.items(), key=lambda x: x[1], reverse=True)[:top_n]
|
||||
]
|
||||
|
||||
return {
|
||||
"total_requests": total,
|
||||
"internal_requests": internal,
|
||||
"external_requests": external,
|
||||
"total_bytes": total_bytes,
|
||||
"top_ips_json": json.dumps(top_ips),
|
||||
"top_countries_json": json.dumps(top_countries),
|
||||
"top_routers_json": json.dumps(top_routers),
|
||||
}
|
||||
|
||||
|
||||
def open_geoip(db_path: str | None):
|
||||
"""Open a MaxMind mmdb file if available. Returns None if not configured or missing."""
|
||||
if not db_path:
|
||||
return None
|
||||
try:
|
||||
import maxminddb # type: ignore[import]
|
||||
return maxminddb.open_database(db_path)
|
||||
except Exception:
|
||||
return None
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user