From a7a281cb110c9e5264e93ddef58769295f4e45cb Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 1 Jun 2026 08:37:24 -0400 Subject: [PATCH 001/126] feat(plugins): fold first-party plugins in-tree; bundled + external roots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First-party plugins (host_agent, http, snmp, traefik, unifi, docker) are now tracked under plugins/ and baked into the image, so they version atomically with core — ending the cross-repo import drift the roundtable->steward rename exposed. History for these files is preserved in the archived Roundtable-plugins repo. Plugin discovery becomes multi-root: PLUGIN_DIR (single) -> PLUGIN_DIRS (bundled first, then external) + PLUGIN_INSTALL_DIR. Bundled ships in the image; third-party plugins still mount at runtime into the external root (STEWARD_PLUGIN_DIR, default /data/plugins) and downloads/installs land there. Bundled shadows external on a name collision. - config.py: load_bootstrap returns plugin_dirs + plugin_install_dir - app.py: iterate PLUGIN_DIRS at the migration + load sites - migration_runner.py: discover_all_in() unions every plugin root - plugin_manager.py: resolve_plugin_path() (pure, first-root-wins); load / install / hot-reload span all roots; installs target the external root - settings/routes.py: _discover_plugins scans all roots, dedup bundled-first - Dockerfile: COPY plugins/ ; docker-compose: drop host bind, document external - tests/test_plugin_dirs.py: resolution, multi-root discovery, bootstrap split Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 10 +- Dockerfile | 4 + docker-compose.yml | 4 +- plugins/__init__.py | 1 + plugins/docker/__init__.py | 24 + plugins/docker/migrations/__init__.py | 0 plugins/docker/migrations/env.py | 70 +++ .../docker/migrations/versions/__init__.py | 0 .../migrations/versions/docker_001_initial.py | 47 ++ plugins/docker/models.py | 49 ++ plugins/docker/plugin.yaml | 17 + plugins/docker/routes.py | 161 ++++++ plugins/docker/scheduler.py | 93 ++++ plugins/docker/scraper.py | 140 +++++ plugins/docker/templates/docker/index.html | 16 + plugins/docker/templates/docker/rows.html | 86 +++ plugins/docker/templates/docker/widget.html | 35 ++ .../templates/docker/widget_resources.html | 36 ++ plugins/host_agent/__init__.py | 23 + plugins/host_agent/agent.py | 370 +++++++++++++ plugins/host_agent/migrations/__init__.py | 0 plugins/host_agent/migrations/env.py | 70 +++ .../migrations/versions/__init__.py | 0 .../versions/host_agent_001_initial.py | 36 ++ plugins/host_agent/models.py | 32 ++ plugins/host_agent/plugin.yaml | 19 + plugins/host_agent/routes.py | 398 +++++++++++++ plugins/host_agent/scheduler.py | 78 +++ plugins/host_agent/templates/install.sh.j2 | 81 +++ .../host_agent/templates/settings_list.html | 65 +++ .../host_agent/templates/widget_history.html | 24 + .../host_agent/templates/widget_table.html | 34 ++ plugins/http/__init__.py | 24 + plugins/http/checker.py | 105 ++++ plugins/http/migrations/__init__.py | 0 plugins/http/migrations/env.py | 70 +++ plugins/http/migrations/versions/__init__.py | 0 .../migrations/versions/http_001_initial.py | 52 ++ plugins/http/models.py | 63 +++ plugins/http/plugin.yaml | 19 + plugins/http/routes.py | 229 ++++++++ plugins/http/scheduler.py | 121 ++++ plugins/http/templates/http/index.html | 77 +++ plugins/http/templates/http/rows.html | 109 ++++ plugins/http/templates/http/widget.html | 56 ++ plugins/index.yaml | 90 +++ plugins/snmp/__init__.py | 23 + plugins/snmp/plugin.yaml | 49 ++ plugins/snmp/poller.py | 94 ++++ plugins/snmp/routes.py | 146 +++++ plugins/snmp/scheduler.py | 78 +++ plugins/snmp/templates/snmp/device.html | 84 +++ plugins/snmp/templates/snmp/index.html | 57 ++ plugins/snmp/templates/snmp/widget.html | 49 ++ plugins/traefik/__init__.py | 30 + plugins/traefik/access_log.py | 199 +++++++ plugins/traefik/migrations/__init__.py | 0 plugins/traefik/migrations/env.py | 75 +++ .../traefik/migrations/versions/__init__.py | 0 .../versions/traefik_001_initial.py | 37 ++ .../migrations/versions/traefik_002_stats.py | 48 ++ .../versions/traefik_003_access_log.py | 40 ++ plugins/traefik/models.py | 67 +++ plugins/traefik/plugin.yaml | 21 + plugins/traefik/routes.py | 373 +++++++++++++ plugins/traefik/scheduler.py | 204 +++++++ plugins/traefik/scraper.py | 276 +++++++++ plugins/traefik/templates/traefik/index.html | 31 ++ plugins/traefik/templates/traefik/rows.html | 127 +++++ .../traefik/templates/traefik/traffic.html | 101 ++++ plugins/traefik/templates/traefik/widget.html | 56 ++ .../templates/traefik/widget_access_log.html | 40 ++ .../traefik/widget_request_chart.html | 75 +++ plugins/unifi/__init__.py | 28 + plugins/unifi/client.py | 170 ++++++ plugins/unifi/migrations/__init__.py | 0 plugins/unifi/migrations/env.py | 72 +++ plugins/unifi/migrations/versions/__init__.py | 0 .../migrations/versions/unifi_001_initial.py | 62 +++ .../migrations/versions/unifi_002_expanded.py | 156 ++++++ plugins/unifi/models.py | 182 ++++++ plugins/unifi/plugin.yaml | 21 + plugins/unifi/routes.py | 357 ++++++++++++ plugins/unifi/scheduler.py | 522 ++++++++++++++++++ plugins/unifi/templates/unifi/dpi.html | 65 +++ plugins/unifi/templates/unifi/events.html | 52 ++ plugins/unifi/templates/unifi/index.html | 63 +++ plugins/unifi/templates/unifi/inventory.html | 146 +++++ plugins/unifi/templates/unifi/rows.html | 203 +++++++ plugins/unifi/templates/unifi/security.html | 56 ++ plugins/unifi/templates/unifi/widget.html | 95 ++++ .../unifi/templates/unifi/widget_clients.html | 42 ++ .../unifi/templates/unifi/widget_devices.html | 36 ++ steward/app.py | 14 +- steward/config.py | 22 +- steward/core/migration_runner.py | 18 +- steward/core/plugin_manager.py | 59 +- steward/settings/routes.py | 40 +- tests/test_plugin_dirs.py | 84 +++ 99 files changed, 7931 insertions(+), 52 deletions(-) create mode 100644 plugins/__init__.py create mode 100644 plugins/docker/__init__.py create mode 100644 plugins/docker/migrations/__init__.py create mode 100644 plugins/docker/migrations/env.py create mode 100644 plugins/docker/migrations/versions/__init__.py create mode 100644 plugins/docker/migrations/versions/docker_001_initial.py create mode 100644 plugins/docker/models.py create mode 100644 plugins/docker/plugin.yaml create mode 100644 plugins/docker/routes.py create mode 100644 plugins/docker/scheduler.py create mode 100644 plugins/docker/scraper.py create mode 100644 plugins/docker/templates/docker/index.html create mode 100644 plugins/docker/templates/docker/rows.html create mode 100644 plugins/docker/templates/docker/widget.html create mode 100644 plugins/docker/templates/docker/widget_resources.html create mode 100644 plugins/host_agent/__init__.py create mode 100644 plugins/host_agent/agent.py create mode 100644 plugins/host_agent/migrations/__init__.py create mode 100644 plugins/host_agent/migrations/env.py create mode 100644 plugins/host_agent/migrations/versions/__init__.py create mode 100644 plugins/host_agent/migrations/versions/host_agent_001_initial.py create mode 100644 plugins/host_agent/models.py create mode 100644 plugins/host_agent/plugin.yaml create mode 100644 plugins/host_agent/routes.py create mode 100644 plugins/host_agent/scheduler.py create mode 100644 plugins/host_agent/templates/install.sh.j2 create mode 100644 plugins/host_agent/templates/settings_list.html create mode 100644 plugins/host_agent/templates/widget_history.html create mode 100644 plugins/host_agent/templates/widget_table.html create mode 100644 plugins/http/__init__.py create mode 100644 plugins/http/checker.py create mode 100644 plugins/http/migrations/__init__.py create mode 100644 plugins/http/migrations/env.py create mode 100644 plugins/http/migrations/versions/__init__.py create mode 100644 plugins/http/migrations/versions/http_001_initial.py create mode 100644 plugins/http/models.py create mode 100644 plugins/http/plugin.yaml create mode 100644 plugins/http/routes.py create mode 100644 plugins/http/scheduler.py create mode 100644 plugins/http/templates/http/index.html create mode 100644 plugins/http/templates/http/rows.html create mode 100644 plugins/http/templates/http/widget.html create mode 100644 plugins/index.yaml create mode 100644 plugins/snmp/__init__.py create mode 100644 plugins/snmp/plugin.yaml create mode 100644 plugins/snmp/poller.py create mode 100644 plugins/snmp/routes.py create mode 100644 plugins/snmp/scheduler.py create mode 100644 plugins/snmp/templates/snmp/device.html create mode 100644 plugins/snmp/templates/snmp/index.html create mode 100644 plugins/snmp/templates/snmp/widget.html create mode 100644 plugins/traefik/__init__.py create mode 100644 plugins/traefik/access_log.py create mode 100644 plugins/traefik/migrations/__init__.py create mode 100644 plugins/traefik/migrations/env.py create mode 100644 plugins/traefik/migrations/versions/__init__.py create mode 100644 plugins/traefik/migrations/versions/traefik_001_initial.py create mode 100644 plugins/traefik/migrations/versions/traefik_002_stats.py create mode 100644 plugins/traefik/migrations/versions/traefik_003_access_log.py create mode 100644 plugins/traefik/models.py create mode 100644 plugins/traefik/plugin.yaml create mode 100644 plugins/traefik/routes.py create mode 100644 plugins/traefik/scheduler.py create mode 100644 plugins/traefik/scraper.py create mode 100644 plugins/traefik/templates/traefik/index.html create mode 100644 plugins/traefik/templates/traefik/rows.html create mode 100644 plugins/traefik/templates/traefik/traffic.html create mode 100644 plugins/traefik/templates/traefik/widget.html create mode 100644 plugins/traefik/templates/traefik/widget_access_log.html create mode 100644 plugins/traefik/templates/traefik/widget_request_chart.html create mode 100644 plugins/unifi/__init__.py create mode 100644 plugins/unifi/client.py create mode 100644 plugins/unifi/migrations/__init__.py create mode 100644 plugins/unifi/migrations/env.py create mode 100644 plugins/unifi/migrations/versions/__init__.py create mode 100644 plugins/unifi/migrations/versions/unifi_001_initial.py create mode 100644 plugins/unifi/migrations/versions/unifi_002_expanded.py create mode 100644 plugins/unifi/models.py create mode 100644 plugins/unifi/plugin.yaml create mode 100644 plugins/unifi/routes.py create mode 100644 plugins/unifi/scheduler.py create mode 100644 plugins/unifi/templates/unifi/dpi.html create mode 100644 plugins/unifi/templates/unifi/events.html create mode 100644 plugins/unifi/templates/unifi/index.html create mode 100644 plugins/unifi/templates/unifi/inventory.html create mode 100644 plugins/unifi/templates/unifi/rows.html create mode 100644 plugins/unifi/templates/unifi/security.html create mode 100644 plugins/unifi/templates/unifi/widget.html create mode 100644 plugins/unifi/templates/unifi/widget_clients.html create mode 100644 plugins/unifi/templates/unifi/widget_devices.html create mode 100644 tests/test_plugin_dirs.py diff --git a/.gitignore b/.gitignore index fb2b9af..b4b5ccd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ # Planning files docs/superpowers/ -# Plugin directory — managed as a separate repo (bvandeusen/steward-plugins) -# PLUGIN_DIR in config.yaml points here at runtime -/plugins/ +# First-party plugins are tracked in-tree under plugins/ and ship in the image. +# Third-party plugins are mounted at runtime into the external dir +# (STEWARD_PLUGIN_DIR, default /data/plugins) and are not tracked here. +/plugins/**/*.zip # Python __pycache__/ @@ -50,3 +51,6 @@ Thumbs.db # Playbook cache (runtime data) playbook_cache/ + +# Superpowers brainstorm scratch (visual companion mockups, state) +.superpowers/ diff --git a/Dockerfile b/Dockerfile index 938501c..4172e1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,10 @@ COPY steward/ steward/ RUN pip install --no-cache-dir . COPY alembic.ini . +# First-party plugins ship inside the image (bundled root at /app/plugins). +# Third-party plugins are mounted at runtime into the external root +# (STEWARD_PLUGIN_DIR, default /data/plugins). +COPY plugins/ plugins/ COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh diff --git a/docker-compose.yml b/docker-compose.yml index faedd9f..3e1ce35 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,10 @@ services: ports: - "5000:5000" volumes: + # First-party plugins are baked into the image at /app/plugins. + # Third-party plugins persist in the external root /data/plugins + # (inside app_data); STEWARD_PLUGIN_DIR defaults there, no extra mount needed. - app_data:/data - - ./plugins:/app/plugins:ro - /mnt/Data/traefik/log:/var/log/traefik:ro - /var/run/docker.sock:/var/run/docker.sock:ro environment: diff --git a/plugins/__init__.py b/plugins/__init__.py new file mode 100644 index 0000000..1648f8e --- /dev/null +++ b/plugins/__init__.py @@ -0,0 +1 @@ +# plugins/__init__.py diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py new file mode 100644 index 0000000..d569556 --- /dev/null +++ b/plugins/docker/__init__.py @@ -0,0 +1,24 @@ +# plugins/docker/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + from .models import DockerContainer, DockerMetric # noqa: F401 — register with Base + + +def get_scheduled_tasks() -> list: + from .scheduler import make_task + return [make_task(_app)] + + +def get_blueprint(): + from .routes import docker_bp + return docker_bp diff --git a/plugins/docker/migrations/__init__.py b/plugins/docker/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/docker/migrations/env.py b/plugins/docker/migrations/env.py new file mode 100644 index 0000000..5c20755 --- /dev/null +++ b/plugins/docker/migrations/env.py @@ -0,0 +1,70 @@ +# plugins/docker/migrations/env.py +"""Alembic env.py for the Docker plugin.""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from fabledscryer.models.base import Base +import fabledscryer.models # noqa: F401 +from plugins.docker.models import DockerContainer, DockerMetric # noqa: F401 + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("FABLEDSCRYER_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/docker/migrations/versions/__init__.py b/plugins/docker/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/docker/migrations/versions/docker_001_initial.py b/plugins/docker/migrations/versions/docker_001_initial.py new file mode 100644 index 0000000..90e5298 --- /dev/null +++ b/plugins/docker/migrations/versions/docker_001_initial.py @@ -0,0 +1,47 @@ +"""Docker plugin initial tables + +Revision ID: docker_001_initial +Revises: (none — branch off core via depends_on) +Create Date: 2026-03-22 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "docker" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "docker_containers", + sa.Column("name", sa.String(255), primary_key=True), + sa.Column("container_id", sa.String(64), nullable=False, server_default=""), + sa.Column("image", sa.String(512), nullable=False, server_default=""), + sa.Column("status", sa.String(32), nullable=False, server_default="unknown"), + sa.Column("cpu_pct", sa.Float, nullable=True), + sa.Column("mem_usage_bytes", sa.Integer, nullable=True), + sa.Column("mem_limit_bytes", sa.Integer, nullable=True), + sa.Column("mem_pct", sa.Float, nullable=True), + sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"), + sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "docker_metrics", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("container_name", sa.String(255), nullable=False, index=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True), + sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"), + ) + + +def downgrade() -> None: + op.drop_table("docker_metrics") + op.drop_table("docker_containers") diff --git a/plugins/docker/models.py b/plugins/docker/models.py new file mode 100644 index 0000000..09c1cf3 --- /dev/null +++ b/plugins/docker/models.py @@ -0,0 +1,49 @@ +# plugins/docker/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import DateTime, Float, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from fabledscryer.models.base import Base + + +class DockerContainer(Base): + """Latest known state per container — upserted by name on each scrape.""" + __tablename__ = "docker_containers" + + # Container name is the stable natural key; container_id changes on recreation + name: Mapped[str] = mapped_column(String(255), primary_key=True) + container_id: Mapped[str] = mapped_column(String(64), nullable=False, default="") + image: Mapped[str] = mapped_column(String(512), nullable=False, default="") + status: Mapped[str] = mapped_column(String(32), nullable=False, default="unknown") + # running | stopped | paused | exited | dead + cpu_pct: Mapped[float | None] = mapped_column(Float, nullable=True) + mem_usage_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) + mem_limit_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) + mem_pct: Mapped[float | None] = mapped_column(Float, nullable=True) + restart_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + ports_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + # JSON: [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}] + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + +class DockerMetric(Base): + """Time-series CPU/memory per container — one row per scrape per running container.""" + __tablename__ = "docker_metrics" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + index=True, + ) + cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + mem_usage_bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0) diff --git a/plugins/docker/plugin.yaml b/plugins/docker/plugin.yaml new file mode 100644 index 0000000..8745302 --- /dev/null +++ b/plugins/docker/plugin.yaml @@ -0,0 +1,17 @@ +name: docker +version: "1.0.0" +description: "Docker container status, resource usage, restart tracking via Docker socket" +author: "FabledScryer" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/docker" +tags: + - containers + - docker + - infrastructure + +config: + socket_path: /var/run/docker.sock + scrape_interval_seconds: 60 + include_stopped: false diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py new file mode 100644 index 0000000..12d0383 --- /dev/null +++ b/plugins/docker/routes.py @@ -0,0 +1,161 @@ +# plugins/docker/routes.py +from __future__ import annotations +import json +from datetime import datetime, timezone +from quart import Blueprint, current_app, render_template, request +from sqlalchemy import Integer, cast, func, select + +from fabledscryer.auth.middleware import require_role +from fabledscryer.models.users import UserRole +from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds +from .models import DockerContainer, DockerMetric + +docker_bp = Blueprint("docker", __name__, template_folder="templates") + + +def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: + if len(values) < 2: + return f'' + mn, mx = min(values), max(values) + if mx == mn: + mx = mn + 1.0 + step = width / (len(values) - 1) + pts = [] + for i, v in enumerate(values): + x = i * step + y = height - (v - mn) / (mx - mn) * (height - 2) - 1 + pts.append(f"{x:.1f},{y:.1f}") + poly = " ".join(pts) + return ( + f'' + f'' + f'' + ) + + +@docker_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + current_range = request.args.get("range", DEFAULT_RANGE) + return await render_template( + "docker/index.html", + poll_interval=poll_interval, + current_range=current_range, + ) + + +@docker_bp.get("/rows") +@require_role(UserRole.viewer) +async def rows(): + """HTMX fragment: container list with status and resource sparklines.""" + since, range_key = parse_range(request.args.get("range")) + b_secs = bucket_seconds(since) + + bucket_col = ( + cast(func.strftime('%s', DockerMetric.scraped_at), Integer) / b_secs + ).label("bucket") + + async with current_app.db_sessionmaker() as db: + # All known containers ordered by running first, then name + result = await db.execute( + select(DockerContainer) + .order_by( + # running first + (DockerContainer.status == "running").desc(), + DockerContainer.name, + ) + ) + containers = list(result.scalars()) + + # Build per-container sparkline histories + histories: dict[str, list] = {} + for c in containers: + result = await db.execute( + select( + func.avg(DockerMetric.cpu_pct).label("cpu_pct"), + func.avg(DockerMetric.mem_pct).label("mem_pct"), + bucket_col, + ) + .where(DockerMetric.container_name == c.name) + .where(DockerMetric.scraped_at >= since) + .group_by(bucket_col) + .order_by(bucket_col) + ) + histories[c.name] = result.all() + + running = sum(1 for c in containers if c.status == "running") + stopped = len(containers) - running + + container_data = [] + for c in containers: + hist = histories.get(c.name, []) + ports = json.loads(c.ports_json) if c.ports_json else [] + container_data.append({ + "container": c, + "ports": ports, + "sparkline_cpu": _sparkline([r.cpu_pct or 0 for r in hist]), + "sparkline_mem": _sparkline([r.mem_pct or 0 for r in hist]), + }) + + return await render_template( + "docker/rows.html", + container_data=container_data, + running=running, + stopped=stopped, + range_key=range_key, + ) + + +@docker_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """HTMX dashboard widget: container status overview.""" + show_stopped = request.args.get("show_stopped", "no") == "yes" + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(DockerContainer) + .order_by( + (DockerContainer.status == "running").desc(), + DockerContainer.name, + ) + ) + all_containers = list(result.scalars()) + + running = [c for c in all_containers if c.status == "running"] + stopped = [c for c in all_containers if c.status != "running"] + display = all_containers if show_stopped else running + + return await render_template( + "docker/widget.html", + containers=display, + running_count=len(running), + stopped_count=len(stopped), + show_stopped=show_stopped, + widget_id=widget_id, + ) + + +@docker_bp.get("/widget/resources") +@require_role(UserRole.viewer) +async def widget_resources(): + """HTMX dashboard widget: CPU + memory usage for running containers.""" + limit = max(1, min(20, int(request.args.get("limit", 10) or 10))) + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(DockerContainer) + .where(DockerContainer.status == "running") + .order_by(DockerContainer.cpu_pct.desc().nullslast()) + ) + containers = list(result.scalars())[:limit] + + return await render_template( + "docker/widget_resources.html", + containers=containers, + widget_id=widget_id, + ) diff --git a/plugins/docker/scheduler.py b/plugins/docker/scheduler.py new file mode 100644 index 0000000..eef8c61 --- /dev/null +++ b/plugins/docker/scheduler.py @@ -0,0 +1,93 @@ +# plugins/docker/scheduler.py +from __future__ import annotations +import json +import logging +from datetime import datetime, timezone + +from fabledscryer.core.scheduler import ScheduledTask +from fabledscryer.core.alerts import record_metric + +logger = logging.getLogger(__name__) + + +def make_task(app) -> ScheduledTask: + interval = int( + app.config["PLUGINS"]["docker"].get("scrape_interval_seconds", 60) + ) + + async def scrape(): + await _do_scrape(app) + + return ScheduledTask( + name="docker_scrape", + coro_factory=scrape, + interval_seconds=interval, + run_on_startup=True, + ) + + +async def _do_scrape(app) -> None: + from .scraper import scrape_docker + from .models import DockerContainer, DockerMetric + + cfg = app.config["PLUGINS"]["docker"] + socket_path = cfg.get("socket_path", "/var/run/docker.sock") + include_stopped = bool(cfg.get("include_stopped", False)) + + try: + containers = await scrape_docker(socket_path, include_stopped) + except ConnectionError as exc: + logger.error("Docker scrape failed: %s", exc) + return + except Exception: + logger.exception("Docker scrape error") + return + + now = datetime.now(timezone.utc) + + async with app.db_sessionmaker() as session: + async with session.begin(): + for c in containers: + # Upsert container state + existing = await session.get(DockerContainer, c["name"]) + if existing is None: + existing = DockerContainer(name=c["name"]) + session.add(existing) + existing.container_id = c["container_id"] + existing.image = c["image"] + existing.status = c["status"] + existing.cpu_pct = c["cpu_pct"] + existing.mem_usage_bytes = c["mem_usage_bytes"] + existing.mem_limit_bytes = c["mem_limit_bytes"] + existing.mem_pct = c["mem_pct"] + existing.restart_count = c["restart_count"] + existing.ports_json = json.dumps(c["ports"]) + existing.started_at = c["started_at"] + existing.scraped_at = now + + # Time-series metric (running containers only) + if c["status"] == "running" and c["cpu_pct"] is not None: + session.add(DockerMetric( + container_name=c["name"], + scraped_at=now, + cpu_pct=c["cpu_pct"], + mem_pct=c["mem_pct"] or 0.0, + mem_usage_bytes=c["mem_usage_bytes"] or 0, + )) + + # Feed alert pipeline + await record_metric( + session=session, + source_module="docker", + resource_name=c["name"], + metric_name="cpu_pct", + value=c["cpu_pct"], + ) + if c["mem_pct"] is not None: + await record_metric( + session=session, + source_module="docker", + resource_name=c["name"], + metric_name="mem_pct", + value=c["mem_pct"], + ) diff --git a/plugins/docker/scraper.py b/plugins/docker/scraper.py new file mode 100644 index 0000000..96f601b --- /dev/null +++ b/plugins/docker/scraper.py @@ -0,0 +1,140 @@ +# plugins/docker/scraper.py +"""Docker API client via Unix socket.""" +from __future__ import annotations +import asyncio +import logging +from datetime import datetime, timezone + +import httpx + +logger = logging.getLogger(__name__) + + +def _short_id(container_id: str) -> str: + return container_id[:12] if container_id else "" + + +def _calc_cpu_pct(stats: dict) -> float: + """Calculate CPU % from a Docker stats snapshot (one-shot).""" + try: + cpu = stats.get("cpu_stats", {}) + precpu = stats.get("precpu_stats", {}) + cpu_delta = ( + cpu["cpu_usage"]["total_usage"] - precpu["cpu_usage"]["total_usage"] + ) + sys_delta = cpu.get("system_cpu_usage", 0) - precpu.get("system_cpu_usage", 0) + num_cpus = cpu.get("online_cpus") or len( + cpu["cpu_usage"].get("percpu_usage") or [None] + ) + if sys_delta <= 0 or cpu_delta < 0: + return 0.0 + return round((cpu_delta / sys_delta) * num_cpus * 100.0, 2) + except (KeyError, TypeError, ZeroDivisionError): + return 0.0 + + +def _calc_mem(stats: dict) -> tuple[int, int, float]: + """Return (usage_bytes, limit_bytes, mem_pct) from Docker stats.""" + try: + mem = stats.get("memory_stats", {}) + usage = mem.get("usage", 0) + limit = mem.get("limit", 0) + # Subtract page cache for working set (cgroup v1: "cache", v2: "inactive_file") + cache = mem.get("stats", {}).get("cache", 0) or \ + mem.get("stats", {}).get("inactive_file", 0) + actual = max(0, usage - cache) + pct = round(actual / limit * 100.0, 2) if limit > 0 else 0.0 + return actual, limit, pct + except (KeyError, TypeError): + return 0, 0, 0.0 + + +def _parse_ports(ports: list) -> list[dict]: + """Normalise Docker port bindings to a compact list.""" + result = [] + for p in (ports or []): + if p.get("PublicPort"): + result.append({ + "host_port": p["PublicPort"], + "container_port": p["PrivatePort"], + "protocol": p.get("Type", "tcp"), + }) + return result + + +async def scrape_docker(socket_path: str, include_stopped: bool) -> list[dict]: + """ + Fetch all containers + resource stats from the Docker daemon. + Returns a list of normalised dicts ready for the scheduler to persist. + Raises ConnectionError if the socket is unreachable. + """ + transport = httpx.AsyncHTTPTransport(uds=socket_path) + try: + async with httpx.AsyncClient( + transport=transport, + base_url="http://docker", + timeout=10.0, + ) as client: + resp = await client.get( + "/containers/json", + params={"all": "true" if include_stopped else "false"}, + ) + resp.raise_for_status() + containers = resp.json() + + async def _get_stats(c: dict) -> tuple[dict, dict | None]: + if c.get("State") != "running": + return c, None + try: + r = await client.get( + f"/containers/{c['Id']}/stats", + params={"stream": "false", "one-shot": "true"}, + timeout=5.0, + ) + return c, r.json() + except Exception as exc: + logger.debug( + "Stats unavailable for %s: %s", _short_id(c.get("Id", "")), exc + ) + return c, None + + pairs = await asyncio.gather(*[_get_stats(c) for c in containers]) + + except httpx.ConnectError as exc: + raise ConnectionError( + f"Cannot connect to Docker socket at {socket_path}: {exc}" + ) from exc + + results = [] + now = datetime.now(timezone.utc) + for container, stats in pairs: + names = container.get("Names") or [] + name = names[0].lstrip("/") if names else _short_id(container.get("Id", "")) + + cpu_pct = mem_usage = mem_limit = mem_pct = None + if stats is not None: + cpu_pct = _calc_cpu_pct(stats) + mem_usage, mem_limit, mem_pct = _calc_mem(stats) + + # Created is a Unix epoch int from /containers/json + created_ts = container.get("Created") + started_at = ( + datetime.fromtimestamp(created_ts, tz=timezone.utc) + if isinstance(created_ts, (int, float)) else None + ) + + results.append({ + "name": name, + "container_id": _short_id(container.get("Id", "")), + "image": container.get("Image", ""), + "status": container.get("State", "unknown"), + "cpu_pct": cpu_pct, + "mem_usage_bytes": mem_usage, + "mem_limit_bytes": mem_limit, + "mem_pct": mem_pct, + "restart_count": 0, # would require /inspect; left as 0 for now + "ports": _parse_ports(container.get("Ports", [])), + "started_at": started_at, + }) + + return results diff --git a/plugins/docker/templates/docker/index.html b/plugins/docker/templates/docker/index.html new file mode 100644 index 0000000..42c9bb5 --- /dev/null +++ b/plugins/docker/templates/docker/index.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} +{% block title %}Docker — Fabled Scryer{% endblock %} +{% block content %} +
+

Docker

+ {% include "_time_range.html" %} +
+ +
+
Loading...
+
+{% endblock %} diff --git a/plugins/docker/templates/docker/rows.html b/plugins/docker/templates/docker/rows.html new file mode 100644 index 0000000..3b6c84e --- /dev/null +++ b/plugins/docker/templates/docker/rows.html @@ -0,0 +1,86 @@ +{# docker/rows.html — HTMX fragment for the Docker main page #} + +{# ── Summary strip ─────────────────────────────────────────────────────────── #} +
+
+
Running
+ {{ running }} +
+
+
Stopped
+ {{ stopped }} +
+
+
Total
+ {{ container_data | length }} +
+
+ +{# ── Container table ─────────────────────────────────────────────────────── #} +{% if container_data %} +
+ + + + + + + + + + + + + + {% for item in container_data %} + {% set c = item.container %} + + + + + + + + + + {% endfor %} + +
ContainerImagePortsCPU %CPU historyMem %Mem history
+
+ +
+
{{ c.name }}
+
{{ c.status }}
+
+
+ 		+ {{ c.image }} + + {% for p in item.ports %} +
{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}
+ {% endfor %} + 		+ {% if c.cpu_pct is not none %} + + {{ "%.1f" | format(c.cpu_pct) }}% + + {% else %} + — + {% endif %} + {{ item.sparkline_cpu | safe }} + {% if c.mem_pct is not none %} + + {{ "%.1f" | format(c.mem_pct) }}% + + {% else %} + — + {% endif %} + {{ item.sparkline_mem | safe }}
+
+{% else %} +
+
+ No containers found. Make sure the Docker socket is accessible and the plugin is configured correctly. +
+
+{% endif %} diff --git a/plugins/docker/templates/docker/widget.html b/plugins/docker/templates/docker/widget.html new file mode 100644 index 0000000..427018f --- /dev/null +++ b/plugins/docker/templates/docker/widget.html @@ -0,0 +1,35 @@ +{# docker/widget.html — dashboard widget: container status overview #} +{% if not containers and running_count == 0 and stopped_count == 0 %} +
No containers found.
+{% else %} + +
+
+ {{ running_count }} + running +
+ {% if stopped_count %} +
+ {{ stopped_count }} + stopped +
+ {% endif %} +
+ +
+ {% for c in containers %} +
+ + + {{ c.name }} + + {% if c.cpu_pct is not none %} + + {{ "%.1f" | format(c.cpu_pct) }}% + + {% endif %} +
+ {% endfor %} +
+ +{% endif %} diff --git a/plugins/docker/templates/docker/widget_resources.html b/plugins/docker/templates/docker/widget_resources.html new file mode 100644 index 0000000..39c90dc --- /dev/null +++ b/plugins/docker/templates/docker/widget_resources.html @@ -0,0 +1,36 @@ +{# docker/widget_resources.html — dashboard widget: CPU + memory bars for running containers #} +{% if not containers %} +
No running containers.
+{% else %} +
+ {% for c in containers %} +
+
+ + {{ c.name }} + + + {% if c.cpu_pct is not none %}CPU {{ "%.1f" | format(c.cpu_pct) }}%{% endif %} + {% if c.mem_pct is not none %} · Mem {{ "%.1f" | format(c.mem_pct) }}%{% endif %} + +
+ {% if c.cpu_pct is not none %} +
+
+
+
+ {% endif %} + {% if c.mem_pct is not none %} +
+
+
+
+ {% endif %} +
+ {% endfor %} +
+{% endif %} diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py new file mode 100644 index 0000000..6a7b229 --- /dev/null +++ b/plugins/host_agent/__init__.py @@ -0,0 +1,23 @@ +# plugins/host_agent/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + + +def get_scheduled_tasks() -> list: + from .scheduler import make_task + return [make_task(_app)] + + +def get_blueprint(): + from .routes import host_agent_bp + return host_agent_bp diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py new file mode 100644 index 0000000..34f52d6 --- /dev/null +++ b/plugins/host_agent/agent.py @@ -0,0 +1,370 @@ +# plugins/host_agent/agent.py +"""Steward host agent — pushes resource metrics to a Steward instance. + +Python 3.8+ stdlib only. Target ~300 lines. Served to targets at +GET /plugins/host_agent/agent.py. +""" +from __future__ import annotations + +import json +import os +import shutil +import signal +import socket +import sys +import time +import urllib.error +import urllib.request +from collections import deque +from datetime import datetime, timezone + +AGENT_VERSION = "1.0.0" + + +class ConfigError(Exception): + pass + + +REQUIRED_KEYS = ("url", "token") +INT_KEYS = ("interval_seconds",) +LIST_KEYS = ("mounts",) + + +def read_config(path: str) -> dict: + """Parse a flat `key = value` config file.""" + cfg: dict = {} + try: + with open(path, "r", encoding="utf-8") as f: + for lineno, raw in enumerate(f, 1): + line = raw.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + raise ConfigError(f"{path}:{lineno}: expected 'key = value'") + key, _, value = line.partition("=") + key = key.strip() + value = value.strip() + if key in INT_KEYS: + try: + cfg[key] = int(value) + except ValueError: + raise ConfigError(f"{path}:{lineno}: {key} must be int") + elif key in LIST_KEYS: + cfg[key] = [v.strip() for v in value.split(",") if v.strip()] + else: + cfg[key] = value + except FileNotFoundError: + raise ConfigError(f"{path}: not found") + + missing = [k for k in REQUIRED_KEYS if k not in cfg] + if missing: + raise ConfigError(f"{path}: missing required key(s): {', '.join(missing)}") + + cfg.setdefault("interval_seconds", 30) + return cfg + + +# ─── collectors ────────────────────────────────────────────────────────────── + +STAT_PATH = "/proc/stat" +MEMINFO_PATH = "/proc/meminfo" +LOADAVG_PATH = "/proc/loadavg" +UPTIME_PATH = "/proc/uptime" +OS_RELEASE_PATH = "/etc/os-release" + + +def _read_file(path: str) -> str: + with open(path, "r", encoding="utf-8") as f: + return f.read() + + +def _parse_cpu_line(stat_text: str) -> tuple[int, int]: + """Return (total_jiffies, idle_jiffies) for the aggregate cpu line.""" + for line in stat_text.splitlines(): + if line.startswith("cpu "): + parts = line.split() + fields = [int(x) for x in parts[1:]] + idle = fields[3] + (fields[4] if len(fields) > 4 else 0) # idle + iowait + total = sum(fields) + return total, idle + raise RuntimeError("no aggregate cpu line in /proc/stat") + + +def collect_cpu(sample_window: float = 0.2) -> float: + """Return CPU utilization % over sample_window seconds.""" + t1, i1 = _parse_cpu_line(_read_file(STAT_PATH)) + time.sleep(sample_window) + t2, i2 = _parse_cpu_line(_read_file(STAT_PATH)) + dt = t2 - t1 + di = i2 - i1 + if dt <= 0: + return 0.0 + return round(100.0 * (dt - di) / dt, 2) + + +def collect_memory() -> dict: + info: dict[str, int] = {} + for line in _read_file(MEMINFO_PATH).splitlines(): + if ":" not in line: + continue + key, _, rest = line.partition(":") + parts = rest.strip().split() + if not parts: + continue + try: + value_kb = int(parts[0]) + except ValueError: + continue + info[key] = value_kb * 1024 # bytes + total = info.get("MemTotal", 0) + available = info.get("MemAvailable", 0) + swap_total = info.get("SwapTotal", 0) + swap_free = info.get("SwapFree", 0) + return { + "total_bytes": total, + "used_bytes": max(total - available, 0), + "available_bytes": available, + "swap_used_bytes": max(swap_total - swap_free, 0), + } + + +def collect_storage(mounts: list[str]) -> list[dict]: + out: list[dict] = [] + for m in mounts: + try: + u = shutil.disk_usage(m) + except (FileNotFoundError, PermissionError): + continue + out.append({"mount": m, "total_bytes": u.total, "used_bytes": u.used}) + return out + + +def collect_load() -> dict: + parts = _read_file(LOADAVG_PATH).split() + return {"1m": float(parts[0]), "5m": float(parts[1]), "15m": float(parts[2])} + + +def collect_uptime() -> int: + return int(float(_read_file(UPTIME_PATH).split()[0])) + + +def collect_metadata() -> dict: + u = os.uname() + distro = "unknown" + try: + for line in _read_file(OS_RELEASE_PATH).splitlines(): + if line.startswith("PRETTY_NAME="): + distro = line.split("=", 1)[1].strip().strip('"') + break + except (OSError, FileNotFoundError): + pass + return {"kernel": u.release, "distro": distro, "arch": u.machine} + + +def default_mounts() -> list[str]: + """Return real mount points from /proc/mounts, skipping pseudo filesystems.""" + skip_types = {"tmpfs", "devtmpfs", "proc", "sysfs", "cgroup", "cgroup2", + "overlay", "squashfs", "ramfs", "devpts", "mqueue", "pstore", + "securityfs", "debugfs", "tracefs", "hugetlbfs", "fusectl", + "configfs", "bpf", "autofs", "efivarfs", "binfmt_misc"} + mounts: list[str] = [] + try: + with open("/proc/mounts", "r") as f: + for line in f: + parts = line.split() + if len(parts) < 3: + continue + if parts[2] in skip_types: + continue + mounts.append(parts[1]) + except OSError: + return ["/"] + return mounts or ["/"] + + +# ─── ring buffer ───────────────────────────────────────────────────────────── + + +class RingBuffer: + def __init__(self, maxlen: int = 20) -> None: + self._dq: deque = deque(maxlen=maxlen) + + def push(self, item) -> None: + self._dq.append(item) + + def drain(self) -> list: + out = list(self._dq) + self._dq.clear() + return out + + def __len__(self) -> int: + return len(self._dq) + + +# ─── payload ───────────────────────────────────────────────────────────────── + + +def build_sample(mounts: list[str]) -> dict: + """Collect one full sample. Partial samples allowed if a collector fails.""" + sample: dict = {"ts": datetime.now(timezone.utc).isoformat()} + try: + sample["cpu_pct"] = collect_cpu() + except Exception: + sample["cpu_pct"] = None + try: + sample["mem"] = collect_memory() + except Exception: + sample["mem"] = None + try: + sample["load"] = collect_load() + except Exception: + sample["load"] = None + try: + sample["uptime_secs"] = collect_uptime() + except Exception: + sample["uptime_secs"] = None + try: + sample["storage"] = collect_storage(mounts) + except Exception: + sample["storage"] = [] + return sample + + +def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict: + return { + "agent_version": AGENT_VERSION, + "hostname": hostname, + "metadata": metadata, + "samples": samples, + } + + +# ─── POST + backoff ────────────────────────────────────────────────────────── + + +BACKOFF_CAP = 300 + + +def next_backoff(current: int) -> int: + if current <= 0: + return 30 + return min(current * 2, BACKOFF_CAP) + + +def post_payload(url: str, token: str, payload: dict) -> tuple[bool, int | None]: + body = json.dumps(payload).encode("utf-8") + req = urllib.request.Request( + url.rstrip("/") + "/plugins/host_agent/ingest", + data=body, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + "User-Agent": f"steward-host-agent/{AGENT_VERSION}", + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + return (200 <= resp.status < 300, resp.status) + except urllib.error.HTTPError as e: + return (False, e.code) + except (urllib.error.URLError, TimeoutError, socket.timeout, OSError): + return (False, None) + + +# ─── main loop ─────────────────────────────────────────────────────────────── + + +_reload_requested = False +_shutdown_requested = False + + +def _handle_hup(_signum, _frame): + global _reload_requested + _reload_requested = True + + +def _handle_term(_signum, _frame): + global _shutdown_requested + _shutdown_requested = True + + +def _log(level: str, msg: str) -> None: + sys.stderr.write(f"[{level}] {msg}\n") + sys.stderr.flush() + + +def main_loop(conf_path: str) -> int: + global _reload_requested, _shutdown_requested + signal.signal(signal.SIGHUP, _handle_hup) + signal.signal(signal.SIGTERM, _handle_term) + + try: + cfg = read_config(conf_path) + except ConfigError as e: + _log("ERROR", str(e)) + return 2 + + metadata = collect_metadata() + hostname = cfg.get("hostname") or socket.gethostname() + mounts = cfg.get("mounts") or default_mounts() + buffer = RingBuffer(maxlen=20) + backoff = 0 + + _log("INFO", f"steward-host-agent {AGENT_VERSION} starting " + f"(url={cfg['url']}, interval={cfg['interval_seconds']}s)") + + while not _shutdown_requested: + if _reload_requested: + try: + cfg = read_config(conf_path) + mounts = cfg.get("mounts") or default_mounts() + _log("INFO", "config reloaded") + except ConfigError as e: + _log("ERROR", f"reload failed: {e}") + _reload_requested = False + + sample = build_sample(mounts) + buffered = buffer.drain() + payload = build_payload( + samples=buffered + [sample], + hostname=hostname, + metadata=metadata, + ) + ok, status = post_payload(cfg["url"], cfg["token"], payload) + + if ok: + if buffered: + _log("INFO", f"flushed {len(buffered)} buffered samples") + backoff = 0 + sleep_for = cfg["interval_seconds"] + else: + if status == 400: + _log("ERROR", "server rejected payload (400) — dropping sample") + elif status == 401: + _log("ERROR", "token rejected (401) — check config + UI") + buffer.push(sample) + else: + _log("WARN", f"POST failed (status={status}); buffering") + for s in buffered: + buffer.push(s) + buffer.push(sample) + backoff = next_backoff(backoff) + sleep_for = backoff + + slept = 0.0 + while slept < sleep_for and not _shutdown_requested and not _reload_requested: + time.sleep(min(1.0, sleep_for - slept)) + slept += 1.0 + + _log("INFO", "SIGTERM — flushing and exiting") + final = buffer.drain() + if final: + post_payload(cfg["url"], cfg["token"], + build_payload(final, hostname, metadata)) + return 0 + + +if __name__ == "__main__": + conf = os.environ.get("STEWARD_AGENT_CONFIG", "/etc/steward-agent.conf") + sys.exit(main_loop(conf)) diff --git a/plugins/host_agent/migrations/__init__.py b/plugins/host_agent/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/host_agent/migrations/env.py b/plugins/host_agent/migrations/env.py new file mode 100644 index 0000000..6545c0e --- /dev/null +++ b/plugins/host_agent/migrations/env.py @@ -0,0 +1,70 @@ +# plugins/host_agent/migrations/env.py +"""Alembic env.py for the host_agent plugin.""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from steward.models.base import Base +import steward.models # noqa: F401 +from plugins.host_agent.models import HostAgentRegistration # noqa: F401 + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("STEWARD_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/host_agent/migrations/versions/__init__.py b/plugins/host_agent/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/host_agent/migrations/versions/host_agent_001_initial.py b/plugins/host_agent/migrations/versions/host_agent_001_initial.py new file mode 100644 index 0000000..d241257 --- /dev/null +++ b/plugins/host_agent/migrations/versions/host_agent_001_initial.py @@ -0,0 +1,36 @@ +# plugins/host_agent/migrations/versions/host_agent_001_initial.py +"""host_agent plugin initial tables + +Revision ID: host_agent_001_initial +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "host_agent_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "host_agent" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "host_agent_registrations", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), + nullable=False, unique=True), + sa.Column("token_hash", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("token_created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("agent_version", sa.String(32), nullable=True), + sa.Column("kernel", sa.String(128), nullable=True), + sa.Column("distro", sa.String(128), nullable=True), + sa.Column("arch", sa.String(32), nullable=True), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("host_agent_registrations") diff --git a/plugins/host_agent/models.py b/plugins/host_agent/models.py new file mode 100644 index 0000000..b93d725 --- /dev/null +++ b/plugins/host_agent/models.py @@ -0,0 +1,32 @@ +# plugins/host_agent/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import Column, String, DateTime, ForeignKey +from steward.models.base import Base + + +def _uuid() -> str: + return str(uuid.uuid4()) + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +class HostAgentRegistration(Base): + __tablename__ = "host_agent_registrations" + + id = Column(String(36), primary_key=True, default=_uuid) + host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"), + unique=True, nullable=False) + token_hash = Column(String(64), nullable=False, unique=True, index=True) + token_created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) + agent_version = Column(String(32), nullable=True) + kernel = Column(String(128), nullable=True) + distro = Column(String(128), nullable=True) + arch = Column(String(32), nullable=True) + last_seen_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) + updated_at = Column(DateTime(timezone=True), nullable=False, + default=_utcnow, onupdate=_utcnow) diff --git a/plugins/host_agent/plugin.yaml b/plugins/host_agent/plugin.yaml new file mode 100644 index 0000000..09b9022 --- /dev/null +++ b/plugins/host_agent/plugin.yaml @@ -0,0 +1,19 @@ +# plugins/host_agent/plugin.yaml +name: host_agent +version: "1.0.0" +description: "Remote Linux host resource monitoring via a lightweight Python push agent (CPU, memory, storage, load, uptime)" +author: "Steward" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/host_agent" +tags: + - host + - monitoring + - cpu + - memory + - storage + +config: + stale_after_seconds: 180 + default_interval_seconds: 30 diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py new file mode 100644 index 0000000..f023811 --- /dev/null +++ b/plugins/host_agent/routes.py @@ -0,0 +1,398 @@ +# plugins/host_agent/routes.py +from __future__ import annotations + +import hashlib +from datetime import datetime, timezone +from typing import Any + +from pathlib import Path + +import secrets +from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for +from steward.auth.middleware import require_role +from steward.models.users import UserRole +from sqlalchemy import select, func +from datetime import timedelta + +from steward.core.settings import public_base_url +from steward.models.hosts import Host +from steward.models.metrics import PluginMetric +from .models import HostAgentRegistration + +host_agent_bp = Blueprint("host_agent", __name__, template_folder="templates") + +SOURCE_MODULE = "host_agent" + + +def _hash_token(raw: str) -> str: + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def _parse_ts(ts: str) -> datetime: + if ts.endswith("Z"): + ts = ts[:-1] + "+00:00" + return datetime.fromisoformat(ts) + + +def _expand_sample_to_metrics( + sample: dict, host_name: str, recorded_at: datetime +) -> list[PluginMetric]: + rows: list[PluginMetric] = [] + + def row(metric: str, resource: str, value: float) -> None: + rows.append(PluginMetric( + source_module=SOURCE_MODULE, + resource_name=resource, + metric_name=metric, + value=float(value), + recorded_at=recorded_at, + )) + + if sample.get("cpu_pct") is not None: + row("cpu_pct", host_name, sample["cpu_pct"]) + + mem = sample.get("mem") or {} + if mem.get("total_bytes"): + total = mem["total_bytes"] + available = mem.get("available_bytes", 0) + used_pct = 100.0 * (total - available) / total if total else 0.0 + row("mem_used_pct", host_name, used_pct) + row("mem_available_bytes", host_name, available) + row("swap_used_bytes", host_name, mem.get("swap_used_bytes", 0)) + + load = sample.get("load") or {} + for key in ("1m", "5m", "15m"): + if key in load: + row(f"load_{key}", host_name, load[key]) + + if sample.get("uptime_secs") is not None: + row("uptime_secs", host_name, sample["uptime_secs"]) + + worst_pct = 0.0 + for disk in sample.get("storage") or []: + total = disk.get("total_bytes", 0) + used = disk.get("used_bytes", 0) + pct = 100.0 * used / total if total else 0.0 + mount_res = f"{host_name}:{disk['mount']}" + row("disk_used_pct", mount_res, pct) + row("disk_used_bytes", mount_res, used) + row("disk_total_bytes", mount_res, total) + if pct > worst_pct: + worst_pct = pct + if sample.get("storage"): + row("disk_used_pct_worst", host_name, worst_pct) + + return rows + + +def _error(status: int, code: str, detail: str | None = None) -> tuple[Any, int]: + body: dict = {"ok": False, "error": code} + if detail: + body["detail"] = detail + return jsonify(body), status + + +@host_agent_bp.post("/ingest") +async def ingest(): + auth = request.headers.get("Authorization", "") + if not auth.startswith("Bearer "): + return _error(401, "invalid_token") + raw = auth[len("Bearer "):].strip() + if not raw: + return _error(401, "invalid_token") + + try: + payload = await request.get_json(force=True) + except Exception: + return _error(400, "malformed_payload", "invalid JSON") + if not isinstance(payload, dict) or "samples" not in payload: + return _error(400, "malformed_payload", "missing 'samples'") + + samples = payload.get("samples") or [] + if not isinstance(samples, list) or not samples: + return _error(400, "malformed_payload", "'samples' must be a non-empty list") + + token_hash = _hash_token(raw) + async with current_app.db_sessionmaker() as session: + reg = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.token_hash == token_hash) + )).scalar_one_or_none() + if reg is None: + return _error(401, "invalid_token") + host = (await session.execute( + select(Host).where(Host.id == reg.host_id) + )).scalar_one_or_none() + if host is None: + return _error(401, "invalid_token") + + accepted = 0 + latest_ts: datetime | None = None + for sample in samples: + try: + recorded_at = _parse_ts(sample["ts"]) + except (KeyError, ValueError): + continue + metrics = _expand_sample_to_metrics(sample, host.name, recorded_at) + for m in metrics: + session.add(m) + accepted += 1 + if latest_ts is None or recorded_at > latest_ts: + latest_ts = recorded_at + + if accepted == 0: + await session.rollback() + return _error(400, "malformed_payload", "no valid samples") + + md = payload.get("metadata") or {} + changed = False + if latest_ts and (reg.last_seen_at is None or latest_ts > reg.last_seen_at): + reg.last_seen_at = latest_ts + changed = True + version = payload.get("agent_version") + if version and reg.agent_version != version: + reg.agent_version = version + changed = True + for field in ("kernel", "distro", "arch"): + if field in md and getattr(reg, field) != md[field]: + setattr(reg, field, md[field]) + changed = True + if changed: + reg.updated_at = datetime.now(timezone.utc) + + if latest_ts: + skew = abs((datetime.now(timezone.utc) - latest_ts).total_seconds()) + if skew > 300: + current_app.logger.warning( + "host_agent ingest: clock skew %.0fs for host=%s", skew, host.name) + + await session.commit() + + return jsonify({"ok": True, "samples_accepted": accepted}), 200 + + +AGENT_SOURCE_PATH = Path(__file__).parent / "agent.py" + + +def _agent_version() -> str: + try: + for line in AGENT_SOURCE_PATH.read_text().splitlines(): + if line.startswith("AGENT_VERSION"): + return line.split("=", 1)[1].strip().strip('"').strip("'") + except OSError: + pass + return "unknown" + + +@host_agent_bp.get("/install.sh") +async def install_script(): + token = request.args.get("token", "") + if not token: + return _error(401, "invalid_token") + async with current_app.db_sessionmaker() as session: + reg = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.token_hash == _hash_token(token)) + )).scalar_one_or_none() + if reg is None: + return _error(401, "invalid_token") + host = (await session.execute( + select(Host).where(Host.id == reg.host_id) + )).scalar_one_or_none() + if host is None: + return _error(401, "invalid_token") + + url = public_base_url(request) + + rendered = await render_template( + "install.sh.j2", + url=url, + token=token, + agent_version=_agent_version(), + host_name=host.name, + host_address=host.address, + ) + return Response(rendered, mimetype="text/plain") + + +@host_agent_bp.get("/agent.py") +async def agent_source(): + try: + body = AGENT_SOURCE_PATH.read_text(encoding="utf-8") + except OSError: + return _error(500, "agent_missing") + return Response(body, mimetype="text/x-python") + + +@host_agent_bp.get("/widget") +async def widget_table(): + """Fleet-glance table: one row per monitored host, latest metrics.""" + async with current_app.db_sessionmaker() as session: + regs = (await session.execute(select(HostAgentRegistration))).scalars().all() + host_ids = [r.host_id for r in regs] + hosts = { + h.id: h for h in (await session.execute( + select(Host).where(Host.id.in_(host_ids)) + )).scalars().all() + } if host_ids else {} + + subq = ( + select( + PluginMetric.resource_name, + PluginMetric.metric_name, + func.max(PluginMetric.recorded_at).label("max_ts"), + ) + .where(PluginMetric.source_module == SOURCE_MODULE) + .group_by(PluginMetric.resource_name, PluginMetric.metric_name) + ).subquery() + latest_rows = (await session.execute( + select(PluginMetric).join( + subq, + (PluginMetric.resource_name == subq.c.resource_name) & + (PluginMetric.metric_name == subq.c.metric_name) & + (PluginMetric.recorded_at == subq.c.max_ts), + ).where(PluginMetric.source_module == SOURCE_MODULE) + )).scalars().all() + + latest: dict[str, dict[str, float]] = {} + for row in latest_rows: + if ":" in row.resource_name: + continue + latest.setdefault(row.resource_name, {})[row.metric_name] = row.value + + rows = [] + for reg in regs: + host = hosts.get(reg.host_id) + if host is None: + continue + m = latest.get(host.name, {}) + rows.append({ + "host": host, + "reg": reg, + "cpu_pct": m.get("cpu_pct"), + "mem_used_pct": m.get("mem_used_pct"), + "disk_worst": m.get("disk_used_pct_worst"), + "load_1m": m.get("load_1m"), + }) + + return await render_template("widget_table.html", rows=rows) + + +@host_agent_bp.get("/widget/history") +async def widget_history(): + host_id = request.args.get("host_id", "") + hours = int(request.args.get("hours", "6")) + cutoff = datetime.now(timezone.utc) - timedelta(hours=hours) + + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return _error(404, "not_found") + points = (await session.execute( + select(PluginMetric).where( + PluginMetric.source_module == SOURCE_MODULE, + PluginMetric.resource_name == host.name, + PluginMetric.metric_name.in_( + ("cpu_pct", "mem_used_pct", "disk_used_pct_worst")), + PluginMetric.recorded_at >= cutoff, + ).order_by(PluginMetric.recorded_at) + )).scalars().all() + + series: dict[str, list[dict]] = {"cpu_pct": [], "mem_used_pct": [], "disk_used_pct_worst": []} + for p in points: + series[p.metric_name].append({"t": p.recorded_at.isoformat(), "v": p.value}) + + return await render_template("widget_history.html", host=host, series=series, hours=hours) + + +def _new_token_pair() -> tuple[str, str]: + raw = secrets.token_urlsafe(32) + return raw, _hash_token(raw) + + +@host_agent_bp.get("/settings/") +@require_role(UserRole.admin) +async def settings_list(): + async with current_app.db_sessionmaker() as session: + regs = (await session.execute(select(HostAgentRegistration))).scalars().all() + hosts_by_id = { + h.id: h for h in (await session.execute( + select(Host).where(Host.id.in_([r.host_id for r in regs])))).scalars().all() + } if regs else {} + + new_token = request.args.get("new_token") + new_host_id = request.args.get("host_id") + install_url = None + if new_token and new_host_id: + install_url = f"{public_base_url(request)}/plugins/host_agent/install.sh?token={new_token}" + + return await render_template( + "settings_list.html", + registrations=[ + {"reg": r, "host": hosts_by_id.get(r.host_id)} for r in regs + ], + new_token=new_token, + install_url=install_url, + ) + + +@host_agent_bp.post("/settings/add-host") +@require_role(UserRole.admin) +async def add_host(): + form = await request.form + name = (form.get("name") or "").strip() + address = (form.get("address") or "").strip() + if not name: + return _error(400, "missing_name") + + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.name == name))).scalar_one_or_none() + if host is None: + host = Host(name=name, address=address) + session.add(host) + await session.flush() + existing = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host.id))).scalar_one_or_none() + if existing is not None: + await session.rollback() + return _error(400, "already_registered") + + raw, hashed = _new_token_pair() + reg = HostAgentRegistration(host_id=host.id, token_hash=hashed) + session.add(reg) + await session.commit() + host_id = host.id + + return redirect(url_for("host_agent.settings_list") + f"?new_token={raw}&host_id={host_id}") + + +@host_agent_bp.post("/settings//rotate-token") +@require_role(UserRole.admin) +async def rotate_token(host_id: str): + async with current_app.db_sessionmaker() as session: + reg = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + if reg is None: + return _error(404, "not_found") + raw, hashed = _new_token_pair() + reg.token_hash = hashed + reg.token_created_at = datetime.now(timezone.utc) + await session.commit() + return redirect(url_for("host_agent.settings_list") + f"?new_token={raw}&host_id={host_id}") + + +@host_agent_bp.post("/settings//delete") +@require_role(UserRole.admin) +async def delete_registration(host_id: str): + async with current_app.db_sessionmaker() as session: + reg = (await session.execute( + select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + if reg is not None: + await session.delete(reg) + await session.commit() + return redirect(url_for("host_agent.settings_list")) diff --git a/plugins/host_agent/scheduler.py b/plugins/host_agent/scheduler.py new file mode 100644 index 0000000..85056c1 --- /dev/null +++ b/plugins/host_agent/scheduler.py @@ -0,0 +1,78 @@ +# plugins/host_agent/scheduler.py +from __future__ import annotations +import logging +from datetime import datetime, timedelta, timezone +from typing import Iterable + +from sqlalchemy import select + +from steward.core.scheduler import ScheduledTask +from steward.models.hosts import Host +from .models import HostAgentRegistration + +logger = logging.getLogger(__name__) + + +def _filter_stale( + regs: Iterable, + *, + now: datetime, + stale_after_seconds: int, +) -> list: + """Pure staleness filter: returns the subset with last_seen_at strictly + older than (now - stale_after_seconds). Rows with last_seen_at=None are + never stale (they are unregistered-in-practice).""" + cutoff = now - timedelta(seconds=stale_after_seconds) + return [ + r for r in regs + if r.last_seen_at is not None and r.last_seen_at < cutoff + ] + + +async def find_stale_registrations(app, stale_after_seconds: int = 180) -> list[dict]: + async with app.db_sessionmaker() as session: + all_regs = (await session.execute( + select(HostAgentRegistration) + )).scalars().all() + stale_rows = _filter_stale( + all_regs, + now=datetime.now(timezone.utc), + stale_after_seconds=stale_after_seconds, + ) + if not stale_rows: + return [] + hosts = { + h.id: h for h in (await session.execute( + select(Host).where(Host.id.in_([r.host_id for r in stale_rows])) + )).scalars().all() + } + return [ + { + "host_id": r.host_id, + "host_name": hosts[r.host_id].name if r.host_id in hosts else "?", + "last_seen_at": r.last_seen_at, + } + for r in stale_rows + ] + + +def make_task(app) -> ScheduledTask: + async def _check_stale(): + try: + stale = await find_stale_registrations(app) + except Exception: + logger.exception("host_agent stale check failed") + return + if stale: + logger.info( + "host_agent: %d stale agent(s): %s", + len(stale), + [s["host_name"] for s in stale], + ) + + return ScheduledTask( + name="host_agent_stale_check", + coro_factory=_check_stale, + interval_seconds=60, + run_on_startup=False, + ) diff --git a/plugins/host_agent/templates/install.sh.j2 b/plugins/host_agent/templates/install.sh.j2 new file mode 100644 index 0000000..02d55be --- /dev/null +++ b/plugins/host_agent/templates/install.sh.j2 @@ -0,0 +1,81 @@ +#!/bin/sh +# Steward host agent installer +# Generated for: {{ host_name }} ({{ host_address }}) +# Steward URL: {{ url }} +set -e + +STEWARD_URL="{{ url }}" +AGENT_TOKEN="{{ token }}" +AGENT_VERSION="{{ agent_version }}" + +AGENT_USER="steward-agent" +AGENT_DIR="/usr/local/lib/steward-agent" +CONF_FILE="/etc/steward-agent.conf" +UNIT_FILE="/etc/systemd/system/steward-agent.service" + +# ── preflight ──────────────────────────────────────────────────────────────── +[ "$(id -u)" = "0" ] || { echo "must run as root (use sudo)"; exit 1; } +command -v systemctl >/dev/null 2>&1 || { echo "systemd not found — unsupported init system"; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo "python3 not found — install python3 first"; exit 1; } + +# Handle --uninstall +if [ "${1:-}" = "--uninstall" ]; then + systemctl disable --now steward-agent.service 2>/dev/null || true + rm -f "$UNIT_FILE" "$CONF_FILE" + rm -rf "$AGENT_DIR" + systemctl daemon-reload + # keep the system user — removing it risks orphaned files elsewhere + echo "uninstalled" + exit 0 +fi + +# ── create system user ─────────────────────────────────────────────────────── +if ! id "$AGENT_USER" >/dev/null 2>&1; then + useradd --system --no-create-home --shell /usr/sbin/nologin "$AGENT_USER" +fi + +# ── drop the agent file ────────────────────────────────────────────────────── +mkdir -p "$AGENT_DIR" +curl -sSL "${STEWARD_URL}/plugins/host_agent/agent.py" -o "$AGENT_DIR/agent.py" +chmod 0755 "$AGENT_DIR/agent.py" +chown root:root "$AGENT_DIR/agent.py" + +# ── write config ───────────────────────────────────────────────────────────── +cat > "$CONF_FILE" < "$UNIT_FILE" <Host Agent — Registered Hosts + +{% if new_token %} +
+

New token — copy this install command now

+

+ This is the only time the raw token is shown. Rotate if you lose it. +

+ 
curl -sSL '{{ install_url }}' | sudo sh
+
+{% endif %} + +
+
+
+ + +
+
+ + +
+ +
+
+ +
+ + + + + + + + + {% for item in registrations %} + + + + + + + + {% else %} + + {% endfor %} + +
HostAgent versionDistroLast seenActions
{{ item.host.name if item.host else item.reg.host_id }}{{ item.reg.agent_version or "—" }}{{ item.reg.distro or "—" }}{{ item.reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") if item.reg.last_seen_at else "never" }} +
+ +
+
+ +
+
No hosts registered. Add one above.
+
+{% endblock %} diff --git a/plugins/host_agent/templates/widget_history.html b/plugins/host_agent/templates/widget_history.html new file mode 100644 index 0000000..35e363f --- /dev/null +++ b/plugins/host_agent/templates/widget_history.html @@ -0,0 +1,24 @@ +
+

{{ host.name }} — last {{ hours }}h

+ + +
diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html new file mode 100644 index 0000000..a646fad --- /dev/null +++ b/plugins/host_agent/templates/widget_table.html @@ -0,0 +1,34 @@ +{# host_agent widget — fleet glance, flex rows (no header wrap) #} +{% if rows %} +
+ {% for r in rows %} + {% set stale = not r.reg.last_seen_at %} +
+ + + {{ r.host.name }} + + + {% if r.cpu_pct is not none %} + cpu {{ "%.0f"|format(r.cpu_pct) }}% + {% endif %} + {% if r.mem_used_pct is not none %} + mem {{ "%.0f"|format(r.mem_used_pct) }}% + {% endif %} + {% if r.disk_worst is not none %} + disk {{ "%.0f"|format(r.disk_worst) }}% + {% endif %} + {% if r.load_1m is not none %} + load {{ "%.2f"|format(r.load_1m) }} + {% endif %} + + + {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }} + +
+ {% endfor %} +
+{% else %} +

No hosts with agent data yet.

+{% endif %} diff --git a/plugins/http/__init__.py b/plugins/http/__init__.py new file mode 100644 index 0000000..cb79685 --- /dev/null +++ b/plugins/http/__init__.py @@ -0,0 +1,24 @@ +# plugins/http/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + from .models import HttpMonitor, HttpResult # noqa: F401 + + +def get_scheduled_tasks() -> list: + from .scheduler import make_task + return [make_task(_app)] + + +def get_blueprint(): + from .routes import http_bp + return http_bp diff --git a/plugins/http/checker.py b/plugins/http/checker.py new file mode 100644 index 0000000..7eec743 --- /dev/null +++ b/plugins/http/checker.py @@ -0,0 +1,105 @@ +# plugins/http/checker.py +"""Core HTTP check logic — runs a single monitor check and returns a result dict.""" +from __future__ import annotations +import asyncio +import json +import logging +import ssl +import time +from datetime import datetime, timezone +from urllib.parse import urlparse + +import httpx + +logger = logging.getLogger(__name__) + + +async def _get_tls_expiry(hostname: str, port: int) -> datetime | None: + """Attempt a bare TLS handshake to extract the certificate expiry date.""" + ctx = ssl.create_default_context() + try: + reader, writer = await asyncio.wait_for( + asyncio.open_connection(hostname, port, ssl=ctx), + timeout=5.0, + ) + cert = writer.get_extra_info("ssl_object").getpeercert() + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + exp_str = cert.get("notAfter", "") + if not exp_str: + return None + # Format: "Mar 22 12:00:00 2027 GMT" + return datetime.strptime(exp_str, "%b %d %H:%M:%S %Y %Z").replace( + tzinfo=timezone.utc + ) + except Exception as exc: + logger.debug("TLS check failed for %s:%d — %s", hostname, port, exc) + return None + + +async def run_check( + url: str, + method: str = "GET", + expected_status: int = 200, + content_match: str = "", + headers: dict | None = None, + timeout_seconds: int = 10, + follow_redirects: bool = True, + verify_ssl: bool = True, +) -> dict: + """ + Perform one HTTP check. Returns a dict with: + is_up, status_code, response_ms, content_matched, error_msg, tls_expires_at + """ + result: dict = { + "is_up": False, + "status_code": None, + "response_ms": None, + "content_matched": None, + "error_msg": None, + "tls_expires_at": None, + } + + parsed = urlparse(url) + is_https = parsed.scheme.lower() == "https" + + t0 = time.monotonic() + try: + async with httpx.AsyncClient( + follow_redirects=follow_redirects, + verify=verify_ssl, + timeout=timeout_seconds, + ) as client: + response = await client.request( + method, + url, + headers=headers or {}, + ) + result["response_ms"] = round((time.monotonic() - t0) * 1000, 1) + result["status_code"] = response.status_code + + status_ok = response.status_code == expected_status + if content_match: + matched = content_match in response.text + result["content_matched"] = matched + result["is_up"] = status_ok and matched + else: + result["is_up"] = status_ok + + except httpx.TimeoutException: + result["response_ms"] = round((time.monotonic() - t0) * 1000, 1) + result["error_msg"] = "Timeout" + except httpx.ConnectError as exc: + result["error_msg"] = f"Connection error: {exc}" + except Exception as exc: + result["error_msg"] = str(exc)[:512] + + # TLS expiry — only for HTTPS, and only when the check succeeded or we got a response + if is_https and result["status_code"] is not None: + port = parsed.port or 443 + result["tls_expires_at"] = await _get_tls_expiry(parsed.hostname, port) + + return result diff --git a/plugins/http/migrations/__init__.py b/plugins/http/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/http/migrations/env.py b/plugins/http/migrations/env.py new file mode 100644 index 0000000..fd308a2 --- /dev/null +++ b/plugins/http/migrations/env.py @@ -0,0 +1,70 @@ +# plugins/http/migrations/env.py +"""Alembic env.py for the HTTP monitoring plugin.""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from fabledscryer.models.base import Base +import fabledscryer.models # noqa: F401 +from plugins.http.models import HttpMonitor, HttpResult # noqa: F401 + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("FABLEDSCRYER_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/http/migrations/versions/__init__.py b/plugins/http/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/http/migrations/versions/http_001_initial.py b/plugins/http/migrations/versions/http_001_initial.py new file mode 100644 index 0000000..c546023 --- /dev/null +++ b/plugins/http/migrations/versions/http_001_initial.py @@ -0,0 +1,52 @@ +"""HTTP monitoring plugin initial tables + +Revision ID: http_001_initial +Revises: (none — branch off core via depends_on) +Create Date: 2026-03-22 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "http_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "http" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "http_monitors", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("name", sa.String(128), nullable=False), + sa.Column("url", sa.String(2048), nullable=False), + sa.Column("method", sa.String(8), nullable=False, server_default="GET"), + sa.Column("expected_status", sa.Integer, nullable=False, server_default="200"), + sa.Column("content_match", sa.String(512), nullable=False, server_default=""), + sa.Column("headers_json", sa.Text, nullable=False, server_default="{}"), + sa.Column("timeout_seconds", sa.Integer, nullable=False, server_default="10"), + sa.Column("check_interval_seconds", sa.Integer, nullable=False, server_default="0"), + sa.Column("follow_redirects", sa.Boolean, nullable=False, server_default="1"), + sa.Column("verify_ssl", sa.Boolean, nullable=False, server_default="1"), + sa.Column("enabled", sa.Boolean, nullable=False, server_default="1"), + sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "http_results", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("monitor_id", sa.String(36), nullable=False, index=True), + sa.Column("checked_at", sa.DateTime(timezone=True), nullable=False, index=True), + sa.Column("status_code", sa.Integer, nullable=True), + sa.Column("response_ms", sa.Float, nullable=True), + sa.Column("is_up", sa.Boolean, nullable=False, server_default="0"), + sa.Column("content_matched", sa.Boolean, nullable=True), + sa.Column("error_msg", sa.String(512), nullable=True), + sa.Column("tls_expires_at", sa.DateTime(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + op.drop_table("http_results") + op.drop_table("http_monitors") diff --git a/plugins/http/models.py b/plugins/http/models.py new file mode 100644 index 0000000..180effd --- /dev/null +++ b/plugins/http/models.py @@ -0,0 +1,63 @@ +# plugins/http/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from fabledscryer.models.base import Base + + +class HttpMonitor(Base): + """A configured HTTP endpoint to check periodically.""" + __tablename__ = "http_monitors" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + name: Mapped[str] = mapped_column(String(128), nullable=False) + url: Mapped[str] = mapped_column(String(2048), nullable=False) + method: Mapped[str] = mapped_column(String(8), nullable=False, default="GET") + expected_status: Mapped[int] = mapped_column(Integer, nullable=False, default=200) + # Optional substring to find in the response body (empty = skip check) + content_match: Mapped[str] = mapped_column(String(512), nullable=False, default="") + # JSON dict of extra request headers, e.g. {"Authorization": "Bearer ..."} + headers_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}") + timeout_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=10) + # 0 = use global plugin check_interval_seconds + check_interval_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + follow_redirects: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + verify_ssl: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + # Updated after each check so the scheduler can compute next-run time efficiently + last_checked_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + +class HttpResult(Base): + """One check result for an HttpMonitor.""" + __tablename__ = "http_results" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + monitor_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + checked_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + index=True, + ) + status_code: Mapped[int | None] = mapped_column(Integer, nullable=True) + response_ms: Mapped[float | None] = mapped_column(Float, nullable=True) + is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + # None when no content_match is configured; True/False when it is + content_matched: Mapped[bool | None] = mapped_column(Boolean, nullable=True) + error_msg: Mapped[str | None] = mapped_column(String(512), nullable=True) + # TLS expiry if the endpoint is HTTPS; None for HTTP or on error + tls_expires_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) diff --git a/plugins/http/plugin.yaml b/plugins/http/plugin.yaml new file mode 100644 index 0000000..9aa1475 --- /dev/null +++ b/plugins/http/plugin.yaml @@ -0,0 +1,19 @@ +name: http +version: "1.0.0" +description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry" +author: "FabledScryer" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/http" +tags: + - monitoring + - http + - synthetic + - uptime + +config: + check_interval_seconds: 60 + default_timeout_seconds: 10 + follow_redirects: true + verify_ssl: true diff --git a/plugins/http/routes.py b/plugins/http/routes.py new file mode 100644 index 0000000..eea5ce7 --- /dev/null +++ b/plugins/http/routes.py @@ -0,0 +1,229 @@ +# plugins/http/routes.py +from __future__ import annotations +import json +import uuid +from datetime import datetime, timezone +from quart import Blueprint, current_app, render_template, request, redirect, url_for +from sqlalchemy import Integer, cast, func, select + +from fabledscryer.auth.middleware import require_role +from fabledscryer.models.users import UserRole +from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds +from .models import HttpMonitor, HttpResult + +http_bp = Blueprint("http", __name__, template_folder="templates") + + +def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: + if len(values) < 2: + return f'' + mn, mx = min(values), max(values) + if mx == mn: + mx = mn + 1.0 + step = width / (len(values) - 1) + pts = [] + for i, v in enumerate(values): + x = i * step + y = height - (v - mn) / (mx - mn) * (height - 2) - 1 + pts.append(f"{x:.1f},{y:.1f}") + poly = " ".join(pts) + return ( + f'' + f'' + f'' + ) + + +async def _render_rows(range_key: str = DEFAULT_RANGE): + since, range_key = parse_range(range_key) + b_secs = bucket_seconds(since) + bucket_col = ( + cast(func.extract('epoch', HttpResult.checked_at), Integer) / b_secs + ).label("bucket") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(HttpMonitor).order_by(HttpMonitor.created_at) + ) + monitors = list(result.scalars()) + + # Latest result per monitor + latest_map: dict[str, HttpResult] = {} + histories: dict[str, list] = {} + for m in monitors: + r = await db.execute( + select(HttpResult) + .where(HttpResult.monitor_id == m.id) + .order_by(HttpResult.checked_at.desc()) + .limit(1) + ) + latest = r.scalar_one_or_none() + if latest: + latest_map[m.id] = latest + + r2 = await db.execute( + select( + func.avg(HttpResult.response_ms).label("response_ms"), + func.min(HttpResult.is_up.cast(Integer)).label("had_down"), + bucket_col, + ) + .where(HttpResult.monitor_id == m.id) + .where(HttpResult.checked_at >= since) + .group_by(bucket_col) + .order_by(bucket_col) + ) + histories[m.id] = r2.all() + + monitor_data = [] + for m in monitors: + hist = histories.get(m.id, []) + latest = latest_map.get(m.id) + now = datetime.now(timezone.utc) + tls_days = None + if latest and latest.tls_expires_at: + tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0 + monitor_data.append({ + "monitor": m, + "latest": latest, + "tls_days": tls_days, + "sparkline_ms": _sparkline([r.response_ms or 0 for r in hist]), + }) + + up = sum(1 for d in monitor_data if d["latest"] and d["latest"].is_up) + down = sum(1 for d in monitor_data if d["latest"] and not d["latest"].is_up) + return monitor_data, up, down, range_key + + +@http_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + current_range = request.args.get("range", DEFAULT_RANGE) + return await render_template( + "http/index.html", + poll_interval=poll_interval, + current_range=current_range, + ) + + +@http_bp.get("/rows") +@require_role(UserRole.viewer) +async def rows(): + """HTMX fragment: monitor list with status + sparklines.""" + monitor_data, up, down, range_key = await _render_rows( + request.args.get("range", DEFAULT_RANGE) + ) + return await render_template( + "http/rows.html", + monitor_data=monitor_data, + up=up, + down=down, + range_key=range_key, + ) + + +@http_bp.post("/add") +@require_role(UserRole.operator) +async def add_monitor(): + """Add a new HTTP monitor.""" + form = await request.form + url = form.get("url", "").strip() + if not url: + return redirect(url_for("http.index")) + if not url.startswith(("http://", "https://")): + url = "https://" + url + + monitor = HttpMonitor( + id=str(uuid.uuid4()), + name=form.get("name", "").strip() or url, + url=url, + method=form.get("method", "GET").upper(), + expected_status=int(form.get("expected_status", 200) or 200), + content_match=form.get("content_match", "").strip(), + headers_json="{}", + timeout_seconds=int(form.get("timeout_seconds", 10) or 10), + check_interval_seconds=int(form.get("check_interval_seconds", 0) or 0), + follow_redirects="follow_redirects" in form, + verify_ssl="verify_ssl" in form, + enabled=True, + created_at=datetime.now(timezone.utc), + ) + async with current_app.db_sessionmaker() as db: + async with db.begin(): + db.add(monitor) + + return redirect(url_for("http.index")) + + +@http_bp.post("//delete") +@require_role(UserRole.operator) +async def delete_monitor(monitor_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + m = await db.get(HttpMonitor, monitor_id) + if m: + await db.delete(m) + return redirect(url_for("http.index")) + + +@http_bp.post("//toggle") +@require_role(UserRole.operator) +async def toggle_monitor(monitor_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + m = await db.get(HttpMonitor, monitor_id) + if m: + m.enabled = not m.enabled + return redirect(url_for("http.index")) + + +@http_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """HTMX dashboard widget: up/down counts + monitor list.""" + show_down_only = request.args.get("show_down_only", "no") == "yes" + limit = max(1, min(20, int(request.args.get("limit", 10) or 10))) + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(HttpMonitor) + .where(HttpMonitor.enabled == True) # noqa: E712 + .order_by(HttpMonitor.created_at) + ) + monitors = list(result.scalars()) + + latest_map: dict[str, HttpResult] = {} + for m in monitors: + r = await db.execute( + select(HttpResult) + .where(HttpResult.monitor_id == m.id) + .order_by(HttpResult.checked_at.desc()) + .limit(1) + ) + latest = r.scalar_one_or_none() + if latest: + latest_map[m.id] = latest + + monitor_data = [ + {"monitor": m, "latest": latest_map.get(m.id)} + for m in monitors + ] + + up = sum(1 for d in monitor_data if d["latest"] and d["latest"].is_up) + down = sum(1 for d in monitor_data if d["latest"] and not d["latest"].is_up) + pending = sum(1 for d in monitor_data if not d["latest"]) + + if show_down_only: + monitor_data = [d for d in monitor_data if not (d["latest"] and d["latest"].is_up)] + + return await render_template( + "http/widget.html", + monitor_data=monitor_data[:limit], + up=up, + down=down, + pending=pending, + show_down_only=show_down_only, + widget_id=widget_id, + ) diff --git a/plugins/http/scheduler.py b/plugins/http/scheduler.py new file mode 100644 index 0000000..f9edd67 --- /dev/null +++ b/plugins/http/scheduler.py @@ -0,0 +1,121 @@ +# plugins/http/scheduler.py +from __future__ import annotations +import asyncio +import json +import logging +from datetime import datetime, timezone, timedelta + +from fabledscryer.core.scheduler import ScheduledTask +from fabledscryer.core.alerts import record_metric + +logger = logging.getLogger(__name__) + + +def make_task(app) -> ScheduledTask: + interval = int( + app.config["PLUGINS"]["http"].get("check_interval_seconds", 60) + ) + + async def check(): + await _do_checks(app) + + return ScheduledTask( + name="http_check", + coro_factory=check, + interval_seconds=interval, + run_on_startup=True, + ) + + +async def _do_checks(app) -> None: + from sqlalchemy import select + from .models import HttpMonitor, HttpResult + from .checker import run_check + + cfg = app.config["PLUGINS"]["http"] + global_interval = int(cfg.get("check_interval_seconds", 60)) + global_timeout = int(cfg.get("default_timeout_seconds", 10)) + global_follow = bool(cfg.get("follow_redirects", True)) + global_verify = bool(cfg.get("verify_ssl", True)) + + now = datetime.now(timezone.utc) + + async with app.db_sessionmaker() as session: + result = await session.execute( + select(HttpMonitor).where(HttpMonitor.enabled == True) # noqa: E712 + ) + monitors = list(result.scalars()) + + # Filter to monitors whose next check time has passed + due: list[HttpMonitor] = [] + for m in monitors: + effective_interval = m.check_interval_seconds or global_interval + if m.last_checked_at is None: + due.append(m) + elif (now - m.last_checked_at).total_seconds() >= effective_interval: + due.append(m) + + if not due: + return + + async def _check_one(monitor: HttpMonitor) -> tuple[HttpMonitor, dict]: + try: + headers = json.loads(monitor.headers_json or "{}") + except (ValueError, TypeError): + headers = {} + result = await run_check( + url=monitor.url, + method=monitor.method, + expected_status=monitor.expected_status, + content_match=monitor.content_match, + headers=headers, + timeout_seconds=monitor.timeout_seconds or global_timeout, + follow_redirects=monitor.follow_redirects if monitor.follow_redirects is not None else global_follow, + verify_ssl=monitor.verify_ssl if monitor.verify_ssl is not None else global_verify, + ) + return monitor, result + + pairs = await asyncio.gather(*[_check_one(m) for m in due], return_exceptions=True) + + check_time = datetime.now(timezone.utc) + + async with app.db_sessionmaker() as session: + async with session.begin(): + for item in pairs: + if isinstance(item, Exception): + logger.error("HTTP check task error: %s", item) + continue + monitor, res = item + + session.add(HttpResult( + monitor_id=monitor.id, + checked_at=check_time, + status_code=res["status_code"], + response_ms=res["response_ms"], + is_up=res["is_up"], + content_matched=res["content_matched"], + error_msg=res["error_msg"], + tls_expires_at=res["tls_expires_at"], + )) + + # Update monitor's last_checked_at for next-run scheduling + db_monitor = await session.get(HttpMonitor, monitor.id) + if db_monitor: + db_monitor.last_checked_at = check_time + + # Alert pipeline + if res["response_ms"] is not None: + await record_metric( + session=session, + source_module="http", + resource_name=monitor.name, + metric_name="response_ms", + value=res["response_ms"], + ) + await record_metric( + session=session, + source_module="http", + resource_name=monitor.name, + metric_name="is_up", + value=1.0 if res["is_up"] else 0.0, + ) diff --git a/plugins/http/templates/http/index.html b/plugins/http/templates/http/index.html new file mode 100644 index 0000000..6d1ffd0 --- /dev/null +++ b/plugins/http/templates/http/index.html @@ -0,0 +1,77 @@ +{% extends "base.html" %} +{% block title %}HTTP Monitors — Fabled Scryer{% endblock %} +{% block content %} +
+

HTTP Monitors

+ {% include "_time_range.html" %} +
+ +{# ── Add monitor form ─────────────────────────────────────────────────────── #} +
+
Add Monitor
+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ +
+
+
+ +{# ── Monitor list ─────────────────────────────────────────────────────────── #} +
+
Loading...
+
+{% endblock %} diff --git a/plugins/http/templates/http/rows.html b/plugins/http/templates/http/rows.html new file mode 100644 index 0000000..4666fa3 --- /dev/null +++ b/plugins/http/templates/http/rows.html @@ -0,0 +1,109 @@ +{# http/rows.html — HTMX fragment for HTTP monitor status list #} + +{# ── Summary ─────────────────────────────────────────────────────────────── #} +
+
+
Up
+ {{ up }} +
+
+
Down
+ {{ down }} +
+
+
Monitors
+ {{ monitor_data | length }} +
+
+ +{# ── Monitor list ─────────────────────────────────────────────────────────── #} +{% if monitor_data %} +
+ + + + + + + + + + + + + {% for item in monitor_data %} + {% set m = item.monitor %} + {% set latest = item.latest %} + + + + + + + + + {% endfor %} + +
MonitorStatusResponseHistoryTLS expiry
+
{{ m.name }}
+
+ {{ m.method }} {{ m.url }} +
+ {% if not m.enabled %} + paused + {% endif %} + 		+ {% if not latest %} + pending + {% elif latest.is_up %} +
+ + {{ latest.status_code }} +
+ {% else %} +
+ + + {{ latest.status_code or latest.error_msg or "error" }} + +
+ {% if latest.content_matched == false %} +
content mismatch
+ {% endif %} + {% endif %} + 		+ {% if latest and latest.response_ms is not none %} + + {{ "%.0f" | format(latest.response_ms) }}ms + + {% else %} + — + {% endif %} + {{ item.sparkline_ms | safe }} + {% if item.tls_days is not none %} + + {{ item.tls_days | int }}d + + {% elif m.url.startswith('https://') %} + — + {% endif %} + +
+ +
+
+ +
+
+
+{% else %} +
+
+ No monitors configured yet. Add one using the form above. +
+
+{% endif %} diff --git a/plugins/http/templates/http/widget.html b/plugins/http/templates/http/widget.html new file mode 100644 index 0000000..891f40b --- /dev/null +++ b/plugins/http/templates/http/widget.html @@ -0,0 +1,56 @@ +{# http/widget.html — dashboard widget: HTTP monitor summary #} +{% if not monitor_data and up == 0 and down == 0 and pending == 0 %} +
+ No monitors configured. Add monitors → +
+{% else %} + +
+ {% if up %} +
+ {{ up }} + up +
+ {% endif %} + {% if down %} +
+ {{ down }} + down +
+ {% endif %} + {% if pending %} +
+ {{ pending }} + pending +
+ {% endif %} +
+ +
+ {% for item in monitor_data %} + {% set latest = item.latest %} +
+ {% if not latest %} + + {% elif latest.is_up %} + + {% else %} + + {% endif %} + + {{ item.monitor.name }} + + {% if latest and latest.response_ms is not none %} + + {{ "%.0f" | format(latest.response_ms) }}ms + + {% elif latest and not latest.is_up %} + + {{ latest.status_code or "err" }} + + {% endif %} +
+ {% endfor %} +
+ +{% endif %} diff --git a/plugins/index.yaml b/plugins/index.yaml new file mode 100644 index 0000000..0192afe --- /dev/null +++ b/plugins/index.yaml @@ -0,0 +1,90 @@ +# fabledscryer-plugins / index.yaml +# +# Plugin catalog for Fabled Scryer. +# Fetched by the app at: Settings → Plugins → Plugin Catalog +# +# After updating an entry, commit and push — changes are live within 5 minutes +# (the app caches this file in-process for 300 seconds). +# +# See docs/plugins/index.yaml.example in the main app repo for the full schema. + +version: 1 +updated: "2026-03-22" + +plugins: + + - name: http + version: "1.0.0" + description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry" + author: "FabledScryer" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/http" + download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/http-v1.0.0/http.zip" + checksum_sha256: "" + tags: + - monitoring + - http + - synthetic + - uptime + + - name: docker + version: "1.0.0" + description: "Docker container status, resource usage, and restart tracking via Docker socket" + author: "FabledScryer" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/docker" + download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/docker-v1.0.0/docker.zip" + checksum_sha256: "" + tags: + - containers + - docker + - infrastructure + + - name: traefik + version: "1.0.0" + description: "Traefik reverse proxy metrics and access log integration" + author: "FabledScryer" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/traefik" + download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/traefik-v1.0.0/traefik.zip" + checksum_sha256: "" + tags: + - proxy + - metrics + - access-log + + - name: unifi + version: "1.0.0" + description: "UniFi Network controller integration — WAN health, devices, clients, DPI" + author: "FabledScryer" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/unifi" + download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/unifi-v1.0.0/unifi.zip" + checksum_sha256: "" + tags: + - network + - unifi + - ubiquiti + + - name: ups + version: "1.0.0" + description: "UPS monitoring via NUT (Network UPS Tools) with Ansible shutdown automation" + author: "FabledScryer" + license: "MIT" + min_app_version: "0.1.0" + repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/ups" + download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/ups-v1.0.0/ups.zip" + checksum_sha256: "" + tags: + - ups + - power + - nut diff --git a/plugins/snmp/__init__.py b/plugins/snmp/__init__.py new file mode 100644 index 0000000..8655884 --- /dev/null +++ b/plugins/snmp/__init__.py @@ -0,0 +1,23 @@ +# plugins/snmp/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + + +def get_scheduled_tasks() -> list: + from .scheduler import make_poll_task + return [make_poll_task(_app)] + + +def get_blueprint(): + from .routes import snmp_bp + return snmp_bp diff --git a/plugins/snmp/plugin.yaml b/plugins/snmp/plugin.yaml new file mode 100644 index 0000000..5e98fce --- /dev/null +++ b/plugins/snmp/plugin.yaml @@ -0,0 +1,49 @@ +# plugins/snmp/plugin.yaml +name: snmp +version: "1.0.0" +description: "SNMP polling for network devices — switches, routers, printers, UPSes, etc." +author: "FabledScryer" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/snmp" +tags: + - snmp + - network + - monitoring + +config: + poll_interval_seconds: 60 + # Each device is polled independently. OIDs must resolve to numeric values. + devices: + - name: "core-switch" + host: "192.168.1.1" + port: 161 + community: "public" + version: "2c" # "1", "2c", or "3" + oids: + - oid: "1.3.6.1.2.1.1.3.0" + label: "uptime_centisecs" + - oid: "1.3.6.1.2.1.2.2.1.10.1" + label: "if1_in_octets" + - oid: "1.3.6.1.2.1.2.2.1.16.1" + label: "if1_out_octets" + # Example: network UPS with SNMP management card (RFC 1628 UPS-MIB). + # upsBatteryStatus values: 1=unknown, 2=batteryNormal, 3=batteryLow, 4=batteryDepleted. + # Uncomment and point at your UPS to enable. + # - name: "ups" + # host: "192.168.1.10" + # port: 161 + # community: "public" + # version: "2c" + # oids: + # - oid: "1.3.6.1.2.1.33.1.2.1.0" + # label: "battery_status" + # - oid: "1.3.6.1.2.1.33.1.2.2.0" + # label: "seconds_on_battery" + # - oid: "1.3.6.1.2.1.33.1.2.3.0" + # label: "runtime_minutes_remaining" + # - oid: "1.3.6.1.2.1.33.1.2.4.0" + # label: "battery_charge_pct" + # - oid: "1.3.6.1.2.1.33.1.3.3.1.3.1" + # label: "input_voltage_dV" diff --git a/plugins/snmp/poller.py b/plugins/snmp/poller.py new file mode 100644 index 0000000..2f7e86f --- /dev/null +++ b/plugins/snmp/poller.py @@ -0,0 +1,94 @@ +# plugins/snmp/poller.py +""" +Synchronous SNMP GET helper, run via executor. + +Requires pysnmp-lextudio (maintained pysnmp fork): + pip install 'fabledscryer[snmp]' + +If pysnmp is not installed, poll_device() returns an empty dict and logs a warning. +""" +from __future__ import annotations +import logging + +logger = logging.getLogger(__name__) + + +def _pysnmp_available() -> bool: + try: + import pysnmp # noqa: F401 + return True + except ImportError: + return False + + +def _mp_model(version: str) -> int: + """Map version string to pysnmp mpModel integer.""" + return 0 if version == "1" else 1 + + +def poll_device_sync( + host: str, + port: int, + community: str, + version: str, + oids: list[dict], +) -> dict[str, float]: + """ + Perform SNMP GET for each OID and return {label: float_value}. + Non-numeric OIDs (strings, etc.) are skipped. + Returns empty dict on any error. + """ + if not _pysnmp_available(): + logger.warning("pysnmp not installed — SNMP polling disabled. " + "Install with: pip install 'fabledscryer[snmp]'") + return {} + + from pysnmp.hlapi import ( + CommunityData, + ContextData, + ObjectIdentity, + ObjectType, + SnmpEngine, + UdpTransportTarget, + getCmd, + ) + + results: dict[str, float] = {} + engine = SnmpEngine() + + for oid_cfg in oids: + oid = oid_cfg["oid"] + label = oid_cfg.get("label") or oid + scale = float(oid_cfg.get("scale", 1.0)) + + try: + error_indication, error_status, error_index, var_binds = next( + getCmd( + engine, + CommunityData(community, mpModel=_mp_model(version)), + UdpTransportTarget((host, port), timeout=5, retries=1), + ContextData(), + ObjectType(ObjectIdentity(oid)), + ) + ) + except Exception as exc: + logger.debug("SNMP GET %s@%s OID %s failed: %s", host, port, oid, exc) + continue + + if error_indication: + logger.debug("SNMP error %s@%s OID %s: %s", host, port, oid, error_indication) + continue + if error_status: + logger.debug("SNMP status %s@%s OID %s: %s at %s", + host, port, oid, error_status.prettyPrint(), + error_index and var_binds[int(error_index) - 1][0] or "?") + continue + + for _, val in var_binds: + try: + results[label] = float(val) * scale + except (TypeError, ValueError): + # Non-numeric type (e.g. OctetString description) — skip + logger.debug("SNMP non-numeric value for %s label=%s: %r", oid, label, val) + + return results diff --git a/plugins/snmp/routes.py b/plugins/snmp/routes.py new file mode 100644 index 0000000..cca9697 --- /dev/null +++ b/plugins/snmp/routes.py @@ -0,0 +1,146 @@ +# plugins/snmp/routes.py +from __future__ import annotations +from datetime import datetime, timedelta, timezone + +from quart import Blueprint, current_app, render_template, request +from sqlalchemy import func, select + +from fabledscryer.auth.middleware import require_role +from fabledscryer.models.users import UserRole +from fabledscryer.models.metrics import PluginMetric + +snmp_bp = Blueprint("snmp", __name__, template_folder="templates") + + +async def _latest_readings(db, device_names: list[str]) -> dict[str, dict[str, float]]: + """Return {device_name: {label: latest_value}} for all configured devices.""" + if not device_names: + return {} + + # Latest value per (resource_name, metric_name) pair + subq = ( + select( + PluginMetric.resource_name, + PluginMetric.metric_name, + func.max(PluginMetric.recorded_at).label("latest_at"), + ) + .where(PluginMetric.source_module == "snmp") + .where(PluginMetric.resource_name.in_(device_names)) + .group_by(PluginMetric.resource_name, PluginMetric.metric_name) + .subquery() + ) + result = await db.execute( + select(PluginMetric).join( + subq, + (PluginMetric.resource_name == subq.c.resource_name) + & (PluginMetric.metric_name == subq.c.metric_name) + & (PluginMetric.recorded_at == subq.c.latest_at), + ).where(PluginMetric.source_module == "snmp") + ) + rows = result.scalars().all() + + out: dict[str, dict[str, float]] = {} + for row in rows: + out.setdefault(row.resource_name, {})[row.metric_name] = row.value + return out + + +async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]: + """Return {label: [(recorded_at, value), ...]} for a single device.""" + since = datetime.now(timezone.utc) - timedelta(hours=hours) + result = await db.execute( + select(PluginMetric) + .where( + PluginMetric.source_module == "snmp", + PluginMetric.resource_name == device_name, + PluginMetric.recorded_at >= since, + ) + .order_by(PluginMetric.recorded_at) + ) + rows = result.scalars().all() + + out: dict[str, list] = {} + for row in rows: + out.setdefault(row.metric_name, []).append( + (row.recorded_at.isoformat(), row.value) + ) + return out + + +@snmp_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) + device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg] + + async with current_app.db_sessionmaker() as db: + latest = await _latest_readings(db, device_names) + + # Merge config + latest readings for the template + devices = [] + for cfg in devices_cfg: + name = cfg.get("name") or cfg.get("host", "?") + devices.append({ + "name": name, + "host": cfg.get("host", ""), + "oids": cfg.get("oids", []), + "readings": latest.get(name, {}), + }) + + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + return await render_template("snmp/index.html", + devices=devices, + poll_interval=poll_interval) + + +@snmp_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) + device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg] + + async with current_app.db_sessionmaker() as db: + latest = await _latest_readings(db, device_names[:10]) + + devices = [] + for cfg in devices_cfg[:10]: + name = cfg.get("name") or cfg.get("host", "?") + devices.append({ + "name": name, + "oids": cfg.get("oids", []), + "readings": latest.get(name, {}), + }) + + return await render_template("snmp/widget.html", + devices=devices, + total=len(devices_cfg)) + + +@snmp_bp.get("/device/") +@require_role(UserRole.viewer) +async def device_detail(device_name: str): + devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) + device_cfg = next( + (d for d in devices_cfg if (d.get("name") or d.get("host")) == device_name), + None, + ) + if device_cfg is None: + from quart import abort + abort(404) + + hours = int(request.args.get("hours", 24)) + async with current_app.db_sessionmaker() as db: + hist = await _history(db, device_name, hours=hours) + + import json + history_json = json.dumps(hist) + oid_labels = [o.get("label") or o.get("oid") for o in device_cfg.get("oids", [])] + + return await render_template( + "snmp/device.html", + device=device_cfg, + device_name=device_name, + oid_labels=oid_labels, + history_json=history_json, + hours=hours, + ) diff --git a/plugins/snmp/scheduler.py b/plugins/snmp/scheduler.py new file mode 100644 index 0000000..b8809c3 --- /dev/null +++ b/plugins/snmp/scheduler.py @@ -0,0 +1,78 @@ +# plugins/snmp/scheduler.py +from __future__ import annotations +import asyncio +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +from fabledscryer.core.scheduler import ScheduledTask + +logger = logging.getLogger(__name__) + + +def make_poll_task(app: "Quart") -> ScheduledTask: + interval = app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60) + + async def poll() -> None: + await _do_poll(app) + + return ScheduledTask( + name="snmp_poll", + coro_factory=poll, + interval_seconds=int(interval), + run_on_startup=True, + ) + + +async def _do_poll(app: "Quart") -> None: + from datetime import datetime, timezone + from .poller import poll_device_sync + from fabledscryer.core.alerts import record_metric + + devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", []) + if not devices: + return + + loop = asyncio.get_event_loop() + now = datetime.now(timezone.utc) + + async with app.db_sessionmaker() as session: + async with session.begin(): + for device in devices: + if not isinstance(device, dict): + continue + name = device.get("name") or device.get("host", "unknown") + host = device.get("host", "") + port = int(device.get("port", 161)) + community = device.get("community", "public") + version = str(device.get("version", "2c")) + oids = device.get("oids", []) + + if not host or not oids: + continue + + try: + readings = await loop.run_in_executor( + None, + poll_device_sync, + host, port, community, version, oids, + ) + except Exception: + logger.exception("SNMP poll failed for device %s (%s)", name, host) + continue + + for label, value in readings.items(): + await record_metric( + session=session, + source_module="snmp", + resource_name=name, + metric_name=label, + value=value, + ) + + if readings: + logger.debug("SNMP polled %s (%s): %d OID(s)", name, host, len(readings)) + else: + logger.debug("SNMP polled %s (%s): no readings", name, host) diff --git a/plugins/snmp/templates/snmp/device.html b/plugins/snmp/templates/snmp/device.html new file mode 100644 index 0000000..82022a9 --- /dev/null +++ b/plugins/snmp/templates/snmp/device.html @@ -0,0 +1,84 @@ +{# plugins/snmp/templates/snmp/device.html #} +{% extends "base.html" %} +{% block title %}SNMP — {{ device_name }} — Fabled Scryer{% endblock %} +{% block content %} +
+ ← SNMP +

{{ device_name }}

+ {{ device.host }} +
+ {% for h in [1, 6, 24, 168] %} + + {% if h < 24 %}{{ h }}h{% elif h == 24 %}24h{% else %}7d{% endif %} + + {% endfor %} +
+
+ +{% if oid_labels %} +
+ +
+ + +{% else %} +
+ No OIDs configured for this device. +
+{% endif %} +{% endblock %} diff --git a/plugins/snmp/templates/snmp/index.html b/plugins/snmp/templates/snmp/index.html new file mode 100644 index 0000000..86a1485 --- /dev/null +++ b/plugins/snmp/templates/snmp/index.html @@ -0,0 +1,57 @@ +{# plugins/snmp/templates/snmp/index.html #} +{% extends "base.html" %} +{% block title %}SNMP — Fabled Scryer{% endblock %} +{% block content %} +
+

SNMP Devices

+ {{ devices|length }} device(s) configured +
+ +{% if not devices %} +
+ No SNMP devices configured. Add devices to plugin.yaml. +
+{% else %} + +{% for device in devices %} +
+
+

{{ device.name }}

+ {{ device.host }} + {% if device.readings %} + ● reachable + {% else %} + ○ no data yet + {% endif %} + History +
+ + {% if device.readings %} +
+ {% for oid_cfg in device.oids %} + {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %} + {% set val = device.readings.get(label) %} +
+
{{ label }}
+ {% if val is not none %} +
+ {% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}M + {% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}K + {% else %}{{ "%.4g"|format(val) }}{% endif %} +
+ {% else %} +
—
+ {% endif %} +
+ {% endfor %} +
+ {% else %} +

+ No readings yet — waiting for next poll cycle. +

+ {% endif %} +
+{% endfor %} + +{% endif %} +{% endblock %} diff --git a/plugins/snmp/templates/snmp/widget.html b/plugins/snmp/templates/snmp/widget.html new file mode 100644 index 0000000..fc4d593 --- /dev/null +++ b/plugins/snmp/templates/snmp/widget.html @@ -0,0 +1,49 @@ +{# plugins/snmp/templates/snmp/widget.html #} +{% if devices %} +{% for device in devices %} +
+
+ + {{ device.name }} + + {% if device.readings %} + ● + {% else %} + ○ + {% endif %} + detail → +
+ {% if device.readings %} +
+ {% for oid_cfg in device.oids[:4] %} + {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %} + {% set val = device.readings.get(label) %} + {% if val is not none %} + + {{ label }} + + {% if val >= 1000000 %}{{ "%.1f"|format(val / 1000000) }}M + {% elif val >= 1000 %}{{ "%.0f"|format(val / 1000) }}K + {% else %}{{ "%.4g"|format(val) }}{% endif %} + + + {% endif %} + {% endfor %} + {% if device.oids|length > 4 %} + +{{ device.oids|length - 4 }} more + {% endif %} +
+ {% endif %} +
+{% if not loop.last %} +
+{% endif %} +{% endfor %} +{% if total > devices|length %} +
+ +{{ total - devices|length }} more — view all → +
+{% endif %} +{% else %} +

No SNMP devices configured.

+{% endif %} diff --git a/plugins/traefik/__init__.py b/plugins/traefik/__init__.py new file mode 100644 index 0000000..f2c7ac7 --- /dev/null +++ b/plugins/traefik/__init__.py @@ -0,0 +1,30 @@ +# plugins/traefik/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + from .models import TraefikMetric, TraefikCert, TraefikGlobalStat, TraefikAccessSummary # noqa: F401 + + +def get_scheduled_tasks() -> list: + from .scheduler import make_scrape_task, make_access_log_task + tasks = [make_scrape_task(_app)] + access_log_cfg = _app.config["PLUGINS"]["traefik"].get("access_log", {}) + if not isinstance(access_log_cfg, dict): + access_log_cfg = {} + if access_log_cfg.get("enabled", False): + tasks.append(make_access_log_task(_app)) + return tasks + + +def get_blueprint(): + from .routes import traefik_bp + return traefik_bp diff --git a/plugins/traefik/access_log.py b/plugins/traefik/access_log.py new file mode 100644 index 0000000..909b493 --- /dev/null +++ b/plugins/traefik/access_log.py @@ -0,0 +1,199 @@ +# plugins/traefik/access_log.py +"""Traefik JSON access log tailer and traffic-origin aggregator. + +Reads new lines from Traefik's JSON access log on each scheduler tick, +classifies source IPs as internal (RFC1918) or external, optionally +looks up country codes via MaxMind GeoLite2, and returns an aggregated +summary ready to persist in TraefikAccessSummary. +""" +from __future__ import annotations + +import ipaddress +import json +from collections import defaultdict +from pathlib import Path + +# ── Module-level file-position state ───────────────────────────────────────── +_log_position: int = 0 # byte offset of last read +_log_inode: int = 0 # detect rotation by inode change + +# ── Private IP ranges (RFC1918 + loopback + link-local) ────────────────────── +_PRIVATE_NETS = [ + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("127.0.0.0/8"), + ipaddress.ip_network("169.254.0.0/16"), + ipaddress.ip_network("::1/128"), + ipaddress.ip_network("fc00::/7"), + ipaddress.ip_network("fe80::/10"), +] + + +def _strip_port(raw: str) -> str: + """Strip trailing :port from an IPv4 address string. Leaves IPv6 untouched.""" + if not raw: + return raw + # Bracketed IPv6 with port: [::1]:12345 + if raw.startswith("["): + return raw[1:raw.index("]")] if "]" in raw else raw + # IPv4:port has exactly one colon + if raw.count(":") == 1: + return raw.rsplit(":", 1)[0] + return raw + + +def is_internal(raw_ip: str) -> bool: + """Return True if the address falls within a private/loopback range.""" + ip_str = _strip_port(raw_ip) + try: + addr = ipaddress.ip_address(ip_str) + return any(addr in net for net in _PRIVATE_NETS) + except ValueError: + return False + + +def _country(ip_str: str, geoip_db) -> str | None: + """Return ISO country code via an open maxminddb reader, or None.""" + if geoip_db is None: + return None + clean = _strip_port(ip_str) + try: + record = geoip_db.get(clean) + if record: + return record.get("country", {}).get("iso_code") + except Exception: + pass + return None + + +def read_new_lines(log_path: str) -> list[dict]: + """Return all new JSON-parsed access log entries since the last call. + + Handles log rotation: if the file shrinks or its inode changes, we + reset to the beginning of the new file. + """ + global _log_position, _log_inode + + path = Path(log_path) + if not path.exists(): + return [] + + try: + stat = path.stat() + except OSError: + return [] + + current_inode = stat.st_ino + current_size = stat.st_size + + # Rotation detected + if current_inode != _log_inode or current_size < _log_position: + _log_position = 0 + _log_inode = current_inode + + if current_size == _log_position: + return [] + + entries: list[dict] = [] + try: + with open(log_path, "rb") as f: + f.seek(_log_position) + raw = f.read(current_size - _log_position) + _log_position = current_size + except OSError: + return [] + + for raw_line in raw.split(b"\n"): + raw_line = raw_line.strip() + if not raw_line: + continue + try: + entries.append(json.loads(raw_line)) + except json.JSONDecodeError: + continue + + return entries + + +def aggregate(entries: list[dict], geoip_db=None, top_n: int = 10) -> dict | None: + """Aggregate a batch of access log entries into a summary dict. + + Returns None if there are no entries. + Result keys map directly to TraefikAccessSummary columns (excluding + id and period_start/period_end, which the caller provides). + """ + if not entries: + return None + + total = 0 + internal = 0 + external = 0 + total_bytes = 0.0 + + by_router: dict[str, int] = defaultdict(int) + ip_counts: dict[str, int] = defaultdict(int) + ip_internal: dict[str, bool] = {} + ip_country: dict[str, str | None] = {} + country_counts: dict[str, int] = defaultdict(int) + + for entry in entries: + total += 1 + raw_ip = entry.get("ClientHost", "") + router = entry.get("RouterName") or entry.get("ServiceName") or "unknown" + total_bytes += entry.get("DownstreamContentSize") or 0 + + by_router[router] += 1 + + internal_flag = is_internal(raw_ip) + if internal_flag: + internal += 1 + else: + external += 1 + if raw_ip not in ip_country: + ip_country[raw_ip] = _country(raw_ip, geoip_db) + c = ip_country[raw_ip] + if c: + country_counts[c] += 1 + + ip_counts[raw_ip] += 1 + ip_internal[raw_ip] = internal_flag + + top_ips = [ + { + "ip": ip, + "count": cnt, + "internal": ip_internal[ip], + "country": ip_country.get(ip), + } + for ip, cnt in sorted(ip_counts.items(), key=lambda x: x[1], reverse=True)[:top_n] + ] + top_countries = [ + {"country": c, "count": n} + for c, n in sorted(country_counts.items(), key=lambda x: x[1], reverse=True)[:top_n] + ] + top_routers = [ + {"router": r, "count": n} + for r, n in sorted(by_router.items(), key=lambda x: x[1], reverse=True)[:top_n] + ] + + return { + "total_requests": total, + "internal_requests": internal, + "external_requests": external, + "total_bytes": total_bytes, + "top_ips_json": json.dumps(top_ips), + "top_countries_json": json.dumps(top_countries), + "top_routers_json": json.dumps(top_routers), + } + + +def open_geoip(db_path: str | None): + """Open a MaxMind mmdb file if available. Returns None if not configured or missing.""" + if not db_path: + return None + try: + import maxminddb # type: ignore[import] + return maxminddb.open_database(db_path) + except Exception: + return None diff --git a/plugins/traefik/migrations/__init__.py b/plugins/traefik/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/traefik/migrations/env.py b/plugins/traefik/migrations/env.py new file mode 100644 index 0000000..1919fec --- /dev/null +++ b/plugins/traefik/migrations/env.py @@ -0,0 +1,75 @@ +# plugins/traefik/migrations/env.py +"""Alembic env.py for the Traefik plugin. + +For standalone use during plugin development. +At app startup, migration_runner.py uses the core env.py with version_locations. +""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +# Make fabledscryer importable +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from fabledscryer.models.base import Base +import fabledscryer.models # noqa: F401 — core models +from plugins.traefik.models import TraefikMetric # noqa: F401 — plugin model + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("FABLEDSCRYER_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/traefik/migrations/versions/__init__.py b/plugins/traefik/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/traefik/migrations/versions/traefik_001_initial.py b/plugins/traefik/migrations/versions/traefik_001_initial.py new file mode 100644 index 0000000..aa53f78 --- /dev/null +++ b/plugins/traefik/migrations/versions/traefik_001_initial.py @@ -0,0 +1,37 @@ +# plugins/traefik/migrations/versions/traefik_001_initial.py +"""Traefik metrics table + +Revision ID: traefik_001_initial +Revises: (none — this is a branch off the core head via depends_on) +Create Date: 2026-03-17 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "traefik_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "traefik" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "traefik_metrics", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("router_name", sa.String(255), nullable=False), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("request_rate", sa.Float, nullable=False), + sa.Column("error_rate_4xx_pct", sa.Float, nullable=False), + sa.Column("error_rate_5xx_pct", sa.Float, nullable=False), + sa.Column("latency_p50_ms", sa.Float, nullable=False), + sa.Column("latency_p95_ms", sa.Float, nullable=False), + sa.Column("latency_p99_ms", sa.Float, nullable=False), + ) + op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics", + ["router_name", "scraped_at"]) + + +def downgrade() -> None: + op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics") + op.drop_table("traefik_metrics") diff --git a/plugins/traefik/migrations/versions/traefik_002_stats.py b/plugins/traefik/migrations/versions/traefik_002_stats.py new file mode 100644 index 0000000..8320d53 --- /dev/null +++ b/plugins/traefik/migrations/versions/traefik_002_stats.py @@ -0,0 +1,48 @@ +"""Traefik extended stats: bandwidth, certs, global stats + +Revision ID: traefik_002_stats +Revises: traefik_001_initial +Create Date: 2026-03-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "traefik_002_stats" +down_revision: Union[str, None] = "traefik_001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "traefik_metrics", + sa.Column("response_bytes_rate", sa.Float, nullable=False, server_default="0"), + ) + + op.create_table( + "traefik_certs", + sa.Column("serial", sa.String(255), primary_key=True), + sa.Column("cn", sa.String(512), nullable=True), + sa.Column("sans", sa.Text, nullable=True), + sa.Column("not_after", sa.DateTime(timezone=True), nullable=False), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "traefik_global_stats", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("open_conns_total", sa.Float, nullable=True), + sa.Column("config_reloads_total", sa.Float, nullable=True), + sa.Column("config_last_reload_success", sa.Float, nullable=True), + sa.Column("process_memory_bytes", sa.Float, nullable=True), + ) + op.create_index("ix_traefik_global_stats_scraped", "traefik_global_stats", ["scraped_at"]) + + +def downgrade() -> None: + op.drop_index("ix_traefik_global_stats_scraped", "traefik_global_stats") + op.drop_table("traefik_global_stats") + op.drop_table("traefik_certs") + op.drop_column("traefik_metrics", "response_bytes_rate") diff --git a/plugins/traefik/migrations/versions/traefik_003_access_log.py b/plugins/traefik/migrations/versions/traefik_003_access_log.py new file mode 100644 index 0000000..fbc8009 --- /dev/null +++ b/plugins/traefik/migrations/versions/traefik_003_access_log.py @@ -0,0 +1,40 @@ +"""Traefik access log traffic summaries + +Revision ID: traefik_003_access_log +Revises: traefik_002_stats +Create Date: 2026-03-20 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "traefik_003_access_log" +down_revision: Union[str, None] = "traefik_002_stats" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "traefik_access_summaries", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("period_start", sa.DateTime(timezone=True), nullable=False), + sa.Column("period_end", sa.DateTime(timezone=True), nullable=False), + sa.Column("total_requests", sa.Integer, nullable=False, server_default="0"), + sa.Column("internal_requests", sa.Integer, nullable=False, server_default="0"), + sa.Column("external_requests", sa.Integer, nullable=False, server_default="0"), + sa.Column("total_bytes", sa.Float, nullable=False, server_default="0"), + sa.Column("top_ips_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("top_countries_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("top_routers_json", sa.Text, nullable=False, server_default="[]"), + ) + op.create_index( + "ix_traefik_access_summaries_period_end", + "traefik_access_summaries", + ["period_end"], + ) + + +def downgrade() -> None: + op.drop_index("ix_traefik_access_summaries_period_end", "traefik_access_summaries") + op.drop_table("traefik_access_summaries") diff --git a/plugins/traefik/models.py b/plugins/traefik/models.py new file mode 100644 index 0000000..1ba9e50 --- /dev/null +++ b/plugins/traefik/models.py @@ -0,0 +1,67 @@ +# plugins/traefik/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import DateTime, Float, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from fabledscryer.models.base import Base + + +class TraefikMetric(Base): + __tablename__ = "traefik_metrics" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + router_name: Mapped[str] = mapped_column(String(255), nullable=False) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) + ) + request_rate: Mapped[float] = mapped_column(Float, nullable=False) + error_rate_4xx_pct: Mapped[float] = mapped_column(Float, nullable=False) + error_rate_5xx_pct: Mapped[float] = mapped_column(Float, nullable=False) + latency_p50_ms: Mapped[float] = mapped_column(Float, nullable=False) + latency_p95_ms: Mapped[float] = mapped_column(Float, nullable=False) + latency_p99_ms: Mapped[float] = mapped_column(Float, nullable=False) + response_bytes_rate: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + + +class TraefikCert(Base): + """TLS certificate expiry tracking — upserted by serial each scrape.""" + __tablename__ = "traefik_certs" + + serial: Mapped[str] = mapped_column(String(255), primary_key=True) + cn: Mapped[str | None] = mapped_column(String(512), nullable=True) + sans: Mapped[str | None] = mapped_column(Text, nullable=True) + not_after: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) + ) + + +class TraefikAccessSummary(Base): + """Aggregated traffic-origin summary — one row per scheduler tick.""" + __tablename__ = "traefik_access_summaries" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + period_start: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + period_end: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + total_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + internal_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + external_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + total_bytes: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + top_ips_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + top_countries_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + top_routers_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + + +class TraefikGlobalStat(Base): + """One row per scrape — process-level and config stats.""" + __tablename__ = "traefik_global_stats" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) + ) + open_conns_total: Mapped[float | None] = mapped_column(Float, nullable=True) + config_reloads_total: Mapped[float | None] = mapped_column(Float, nullable=True) + config_last_reload_success: Mapped[float | None] = mapped_column(Float, nullable=True) + process_memory_bytes: Mapped[float | None] = mapped_column(Float, nullable=True) diff --git a/plugins/traefik/plugin.yaml b/plugins/traefik/plugin.yaml new file mode 100644 index 0000000..392713b --- /dev/null +++ b/plugins/traefik/plugin.yaml @@ -0,0 +1,21 @@ +# plugins/traefik/plugin.yaml +name: traefik +version: "1.0.0" +description: "Traefik reverse proxy metrics and access log integration" +author: "FabledScryer" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/traefik" +tags: + - proxy + - metrics + - access-log + +config: + metrics_url: "http://localhost:8080/metrics" + scrape_interval_seconds: 60 + access_log: + enabled: false + path: "/var/log/traefik/access.log" + geoip_db: "" # optional: path to MaxMind GeoLite2-Country.mmdb diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py new file mode 100644 index 0000000..f7fe3ce --- /dev/null +++ b/plugins/traefik/routes.py @@ -0,0 +1,373 @@ +# plugins/traefik/routes.py +from __future__ import annotations +from collections import defaultdict +from datetime import datetime, timezone +from quart import Blueprint, current_app, render_template, request +from sqlalchemy import Integer, cast, func, select +import json + +from fabledscryer.auth.middleware import require_role +from fabledscryer.models.users import UserRole +from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample +from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric + +traefik_bp = Blueprint("traefik", __name__, template_folder="templates") + + +def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: + """Generate an inline SVG sparkline from a list of float values.""" + if len(values) < 2: + return f'' + mn, mx = min(values), max(values) + if mx == mn: + mx = mn + 1.0 + step = width / (len(values) - 1) + pts = [] + for i, v in enumerate(values): + x = i * step + y = height - (v - mn) / (mx - mn) * (height - 2) - 1 + pts.append(f"{x:.1f},{y:.1f}") + poly = " ".join(pts) + return ( + f'' + f'' + f'' + ) + + +@traefik_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + _al_cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {}) + if not isinstance(_al_cfg, dict): + _al_cfg = {} + access_log_enabled = _al_cfg.get("enabled", False) + current_range = request.args.get("range", DEFAULT_RANGE) + return await render_template( + "traefik/index.html", + poll_interval=poll_interval, + access_log_enabled=access_log_enabled, + current_range=current_range, + ) + + +@traefik_bp.get("/rows") +@require_role(UserRole.viewer) +async def rows(): + """HTMX fragment: service cards with sparklines scoped to selected time range.""" + since, range_key = parse_range(request.args.get("range")) + now = datetime.now(timezone.utc) + + async with current_app.db_sessionmaker() as db: + # All known routers + result = await db.execute( + select(TraefikMetric.router_name) + .distinct() + .order_by(TraefikMetric.router_name) + ) + routers = [row[0] for row in result.all()] + + b_secs = bucket_seconds(since) + bucket_col = ( + cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs + ).label("bucket") + + router_data = [] + for router in routers: + # Bucketed history for sparklines — always ~80 points regardless of range + result = await db.execute( + select( + func.avg(TraefikMetric.request_rate).label("request_rate"), + func.avg(TraefikMetric.latency_p95_ms).label("latency_p95_ms"), + func.avg(TraefikMetric.error_rate_5xx_pct).label("error_rate_5xx_pct"), + func.min(TraefikMetric.scraped_at).label("scraped_at"), + bucket_col, + ) + .where(TraefikMetric.router_name == router) + .where(TraefikMetric.scraped_at >= since) + .group_by(bucket_col) + .order_by(bucket_col) + ) + history = result.all() + + # Latest value — always the most recent raw row + result = await db.execute( + select(TraefikMetric) + .where(TraefikMetric.router_name == router) + .order_by(TraefikMetric.scraped_at.desc()) + .limit(1) + ) + latest = result.scalar_one_or_none() + if not latest: + continue + + router_data.append({ + "name": router, + "latest": latest, + "sparkline_req": _sparkline([r.request_rate for r in history]), + "sparkline_p95": _sparkline([r.latency_p95_ms for r in history]), + "sparkline_5xx": _sparkline([r.error_rate_5xx_pct for r in history]), + }) + + # Latest global stat (always most recent, not range-filtered) + result = await db.execute( + select(TraefikGlobalStat) + .order_by(TraefikGlobalStat.scraped_at.desc()) + .limit(1) + ) + global_stat = result.scalar_one_or_none() + + # All certs, sorted by soonest expiry + result = await db.execute( + select(TraefikCert).order_by(TraefikCert.not_after) + ) + raw_certs = result.scalars().all() + + certs = [] + for c in raw_certs: + days = (c.not_after - now).total_seconds() / 86400.0 + certs.append({ + "serial": c.serial, + "cn": c.cn or c.serial, + "sans": c.sans, + "not_after": c.not_after, + "days_remaining": days, + }) + + return await render_template( + "traefik/rows.html", + router_data=router_data, + global_stat=global_stat, + certs=certs, + range_key=range_key, + ) + + +@traefik_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """HTMX dashboard widget fragment.""" + now = datetime.now(timezone.utc) + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(TraefikMetric.router_name).distinct().order_by(TraefikMetric.router_name) + ) + routers = [row[0] for row in result.all()] + router_data = [] + for router in routers: + result = await db.execute( + select(TraefikMetric) + .where(TraefikMetric.router_name == router) + .order_by(TraefikMetric.scraped_at.desc()) + .limit(1) + ) + latest = result.scalar_one_or_none() + if latest: + router_data.append({"name": router, "latest": latest}) + + router_data.sort(key=lambda r: ( + -(1 if r["latest"].error_rate_5xx_pct > 0.1 else 0), + -(1 if r["latest"].error_rate_4xx_pct > 5.0 else 0), + -r["latest"].request_rate, + )) + total_routers = len(router_data) + router_data = router_data[:5] + + result = await db.execute( + select(TraefikCert).order_by(TraefikCert.not_after) + ) + raw_certs = result.scalars().all() + + expiring_certs = [] + for c in raw_certs: + days = (c.not_after - now).total_seconds() / 86400.0 + if days < 30: + expiring_certs.append({"cn": c.cn or c.serial, "days_remaining": days}) + + return await render_template( + "traefik/widget.html", + router_data=router_data, + expiring_certs=expiring_certs, + total_routers=total_routers, + ) + + +@traefik_bp.get("/widget/access_log") +@require_role(UserRole.viewer) +async def widget_access_log(): + """HTMX dashboard widget: traffic origin summary with IP filter.""" + from collections import defaultdict + ip_filter = request.args.get("ip_filter", "all") + limit = max(1, min(50, int(request.args.get("limit", 10) or 10))) + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(TraefikAccessSummary) + .order_by(TraefikAccessSummary.period_end.desc()) + .limit(12) # last ~1 hour of 5-min windows + ) + rows = result.scalars().all() + + if not rows: + return await render_template("traefik/widget_access_log.html", + summary=None, widget_id=widget_id) + + total = sum(r.total_requests for r in rows) + internal = sum(r.internal_requests for r in rows) + external = sum(r.external_requests for r in rows) + + ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True}) + for row in rows: + for entry in json.loads(row.top_ips_json): + k = entry["ip"] + ip_counts[k]["count"] += entry["count"] + ip_counts[k]["internal"] = entry.get("internal", True) + + all_ips = sorted( + [{"ip": k, **v} for k, v in ip_counts.items()], + key=lambda x: x["count"], reverse=True, + ) + if ip_filter == "internal": + all_ips = [e for e in all_ips if e["internal"]] + elif ip_filter == "external": + all_ips = [e for e in all_ips if not e["internal"]] + + summary = { + "total": total, + "internal": internal, + "external": external, + "external_pct": (external / total * 100.0) if total else 0.0, + "top_ips": all_ips[:limit], + "ip_filter": ip_filter, + } + return await render_template("traefik/widget_access_log.html", + summary=summary, widget_id=widget_id) + + +@traefik_bp.get("/widget/request_chart") +@require_role(UserRole.viewer) +async def widget_request_chart(): + """HTMX dashboard widget: Chart.js request rate + error % over time.""" + from datetime import timedelta + hours = max(1, min(24, int(request.args.get("hours", 6) or 6))) + widget_id = request.args.get("wid", "0") + + since = datetime.now(timezone.utc) - timedelta(hours=hours) + b_secs = max(60, (hours * 3600) // 80) # ~80 buckets + + bucket_col = ( + cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs + ).label("bucket") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select( + func.avg(TraefikMetric.request_rate).label("req_rate"), + func.avg(TraefikMetric.error_rate_5xx_pct).label("err_pct"), + func.min(TraefikMetric.scraped_at).label("scraped_at"), + bucket_col, + ) + .where(TraefikMetric.scraped_at >= since) + .group_by(bucket_col) + .order_by(bucket_col) + ) + history = result.all() + + labels = list(range(len(history))) + req_rates = [round(r.req_rate or 0, 3) for r in history] + err_pcts = [round(r.err_pct or 0, 2) for r in history] + + return await render_template( + "traefik/widget_request_chart.html", + labels_json=json.dumps(labels), + req_rate_json=json.dumps(req_rates), + error_rate_json=json.dumps(err_pcts), + widget_id=widget_id, + hours=hours, + ) + + +@traefik_bp.get("/traffic") +@require_role(UserRole.viewer) +async def traffic(): + """HTMX fragment: traffic origin summary scoped to selected time range.""" + since, range_key = parse_range(request.args.get("range")) + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(TraefikAccessSummary) + .where(TraefikAccessSummary.period_end >= since) + .order_by(TraefikAccessSummary.period_end.asc()) + # No LIMIT — all summaries in range needed for accurate totals; + # sparkline subsamples to 80 points afterwards + ) + rows = result.scalars().all() + + if not rows: + return await render_template( + "traefik/traffic.html", summary=None, history=[], range_key=range_key + ) + + total = sum(r.total_requests for r in rows) + internal = sum(r.internal_requests for r in rows) + external = sum(r.external_requests for r in rows) + total_bytes = sum(r.total_bytes for r in rows) + + ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True, "country": None}) + country_counts: dict[str, int] = defaultdict(int) + router_counts: dict[str, int] = defaultdict(int) + + for row in rows: + for entry in json.loads(row.top_ips_json): + k = entry["ip"] + ip_counts[k]["count"] += entry["count"] + ip_counts[k]["internal"] = entry["internal"] + ip_counts[k]["country"] = entry.get("country") + for entry in json.loads(row.top_countries_json): + country_counts[entry["country"]] += entry["count"] + for entry in json.loads(row.top_routers_json): + router_counts[entry["router"]] += entry["count"] + + top_ips = sorted( + [{"ip": k, **v} for k, v in ip_counts.items()], + key=lambda x: x["count"], reverse=True + )[:10] + top_countries = sorted( + [{"country": k, "count": v} for k, v in country_counts.items()], + key=lambda x: x["count"], reverse=True + )[:10] + top_routers = sorted( + [{"router": k, "count": v} for k, v in router_counts.items()], + key=lambda x: x["count"], reverse=True + )[:10] + + history = [ + { + "period_end": r.period_end, + "external_pct": (r.external_requests / r.total_requests * 100.0) + if r.total_requests else 0.0, + } + for r in rows + ] + + summary = { + "total": total, + "internal": internal, + "external": external, + "total_bytes": total_bytes, + "external_pct": (external / total * 100.0) if total else 0.0, + "top_ips": top_ips, + "top_countries": top_countries, + "top_routers": top_routers, + "sparkline_external": _sparkline( + [h["external_pct"] for h in subsample(history, 80)] + ), + } + + return await render_template( + "traefik/traffic.html", summary=summary, history=history, range_key=range_key + ) diff --git a/plugins/traefik/scheduler.py b/plugins/traefik/scheduler.py new file mode 100644 index 0000000..b7eb050 --- /dev/null +++ b/plugins/traefik/scheduler.py @@ -0,0 +1,204 @@ +# plugins/traefik/scheduler.py +from __future__ import annotations +import logging +import time +from typing import TYPE_CHECKING + +from datetime import datetime +from fabledscryer.core.scheduler import ScheduledTask + +if TYPE_CHECKING: + from quart import Quart + +logger = logging.getLogger(__name__) + +# Module-level state: previous scrape data for delta computation +_prev_metrics: dict = {} +_prev_time: float = 0.0 + +# Access log state +_geoip_db = None +_access_log_last_tick: "datetime | None" = None + + +def make_scrape_task(app: "Quart") -> ScheduledTask: + """Return a ScheduledTask that scrapes Traefik metrics on the configured interval.""" + interval = app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"] + + async def scrape() -> None: + await _do_scrape(app) + + return ScheduledTask( + name="traefik_scrape", + coro_factory=scrape, + interval_seconds=int(interval), + run_on_startup=True, + ) + + +async def _do_scrape(app: "Quart") -> None: + """Fetch Traefik metrics, write to DB, and emit plugin_metrics.""" + global _prev_metrics, _prev_time + + from datetime import datetime, timezone + from sqlalchemy import select + from .models import TraefikCert, TraefikGlobalStat, TraefikMetric + from .scraper import ( + compute_global_stats, + compute_router_metrics, + extract_certs, + fetch_metrics, + ) + from fabledscryer.core.alerts import record_metric + + metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"] + + try: + current = await fetch_metrics(metrics_url) + except Exception: + logger.exception("Traefik scrape failed (url=%s)", metrics_url) + return + + now = time.monotonic() + elapsed = now - _prev_time if _prev_time else 60.0 + router_metrics = compute_router_metrics(current, _prev_metrics or None, elapsed) + global_stats = compute_global_stats(current, _prev_metrics or None) + cert_list = extract_certs(current) + + _prev_metrics = current + _prev_time = now + + scraped_at = datetime.now(timezone.utc) + + async with app.db_sessionmaker() as session: + async with session.begin(): + # ── Per-service metrics ─────────────────────────────────────────── + for router_name, metrics in router_metrics.items(): + row = TraefikMetric( + router_name=router_name, + scraped_at=scraped_at, + request_rate=metrics["request_rate"], + error_rate_4xx_pct=metrics["error_rate_4xx_pct"], + error_rate_5xx_pct=metrics["error_rate_5xx_pct"], + latency_p50_ms=metrics["latency_p50_ms"], + latency_p95_ms=metrics["latency_p95_ms"], + latency_p99_ms=metrics["latency_p99_ms"], + response_bytes_rate=metrics["response_bytes_rate"], + ) + session.add(row) + + for metric_name, value in [ + ("request_rate", metrics["request_rate"]), + ("error_rate_4xx_pct", metrics["error_rate_4xx_pct"]), + ("error_rate_5xx_pct", metrics["error_rate_5xx_pct"]), + ("latency_p50_ms", metrics["latency_p50_ms"]), + ("latency_p95_ms", metrics["latency_p95_ms"]), + ("latency_p99_ms", metrics["latency_p99_ms"]), + ("response_bytes_rate", metrics["response_bytes_rate"]), + ]: + await record_metric( + session=session, + source_module="traefik", + resource_name=router_name, + metric_name=metric_name, + value=value, + ) + + # ── Global stats ────────────────────────────────────────────────── + session.add(TraefikGlobalStat(scraped_at=scraped_at, **global_stats)) + + # ── TLS certs (upsert by serial) ────────────────────────────────── + for cert in cert_list: + existing = await session.get(TraefikCert, cert["serial"]) + if existing: + existing.cn = cert["cn"] + existing.sans = cert["sans"] + existing.not_after = cert["not_after"] + existing.scraped_at = scraped_at + else: + session.add(TraefikCert( + serial=cert["serial"], + cn=cert["cn"], + sans=cert["sans"], + not_after=cert["not_after"], + scraped_at=scraped_at, + )) + + # Emit cert_expiry_days for alert rules + days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0 + await record_metric( + session=session, + source_module="traefik", + resource_name=cert["cn"] or cert["serial"], + metric_name="cert_expiry_days", + value=max(0.0, days_remaining), + ) + + logger.debug( + "Traefik scrape: %d router(s), %d cert(s)", + len(router_metrics), + len(cert_list), + ) + + +def make_access_log_task(app: "Quart") -> ScheduledTask: + """Return a ScheduledTask that processes new Traefik access log lines.""" + global _geoip_db + cfg = app.config["PLUGINS"]["traefik"].get("access_log", {}) + if not isinstance(cfg, dict): + cfg = {} + log_path: str = cfg.get("path", "/var/log/traefik/access.log") + geoip_path: str = cfg.get("geoip_db", "") + + from .access_log import open_geoip + _geoip_db = open_geoip(geoip_path or None) + if _geoip_db: + logger.info("Traefik access log: GeoIP database loaded from %s", geoip_path) + else: + logger.info("Traefik access log: GeoIP not configured, country lookup disabled") + + interval = app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"] + + async def process() -> None: + await _do_access_log(app, log_path) + + return ScheduledTask( + name="traefik_access_log", + coro_factory=process, + interval_seconds=int(interval), + run_on_startup=True, + ) + + +async def _do_access_log(app: "Quart", log_path: str) -> None: + """Read new access log lines, aggregate, and persist a summary.""" + global _access_log_last_tick + + from datetime import timezone + from .access_log import read_new_lines, aggregate + from .models import TraefikAccessSummary + + now = datetime.now(timezone.utc) + period_start = _access_log_last_tick or now + _access_log_last_tick = now + + entries = read_new_lines(log_path) + summary = aggregate(entries, geoip_db=_geoip_db) + + if summary is None: + return + + async with app.db_sessionmaker() as session: + async with session.begin(): + session.add(TraefikAccessSummary( + period_start=period_start, + period_end=now, + **summary, + )) + + logger.debug( + "Traefik access log: %d requests (%d internal, %d external)", + summary["total_requests"], + summary["internal_requests"], + summary["external_requests"], + ) diff --git a/plugins/traefik/scraper.py b/plugins/traefik/scraper.py new file mode 100644 index 0000000..6e06dd2 --- /dev/null +++ b/plugins/traefik/scraper.py @@ -0,0 +1,276 @@ +# plugins/traefik/scraper.py +"""Prometheus text-format scraper and per-service metrics calculator for Traefik. + +Traefik exposes Prometheus metrics at /metrics. This module: +1. Fetches the endpoint via httpx +2. Parses the Prometheus text format (counters + histograms) +3. Computes per-service request rate (delta/elapsed), error rates (% 4xx, % 5xx), + latency percentiles (p50, p95, p99) via linear histogram interpolation, + and bandwidth (bytes/sec from response bytes counter delta) +4. Extracts TLS certificate expiry info +5. Computes process-level and config health stats +""" +from __future__ import annotations +import math +import re +from collections import defaultdict +from datetime import datetime, timezone + +import httpx + + +# ────────────────────────────────────────────────────────────────────────────── +# Prometheus text format parser +# ────────────────────────────────────────────────────────────────────────────── + +# Parsed sample: {metric_name: {frozenset(label_items): float}} +ParsedMetrics = dict[str, dict[frozenset, float]] + +_SAMPLE_RE = re.compile( + r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+' + r'([+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?|[+-]?Inf|NaN)' +) +_LABEL_RE = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"\\]*(?:\\.[^"\\]*)*)"') + + +def parse_prometheus(text: str) -> ParsedMetrics: + """Parse Prometheus text exposition format into nested dicts. + + Returns: {metric_name: {frozenset({(label, value), ...}): float}} + """ + metrics: ParsedMetrics = {} + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + m = _SAMPLE_RE.match(line) + if not m: + continue + name, labels_str, value_str = m.groups() + try: + value = float(value_str) + except ValueError: + continue + labels = frozenset( + (lm.group(1), lm.group(2)) + for lm in _LABEL_RE.finditer(labels_str or "") + ) + metrics.setdefault(name, {})[labels] = value + return metrics + + +# ────────────────────────────────────────────────────────────────────────────── +# Per-router metric computation +# ────────────────────────────────────────────────────────────────────────────── + +def compute_router_metrics( + current: ParsedMetrics, + previous: ParsedMetrics | None, + elapsed_seconds: float, +) -> dict[str, dict[str, float]]: + """Compute per-service derived metrics from two consecutive scrapes. + + Uses traefik_service_* metrics (available by default). + + Returns: + {service_name: { + "request_rate": float, # requests/sec + "error_rate_4xx_pct": float, # % 4xx of total + "error_rate_5xx_pct": float, # % 5xx of total + "latency_p50_ms": float, + "latency_p95_ms": float, + "latency_p99_ms": float, + "response_bytes_rate": float, # bytes/sec + }} + """ + prev = previous or {} + elapsed = max(elapsed_seconds, 1.0) + + # ── Request counters ────────────────────────────────────────────────────── + req_samples = current.get("traefik_service_requests_total", {}) + prev_req = prev.get("traefik_service_requests_total", {}) + + service_total: dict[str, float] = defaultdict(float) + service_4xx: dict[str, float] = defaultdict(float) + service_5xx: dict[str, float] = defaultdict(float) + + for labels, value in req_samples.items(): + label_dict = dict(labels) + service = label_dict.get("service", "") + if not service: + continue + code = label_dict.get("code", "") + prev_value = prev_req.get(labels, value) # no delta on first scrape + delta = max(0.0, value - prev_value) + service_total[service] += delta + if code.startswith("4"): + service_4xx[service] += delta + elif code.startswith("5"): + service_5xx[service] += delta + + # ── Response bytes counters ─────────────────────────────────────────────── + bytes_samples = current.get("traefik_service_responses_bytes_total", {}) + prev_bytes = prev.get("traefik_service_responses_bytes_total", {}) + + service_bytes: dict[str, float] = defaultdict(float) + for labels, value in bytes_samples.items(): + service = dict(labels).get("service", "") + if not service: + continue + prev_value = prev_bytes.get(labels, value) + service_bytes[service] += max(0.0, value - prev_value) + + # ── Histogram buckets ───────────────────────────────────────────────────── + hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {}) + + # Collect all known services + all_services: set[str] = set(service_total.keys()) | set(service_bytes.keys()) + for labels in hist_buckets: + service = dict(labels).get("service", "") + if service: + all_services.add(service) + + result: dict[str, dict[str, float]] = {} + + for service in all_services: + total = service_total.get(service, 0.0) + result[service] = { + "request_rate": total / elapsed, + "error_rate_4xx_pct": (service_4xx.get(service, 0.0) / total * 100.0) + if total > 0 else 0.0, + "error_rate_5xx_pct": (service_5xx.get(service, 0.0) / total * 100.0) + if total > 0 else 0.0, + "latency_p50_ms": _percentile_ms(hist_buckets, service, 0.50), + "latency_p95_ms": _percentile_ms(hist_buckets, service, 0.95), + "latency_p99_ms": _percentile_ms(hist_buckets, service, 0.99), + "response_bytes_rate": service_bytes.get(service, 0.0) / elapsed, + } + + return result + + +def compute_global_stats( + current: ParsedMetrics, + previous: ParsedMetrics | None, +) -> dict[str, float | None]: + """Extract process-level and config health metrics from a scrape. + + Returns: + { + "open_conns_total": float | None, + "config_reloads_total": float | None, + "config_last_reload_success": float | None, # 1.0 = success, 0.0 = failure + "process_memory_bytes": float | None, + } + """ + stats: dict[str, float | None] = { + "open_conns_total": None, + "config_reloads_total": None, + "config_last_reload_success": None, + "process_memory_bytes": None, + } + + # Open connections — sum across all entrypoints/protocols + open_conns = current.get("traefik_open_connections", {}) + if open_conns: + stats["open_conns_total"] = sum(open_conns.values()) + + # Config reloads — sum across result labels (success + failure) + reloads = current.get("traefik_config_reloads_total", {}) + if reloads: + stats["config_reloads_total"] = sum(reloads.values()) + + # Last reload success gauge + last_success = current.get("traefik_config_last_reload_success", {}) + if last_success: + # Usually a single sample with no labels + stats["config_last_reload_success"] = next(iter(last_success.values())) + + # Process RSS memory + mem = current.get("process_resident_memory_bytes", {}) + if mem: + stats["process_memory_bytes"] = next(iter(mem.values())) + + return stats + + +def extract_certs(current: ParsedMetrics) -> list[dict]: + """Extract TLS certificate expiry info from traefik_tls_certs_not_after. + + Returns a list of dicts: + [{"serial": str, "cn": str, "sans": str, "not_after": datetime}, ...] + """ + certs = [] + samples = current.get("traefik_tls_certs_not_after", {}) + for labels, ts_value in samples.items(): + label_dict = dict(labels) + serial = label_dict.get("serial", "") + if not serial: + continue + try: + not_after = datetime.fromtimestamp(ts_value, tz=timezone.utc) + except (ValueError, OSError, OverflowError): + continue + certs.append({ + "serial": serial, + "cn": label_dict.get("cn") or None, + "sans": label_dict.get("sans") or None, + "not_after": not_after, + }) + return certs + + +def _percentile_ms( + buckets: dict[frozenset, float], + service: str, + pct: float, +) -> float: + """Linear interpolation of a Prometheus histogram percentile, returned in ms.""" + router_buckets: list[tuple[float, float]] = [] + for labels, count in buckets.items(): + label_dict = dict(labels) + if label_dict.get("service") != service: + continue + le_str = label_dict.get("le", "") + try: + le = float("inf") if le_str == "+Inf" else float(le_str) + router_buckets.append((le, count)) + except ValueError: + continue + + if not router_buckets: + return 0.0 + + router_buckets.sort(key=lambda t: t[0]) + total = router_buckets[-1][1] # +Inf bucket count == total requests + if total == 0.0: + return 0.0 + + target = total * pct + for i, (le, count) in enumerate(router_buckets): + if count < target: + continue + if i == 0: + ratio = target / count if count > 0 else 0.0 + return le * ratio * 1000.0 + prev_le, prev_count = router_buckets[i - 1] + if count == prev_count: + return prev_le * 1000.0 + if math.isinf(le): + return prev_le * 1000.0 + fraction = (target - prev_count) / (count - prev_count) + return (prev_le + fraction * (le - prev_le)) * 1000.0 + + return 0.0 + + +# ────────────────────────────────────────────────────────────────────────────── +# HTTP fetch +# ────────────────────────────────────────────────────────────────────────────── + +async def fetch_metrics(metrics_url: str) -> ParsedMetrics: + """Fetch and parse Prometheus metrics from the given URL.""" + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.get(metrics_url) + response.raise_for_status() + return parse_prometheus(response.text) diff --git a/plugins/traefik/templates/traefik/index.html b/plugins/traefik/templates/traefik/index.html new file mode 100644 index 0000000..da66fc7 --- /dev/null +++ b/plugins/traefik/templates/traefik/index.html @@ -0,0 +1,31 @@ +{# plugins/traefik/templates/traefik/index.html #} +{% extends "base.html" %} +{% block title %}Traefik — Fabled Scryer{% endblock %} +{% block content %} +
+
+

Traefik Services

+ polling every {{ poll_interval }}s +
+ {% include "_time_range.html" %} +
+ +
+
+ +{% if access_log_enabled %} +
+

Traffic Origin

+
+
+
+{% endif %} +{% endblock %} diff --git a/plugins/traefik/templates/traefik/rows.html b/plugins/traefik/templates/traefik/rows.html new file mode 100644 index 0000000..c02b962 --- /dev/null +++ b/plugins/traefik/templates/traefik/rows.html @@ -0,0 +1,127 @@ +{# plugins/traefik/templates/traefik/rows.html #} + +{# ── Global stats bar ──────────────────────────────────────────────────────── #} +{% if global_stat %} +
+ + {% if global_stat.open_conns_total is not none %} + + Open conns + {{ global_stat.open_conns_total | int }} + + {% endif %} + + {% if global_stat.config_last_reload_success is not none %} + + Config + {% if global_stat.config_last_reload_success == 1.0 %} + ✓ OK + {% else %} + ✗ failed + {% endif %} + {% if global_stat.config_reloads_total is not none %} + ({{ global_stat.config_reloads_total | int }} reloads) + {% endif %} + + {% endif %} + + {% if global_stat.process_memory_bytes is not none %} + + Memory + {% set mb = (global_stat.process_memory_bytes / 1048576) %} + + {% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.0f"|format(mb) }} MB{% endif %} + + + {% endif %} + + + {{ global_stat.scraped_at.strftime("%H:%M:%S") }} UTC + +
+{% endif %} + +{# ── Per-service cards + cert card in shared columns layout ───────────────── #} +{% if router_data or certs %} +
+ +{% for r in router_data %} +
+

{{ r.name }}

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Req/s{{ "%.2f"|format(r.latest.request_rate) }}{{ r.sparkline_req | safe }}
Bandwidth + {% set bps = r.latest.response_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s + {% elif bps >= 1024 %}{{ "%.1f"|format(bps / 1024) }} KB/s + {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %} +
p95 ms{{ "%.1f"|format(r.latest.latency_p95_ms) }}{{ r.sparkline_p95 | safe }}
p99 ms{{ "%.1f"|format(r.latest.latency_p99_ms) }}
4xx % + {{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}% +
5xx % + {{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% + {{ r.sparkline_5xx | safe }}
+
+ Last scraped {{ r.latest.scraped_at.strftime("%H:%M:%S") }} UTC +
+
+{% endfor %} + +{% if certs %} +
+
TLS Certificates
+ + {% for cert in certs %} + + + + + + + + {% endfor %} +
{{ cert.cn }} + {{ cert.days_remaining | int }}d +
+ {{ cert.not_after.strftime("%Y-%m-%d") }} +
+
+{% endif %} + +
+{% else %} +
+

No Traefik metrics yet. Configure metrics_url in config.yaml under plugins.traefik.

+
+{% endif %} diff --git a/plugins/traefik/templates/traefik/traffic.html b/plugins/traefik/templates/traefik/traffic.html new file mode 100644 index 0000000..dcdcb96 --- /dev/null +++ b/plugins/traefik/templates/traefik/traffic.html @@ -0,0 +1,101 @@ +{# plugins/traefik/templates/traefik/traffic.html #} +{% if summary %} + +{# ── Overview bar ─────────────────────────────────────────────────────────── #} +
+ + Total + {{ summary.total }} + + + Internal + {{ summary.internal }} + + + External + + {{ summary.external }} ({{ "%.1f"|format(summary.external_pct) }}%) + + + + Bytes + + {% set mb = summary.total_bytes / 1048576 %} + {% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.1f"|format(mb) }} MB{% endif %} + + + + {# External % sparkline #} + + ext % over time + {{ summary.sparkline_external | safe }} + +
+ +{# ── Internal/external fill bar ───────────────────────────────────────────── #} +{% set ext_pct = summary.external_pct %} +
+
+
+ +{# ── Detail cards in columns ──────────────────────────────────────────────── #} +
+ + {# Top source IPs #} +
+
Top Source IPs
+ + {% for entry in summary.top_ips %} + + + {% if entry.country %} + + {% endif %} + + + {% endfor %} +
{{ entry.ip }}{{ entry.country }} + + {{ entry.count }} + +
+
+ + {# Top routers #} +
+
Top Services Hit
+ + {% for entry in summary.top_routers %} + + + + + {% endfor %} +
{{ entry.router }}{{ entry.count }}
+
+ + {# Top countries (only shown if GeoIP is active) #} + {% if summary.top_countries %} +
+
External Traffic by Country
+ + {% for entry in summary.top_countries %} + + + + + {% endfor %} +
{{ entry.country }}{{ entry.count }}
+
+ {% endif %} + +
+ +{% else %} +
+ No access log data yet. + {% if not access_log_enabled %} + Enable access_log.enabled: true in your plugin config and bind-mount the Traefik log directory. + {% endif %} +
+{% endif %} diff --git a/plugins/traefik/templates/traefik/widget.html b/plugins/traefik/templates/traefik/widget.html new file mode 100644 index 0000000..aa43085 --- /dev/null +++ b/plugins/traefik/templates/traefik/widget.html @@ -0,0 +1,56 @@ +{# plugins/traefik/templates/traefik/widget.html #} +{% if expiring_certs %} +
+{% for cert in expiring_certs %} +
+ ⚠ {{ cert.cn }} + {{ cert.days_remaining | int }}d +
+{% endfor %} +
+{% endif %} + +{% if router_data %} +{% for r in router_data %} +
+
+ {{ r.name }} +
+
+ + req/s + {{ "%.2f"|format(r.latest.request_rate) }} + + + bw + + {% set bps = r.latest.response_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s + {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %} + + + + p95 + + {{ "%.0f"|format(r.latest.latency_p95_ms) }}ms + + + {% if r.latest.error_rate_5xx_pct > 0 %} + + {{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% 5xx + + {% endif %} +
+
+{% endfor %} +{% if total_routers > 5 %} +
+ +{{ total_routers - 5 }} more — view all → +
+{% endif %} +{% else %} +

No Traefik data yet.

+{% endif %} diff --git a/plugins/traefik/templates/traefik/widget_access_log.html b/plugins/traefik/templates/traefik/widget_access_log.html new file mode 100644 index 0000000..899e09c --- /dev/null +++ b/plugins/traefik/templates/traefik/widget_access_log.html @@ -0,0 +1,40 @@ +{# traefik/widget_access_log.html — dashboard widget: traffic origin summary #} +{% if not summary %} +
No access log data yet.
+{% else %} + +
+
+ {{ summary.total | int }} + total req +
+
+ + {{ "%.1f" | format(summary.external_pct) }}% + + external +
+
+ +{% if summary.top_ips %} +
+ Top IPs + {% if summary.ip_filter != "all" %} + {{ summary.ip_filter }} + {% endif %} +
+
+ {% for ip in summary.top_ips %} +
+ + + {{ ip.ip }} + + {{ ip.count }} +
+ {% endfor %} +
+{% endif %} + +{% endif %} diff --git a/plugins/traefik/templates/traefik/widget_request_chart.html b/plugins/traefik/templates/traefik/widget_request_chart.html new file mode 100644 index 0000000..272ff39 --- /dev/null +++ b/plugins/traefik/templates/traefik/widget_request_chart.html @@ -0,0 +1,75 @@ +{# traefik/widget_request_chart.html — Chart.js request rate + error % over time #} +{% if not labels_json or labels_json == '[]' %} +
No data for last {{ hours }}h.
+{% else %} +
+ +
+ +
+ last {{ hours }}h +
+{% endif %} diff --git a/plugins/unifi/__init__.py b/plugins/unifi/__init__.py new file mode 100644 index 0000000..4bb589f --- /dev/null +++ b/plugins/unifi/__init__.py @@ -0,0 +1,28 @@ +# plugins/unifi/__init__.py +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quart import Quart + +_app: "Quart | None" = None + + +def setup(app: "Quart") -> None: + global _app + _app = app + from .models import ( # noqa: F401 + UnifiWanStat, UnifiClientSnapshot, UnifiWlanStat, UnifiDpiSnapshot, + UnifiDevice, UnifiKnownClient, UnifiEvent, UnifiAlarm, + UnifiSpeedtestResult, UnifiNetwork, UnifiPortForward, UnifiFwRule, + ) + + +def get_scheduled_tasks() -> list: + from .scheduler import make_poll_task + return [make_poll_task(_app)] + + +def get_blueprint(): + from .routes import unifi_bp + return unifi_bp diff --git a/plugins/unifi/client.py b/plugins/unifi/client.py new file mode 100644 index 0000000..d18a7d8 --- /dev/null +++ b/plugins/unifi/client.py @@ -0,0 +1,170 @@ +# plugins/unifi/client.py +"""Async UniFi Network controller API client. + +Supports UDM/UDM-Pro (firmware 2+) using the /proxy/network/ path prefix +and Bearer-token auth via the /api/auth/login endpoint. + +On 401, re-authenticates automatically once before raising. +""" +from __future__ import annotations +import logging + +import httpx + +logger = logging.getLogger(__name__) + + +class UnifiClient: + def __init__( + self, + host: str, + username: str, + password: str, + site: str = "default", + verify_ssl: bool = False, + ) -> None: + self._host = host.rstrip("/") + self._username = username + self._password = password + self._site = site + self._verify_ssl = verify_ssl + self._http: httpx.AsyncClient | None = None + self._csrf_token: str = "" + + # ── Lifecycle ───────────────────────────────────────────────────────────── + + async def _ensure_http(self) -> None: + if self._http is None: + self._http = httpx.AsyncClient( + verify=self._verify_ssl, + timeout=15.0, + follow_redirects=True, + ) + + async def login(self) -> None: + await self._ensure_http() + resp = await self._http.post( + f"{self._host}/api/auth/login", + json={"username": self._username, "password": self._password, "rememberMe": False}, + ) + resp.raise_for_status() + self._csrf_token = resp.headers.get("X-CSRF-Token", "") + logger.debug("UniFi login OK (csrf=%s…)", self._csrf_token[:8] if self._csrf_token else "none") + + async def close(self) -> None: + if self._http: + await self._http.aclose() + self._http = None + + # ── API helpers ─────────────────────────────────────────────────────────── + + def _api_url(self, path: str) -> str: + return f"{self._host}/proxy/network/api/s/{self._site}{path}" + + def _headers(self) -> dict: + return {"X-CSRF-Token": self._csrf_token} if self._csrf_token else {} + + async def _get(self, path: str, params: dict | None = None) -> list | dict: + await self._ensure_http() + resp = await self._http.get(self._api_url(path), headers=self._headers(), params=params) + if resp.status_code == 401: + await self.login() + resp = await self._http.get(self._api_url(path), headers=self._headers(), params=params) + resp.raise_for_status() + data = resp.json() + return data.get("data", data) + + # ── Core monitoring ─────────────────────────────────────────────────────── + + async def get_health(self) -> list[dict]: + """Subsystem health (wan, wlan, lan, www).""" + result = await self._get("/stat/health") + return result if isinstance(result, list) else [] + + async def get_active_clients(self) -> list[dict]: + """Currently connected clients.""" + result = await self._get("/stat/sta") + return result if isinstance(result, list) else [] + + async def get_devices(self) -> list[dict]: + """Managed network devices (APs, switches, gateways) with full detail.""" + result = await self._get("/stat/device") + return result if isinstance(result, list) else [] + + # ── Events & alarms ─────────────────────────────────────────────────────── + + async def get_events(self, limit: int = 200) -> list[dict]: + """Recent site events (client connects, IDS alerts, device state changes, etc.).""" + result = await self._get("/stat/event", params={"_limit": limit, "_sort": "-time"}) + return result if isinstance(result, list) else [] + + async def get_alarms(self, archived: bool = False) -> list[dict]: + """Active (or archived) alarms.""" + result = await self._get("/stat/alarm") + if not isinstance(result, list): + return [] + return [a for a in result if a.get("archived", False) == archived] + + # ── Traffic & DPI ───────────────────────────────────────────────────────── + + async def get_dpi_stats(self) -> list[dict]: + """DPI per-client application/category traffic stats (cumulative counters).""" + result = await self._get("/stat/dpi") + return result if isinstance(result, list) else [] + + async def get_dpi_site_stats(self) -> list[dict]: + """DPI site-level category totals (not per-client).""" + result = await self._get("/dpi") + return result if isinstance(result, list) else [] + + async def get_speedtest_status(self) -> dict | None: + """Most recent WAN speed test result.""" + result = await self._get("/stat/speedtest-status") + if isinstance(result, list) and result: + return result[0] + if isinstance(result, dict): + return result + return None + + # ── Network inventory ───────────────────────────────────────────────────── + + async def get_known_clients(self) -> list[dict]: + """All historically seen clients (not just currently connected).""" + result = await self._get("/rest/user") + return result if isinstance(result, list) else [] + + async def get_networks(self) -> list[dict]: + """Network/VLAN configurations.""" + result = await self._get("/rest/networkconf") + return result if isinstance(result, list) else [] + + async def get_wlan_configs(self) -> list[dict]: + """WLAN (SSID) configurations.""" + result = await self._get("/rest/wlanconf") + return result if isinstance(result, list) else [] + + async def get_port_forwards(self) -> list[dict]: + """Port forwarding rules.""" + result = await self._get("/rest/portforwarding") + return result if isinstance(result, list) else [] + + async def get_firewall_rules(self) -> list[dict]: + """Firewall rules.""" + result = await self._get("/rest/firewallrule") + return result if isinstance(result, list) else [] + + async def get_firewall_groups(self) -> list[dict]: + """Firewall groups (IP sets / port sets referenced by rules).""" + result = await self._get("/rest/firewallgroup") + return result if isinstance(result, list) else [] + + # ── System ──────────────────────────────────────────────────────────────── + + async def get_sysinfo(self) -> dict | None: + """Controller system information (version, uptime, hostname).""" + result = await self._get("/stat/sysinfo") + if isinstance(result, list) and result: + return result[0] + if isinstance(result, dict): + return result + return None diff --git a/plugins/unifi/migrations/__init__.py b/plugins/unifi/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/unifi/migrations/env.py b/plugins/unifi/migrations/env.py new file mode 100644 index 0000000..56f15d5 --- /dev/null +++ b/plugins/unifi/migrations/env.py @@ -0,0 +1,72 @@ +# plugins/unifi/migrations/env.py +"""Alembic env.py for the UniFi plugin (standalone dev use). +At app startup, migration_runner.py uses the core env.py with version_locations. +""" +import asyncio +import os +import sys +from pathlib import Path +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config +from alembic import context + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from fabledscryer.models.base import Base +import fabledscryer.models # noqa: F401 +from plugins.unifi.models import UnifiWanStat, UnifiClientSnapshot, UnifiDevice # noqa: F401 + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _get_url() -> str: + import yaml + cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml") + try: + with open(cfg_path) as f: + cfg = yaml.safe_load(f) or {} + url = cfg.get("database", {}).get("url", "") + except FileNotFoundError: + url = "" + return os.environ.get("FABLEDSCRYER_DATABASE__URL", url) + + +def run_migrations_offline() -> None: + context.configure( + url=_get_url(), target_metadata=target_metadata, + literal_binds=True, dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + cfg = config.get_section(config.config_ini_section, {}) + cfg["sqlalchemy.url"] = _get_url() + connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/plugins/unifi/migrations/versions/__init__.py b/plugins/unifi/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/unifi/migrations/versions/unifi_001_initial.py b/plugins/unifi/migrations/versions/unifi_001_initial.py new file mode 100644 index 0000000..36158a3 --- /dev/null +++ b/plugins/unifi/migrations/versions/unifi_001_initial.py @@ -0,0 +1,62 @@ +"""UniFi plugin initial tables + +Revision ID: unifi_001_initial +Revises: (none — branch off core via depends_on) +Create Date: 2026-03-20 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "unifi_001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "unifi" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "unifi_wan_stats", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("is_up", sa.Boolean, nullable=False, server_default="false"), + sa.Column("wan_ip", sa.String(64), nullable=True), + sa.Column("latency_ms", sa.Float, nullable=True), + sa.Column("rx_bytes_rate", sa.Float, nullable=True), + sa.Column("tx_bytes_rate", sa.Float, nullable=True), + sa.Column("uptime_seconds", sa.Integer, nullable=True), + ) + op.create_index("ix_unifi_wan_stats_scraped", "unifi_wan_stats", ["scraped_at"]) + + op.create_table( + "unifi_client_snapshots", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("total_clients", sa.Integer, nullable=False, server_default="0"), + sa.Column("wireless_clients", sa.Integer, nullable=False, server_default="0"), + sa.Column("wired_clients", sa.Integer, nullable=False, server_default="0"), + sa.Column("guest_clients", sa.Integer, nullable=False, server_default="0"), + sa.Column("top_clients_json", sa.Text, nullable=False, server_default="[]"), + ) + op.create_index("ix_unifi_client_snapshots_scraped", "unifi_client_snapshots", ["scraped_at"]) + + op.create_table( + "unifi_devices", + sa.Column("mac", sa.String(32), primary_key=True), + sa.Column("name", sa.String(255), nullable=True), + sa.Column("model", sa.String(64), nullable=True), + sa.Column("device_type", sa.String(16), nullable=True), + sa.Column("ip", sa.String(64), nullable=True), + sa.Column("state", sa.Integer, nullable=False, server_default="0"), + sa.Column("uptime_seconds", sa.Integer, nullable=True), + sa.Column("last_seen", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("unifi_devices") + op.drop_index("ix_unifi_client_snapshots_scraped", "unifi_client_snapshots") + op.drop_table("unifi_client_snapshots") + op.drop_index("ix_unifi_wan_stats_scraped", "unifi_wan_stats") + op.drop_table("unifi_wan_stats") diff --git a/plugins/unifi/migrations/versions/unifi_002_expanded.py b/plugins/unifi/migrations/versions/unifi_002_expanded.py new file mode 100644 index 0000000..7336517 --- /dev/null +++ b/plugins/unifi/migrations/versions/unifi_002_expanded.py @@ -0,0 +1,156 @@ +"""UniFi expanded tables: wlan stats, dpi, events, alarms, known clients, +speedtest results, networks, port forwards, firewall rules. +Also adds missing 'version' column to unifi_devices. + +Revision ID: unifi_002_expanded +Revises: unifi_001_initial +Create Date: 2026-03-20 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "unifi_002_expanded" +down_revision: Union[str, None] = "unifi_001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ── Patch unifi_devices to add missing version column ───────────────────── + op.add_column("unifi_devices", sa.Column("version", sa.String(64), nullable=True)) + + # ── WLAN stats (time-series per SSID+radio) ─────────────────────────────── + op.create_table( + "unifi_wlan_stats", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("ssid", sa.String(255), nullable=False), + sa.Column("radio", sa.String(8), nullable=True), + sa.Column("num_sta", sa.Integer, nullable=False, server_default="0"), + sa.Column("tx_bytes", sa.Float, nullable=True), + sa.Column("rx_bytes", sa.Float, nullable=True), + sa.Column("channel", sa.Integer, nullable=True), + sa.Column("satisfaction", sa.Integer, nullable=True), + ) + op.create_index("ix_unifi_wlan_stats_scraped", "unifi_wlan_stats", ["scraped_at"]) + + # ── DPI snapshots (site-level category totals) ──────────────────────────── + op.create_table( + "unifi_dpi_snapshots", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("by_category_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("by_client_json", sa.Text, nullable=False, server_default="[]"), + ) + op.create_index("ix_unifi_dpi_scraped", "unifi_dpi_snapshots", ["scraped_at"]) + + # ── Known clients (upserted by MAC) ─────────────────────────────────────── + op.create_table( + "unifi_known_clients", + sa.Column("mac", sa.String(32), primary_key=True), + sa.Column("hostname", sa.String(255), nullable=True), + sa.Column("name", sa.String(255), nullable=True), + sa.Column("ip", sa.String(64), nullable=True), + sa.Column("mac_vendor", sa.String(128), nullable=True), + sa.Column("note", sa.Text, nullable=True), + sa.Column("is_blocked", sa.Boolean, nullable=False, server_default="false"), + sa.Column("first_seen", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_seen", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + # ── Events (upserted by UniFi event ID) ─────────────────────────────────── + op.create_table( + "unifi_events", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("event_time", sa.DateTime(timezone=True), nullable=False), + sa.Column("event_key", sa.String(64), nullable=False), + sa.Column("category", sa.String(16), nullable=False), + sa.Column("message", sa.Text, nullable=False), + sa.Column("source_mac", sa.String(32), nullable=True), + sa.Column("source_ip", sa.String(64), nullable=True), + sa.Column("details_json", sa.Text, nullable=False, server_default="{}"), + ) + op.create_index("ix_unifi_events_time", "unifi_events", ["event_time"]) + + # ── Alarms (upserted by UniFi alarm ID) ─────────────────────────────────── + op.create_table( + "unifi_alarms", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("alarm_time", sa.DateTime(timezone=True), nullable=False), + sa.Column("alarm_key", sa.String(64), nullable=False), + sa.Column("message", sa.Text, nullable=False), + sa.Column("archived", sa.Boolean, nullable=False, server_default="false"), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index("ix_unifi_alarms_time", "unifi_alarms", ["alarm_time"]) + + # ── Speedtest results (upserted by run timestamp) ───────────────────────── + op.create_table( + "unifi_speedtest_results", + sa.Column("run_at", sa.DateTime(timezone=True), primary_key=True), + sa.Column("download_mbps", sa.Float, nullable=True), + sa.Column("upload_mbps", sa.Float, nullable=True), + sa.Column("latency_ms", sa.Float, nullable=True), + sa.Column("server_name", sa.String(255), nullable=True), + ) + + # ── Networks/VLANs (upserted by UniFi ID) ──────────────────────────────── + op.create_table( + "unifi_networks", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("purpose", sa.String(32), nullable=True), + sa.Column("vlan", sa.Integer, nullable=True), + sa.Column("ip_subnet", sa.String(64), nullable=True), + sa.Column("dhcp_enabled", sa.Boolean, nullable=False, server_default="false"), + sa.Column("is_guest", sa.Boolean, nullable=False, server_default="false"), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + # ── Port forwards (upserted by UniFi ID) ───────────────────────────────── + op.create_table( + "unifi_port_forwards", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("name", sa.String(255), nullable=True), + sa.Column("proto", sa.String(8), nullable=False, server_default="tcp"), + sa.Column("dst_port", sa.String(32), nullable=False), + sa.Column("fwd_ip", sa.String(64), nullable=False), + sa.Column("fwd_port", sa.String(32), nullable=False), + sa.Column("src_filter", sa.String(255), nullable=True), + sa.Column("enabled", sa.Boolean, nullable=False, server_default="true"), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + # ── Firewall rules (upserted by UniFi ID) ──────────────────────────────── + op.create_table( + "unifi_fw_rules", + sa.Column("unifi_id", sa.String(64), primary_key=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("ruleset", sa.String(16), nullable=False), + sa.Column("rule_index", sa.Integer, nullable=True), + sa.Column("action", sa.String(16), nullable=False), + sa.Column("protocol", sa.String(16), nullable=True), + sa.Column("enabled", sa.Boolean, nullable=False, server_default="true"), + sa.Column("src_address", sa.String(255), nullable=True), + sa.Column("dst_address", sa.String(255), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("unifi_fw_rules") + op.drop_table("unifi_port_forwards") + op.drop_table("unifi_networks") + op.drop_table("unifi_speedtest_results") + op.drop_index("ix_unifi_alarms_time", "unifi_alarms") + op.drop_table("unifi_alarms") + op.drop_index("ix_unifi_events_time", "unifi_events") + op.drop_table("unifi_events") + op.drop_table("unifi_known_clients") + op.drop_index("ix_unifi_dpi_scraped", "unifi_dpi_snapshots") + op.drop_table("unifi_dpi_snapshots") + op.drop_index("ix_unifi_wlan_stats_scraped", "unifi_wlan_stats") + op.drop_table("unifi_wlan_stats") + op.drop_column("unifi_devices", "version") diff --git a/plugins/unifi/models.py b/plugins/unifi/models.py new file mode 100644 index 0000000..2ad90e9 --- /dev/null +++ b/plugins/unifi/models.py @@ -0,0 +1,182 @@ +# plugins/unifi/models.py +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from fabledscryer.models.base import Base + + +# ── Time-series (one row per scrape) ────────────────────────────────────────── + +class UnifiWanStat(Base): + """WAN interface health — one row per scrape.""" + __tablename__ = "unifi_wan_stats" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + wan_ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True) + rx_bytes_rate: Mapped[float | None] = mapped_column(Float, nullable=True) + tx_bytes_rate: Mapped[float | None] = mapped_column(Float, nullable=True) + uptime_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) + + +class UnifiClientSnapshot(Base): + """Aggregated client counts + top talkers — one row per scrape.""" + __tablename__ = "unifi_client_snapshots" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + total_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + wireless_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + wired_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + guest_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + top_clients_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + # JSON: [{"mac","hostname","ip","type","tx_rate","rx_rate","signal","essid"}] + + +class UnifiWlanStat(Base): + """Per-SSID/radio stats — one row per scrape per SSID+radio combination.""" + __tablename__ = "unifi_wlan_stats" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + ssid: Mapped[str] = mapped_column(String(255), nullable=False) + radio: Mapped[str | None] = mapped_column(String(8), nullable=True) # ng=2.4GHz, na=5GHz, 6e=6GHz + num_sta: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + tx_bytes: Mapped[float | None] = mapped_column(Float, nullable=True) + rx_bytes: Mapped[float | None] = mapped_column(Float, nullable=True) + channel: Mapped[int | None] = mapped_column(Integer, nullable=True) + satisfaction: Mapped[int | None] = mapped_column(Integer, nullable=True) # 0-100 + + +class UnifiDpiSnapshot(Base): + """DPI application/category traffic snapshot — one row per scrape (JSON blob).""" + __tablename__ = "unifi_dpi_snapshots" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + # JSON: [{"cat_id", "cat_name", "rx_bytes", "tx_bytes"}] aggregated across all clients + by_category_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + # JSON: [{"mac", "hostname", "top_cats": [{"cat_name", "bytes"}]}] + by_client_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") + + +# ── Upserted inventory (keyed by UniFi ID or MAC) ───────────────────────────── + +class UnifiDevice(Base): + """Network infrastructure devices — upserted by MAC each scrape.""" + __tablename__ = "unifi_devices" + + mac: Mapped[str] = mapped_column(String(32), primary_key=True) + name: Mapped[str | None] = mapped_column(String(255), nullable=True) + model: Mapped[str | None] = mapped_column(String(64), nullable=True) + device_type: Mapped[str | None] = mapped_column(String(16), nullable=True) + ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + state: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + uptime_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) + last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + version: Mapped[str | None] = mapped_column(String(64), nullable=True) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiKnownClient(Base): + """All historically seen network clients — upserted by MAC.""" + __tablename__ = "unifi_known_clients" + + mac: Mapped[str] = mapped_column(String(32), primary_key=True) + hostname: Mapped[str | None] = mapped_column(String(255), nullable=True) + name: Mapped[str | None] = mapped_column(String(255), nullable=True) # user-set alias + ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + mac_vendor: Mapped[str | None] = mapped_column(String(128), nullable=True) + note: Mapped[str | None] = mapped_column(Text, nullable=True) + is_blocked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + first_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiEvent(Base): + """Site event log — upserted by UniFi event ID to avoid duplicates.""" + __tablename__ = "unifi_events" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + event_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + event_key: Mapped[str] = mapped_column(String(64), nullable=False) # e.g. EVT_WU_Connected + category: Mapped[str] = mapped_column(String(16), nullable=False) # wlan, wired, ap, ids, wan, system + message: Mapped[str] = mapped_column(Text, nullable=False) + source_mac: Mapped[str | None] = mapped_column(String(32), nullable=True) + source_ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + details_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}") + + +class UnifiAlarm(Base): + """Active alarms — upserted by UniFi alarm ID.""" + __tablename__ = "unifi_alarms" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + alarm_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + alarm_key: Mapped[str] = mapped_column(String(64), nullable=False) + message: Mapped[str] = mapped_column(Text, nullable=False) + archived: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiSpeedtestResult(Base): + """WAN speed test results — upserted by run timestamp.""" + __tablename__ = "unifi_speedtest_results" + + run_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), primary_key=True) + download_mbps: Mapped[float | None] = mapped_column(Float, nullable=True) + upload_mbps: Mapped[float | None] = mapped_column(Float, nullable=True) + latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True) + server_name: Mapped[str | None] = mapped_column(String(255), nullable=True) + + +# ── Config inventory (upserted by UniFi ID) ─────────────────────────────────── + +class UnifiNetwork(Base): + """VLAN/network configurations — upserted by UniFi ID.""" + __tablename__ = "unifi_networks" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + purpose: Mapped[str | None] = mapped_column(String(32), nullable=True) # corporate, guest, vlan-only + vlan: Mapped[int | None] = mapped_column(Integer, nullable=True) + ip_subnet: Mapped[str | None] = mapped_column(String(64), nullable=True) + dhcp_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_guest: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiPortForward(Base): + """Port forwarding rules — upserted by UniFi ID.""" + __tablename__ = "unifi_port_forwards" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + name: Mapped[str | None] = mapped_column(String(255), nullable=True) + proto: Mapped[str] = mapped_column(String(8), nullable=False, default="tcp") + dst_port: Mapped[str] = mapped_column(String(32), nullable=False) + fwd_ip: Mapped[str] = mapped_column(String(64), nullable=False) + fwd_port: Mapped[str] = mapped_column(String(32), nullable=False) + src_filter: Mapped[str | None] = mapped_column(String(255), nullable=True) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + + +class UnifiFwRule(Base): + """Firewall rules — upserted by UniFi ID.""" + __tablename__ = "unifi_fw_rules" + + unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + ruleset: Mapped[str] = mapped_column(String(16), nullable=False) # WAN_IN, LAN_IN, etc. + rule_index: Mapped[int | None] = mapped_column(Integer, nullable=True) + action: Mapped[str] = mapped_column(String(16), nullable=False) # accept, drop, reject + protocol: Mapped[str | None] = mapped_column(String(16), nullable=True) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + src_address: Mapped[str | None] = mapped_column(String(255), nullable=True) + dst_address: Mapped[str | None] = mapped_column(String(255), nullable=True) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) diff --git a/plugins/unifi/plugin.yaml b/plugins/unifi/plugin.yaml new file mode 100644 index 0000000..208ace5 --- /dev/null +++ b/plugins/unifi/plugin.yaml @@ -0,0 +1,21 @@ +name: unifi +version: "1.0.0" +description: "UniFi Network controller integration — WAN health, devices, clients, DPI" +author: "FabledScryer" +license: "MIT" +min_app_version: "0.1.0" +repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/unifi" +tags: + - network + - unifi + - ubiquiti + +config: + host: "https://192.168.1.1" + username: "admin" + password: "" + site: "default" + verify_ssl: false + poll_interval_seconds: 60 + top_clients: 10 # number of top-talker clients to store per snapshot diff --git a/plugins/unifi/routes.py b/plugins/unifi/routes.py new file mode 100644 index 0000000..2a57330 --- /dev/null +++ b/plugins/unifi/routes.py @@ -0,0 +1,357 @@ +# plugins/unifi/routes.py +from __future__ import annotations +import json +from datetime import datetime, timezone +from quart import Blueprint, current_app, render_template, request +from sqlalchemy import Integer, cast, func, select + +from fabledscryer.auth.middleware import require_role +from fabledscryer.models.users import UserRole +from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample +from .models import ( + UnifiAlarm, UnifiClientSnapshot, UnifiDevice, UnifiDpiSnapshot, + UnifiEvent, UnifiFwRule, UnifiKnownClient, UnifiNetwork, + UnifiPortForward, UnifiSpeedtestResult, UnifiWanStat, UnifiWlanStat, +) + +unifi_bp = Blueprint("unifi", __name__, template_folder="templates") + + +def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: + if len(values) < 2: + return f'' + mn, mx = min(values), max(values) + if mx == mn: + mx = mn + 1.0 + step = width / (len(values) - 1) + pts = [] + for i, v in enumerate(values): + x = i * step + y = height - (v - mn) / (mx - mn) * (height - 2) - 1 + pts.append(f"{x:.1f},{y:.1f}") + poly = " ".join(pts) + return ( + f'' + f'' + f'' + ) + + +@unifi_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + current_range = request.args.get("range", DEFAULT_RANGE) + return await render_template( + "unifi/index.html", + poll_interval=poll_interval, + current_range=current_range, + ) + + +@unifi_bp.get("/rows") +@require_role(UserRole.viewer) +async def rows(): + """HTMX fragment: WAN stats, devices, clients, WLAN — scoped to selected range.""" + since, range_key = parse_range(request.args.get("range")) + + b_secs = bucket_seconds(since) + + async with current_app.db_sessionmaker() as db: + # WAN history — bucketed for accurate full-range sparklines + wan_bucket = ( + cast(func.strftime('%s', UnifiWanStat.scraped_at), Integer) / b_secs + ).label("bucket") + result = await db.execute( + select( + func.avg(UnifiWanStat.latency_ms).label("latency_ms"), + func.avg(UnifiWanStat.rx_bytes_rate).label("rx_bytes_rate"), + func.avg(UnifiWanStat.tx_bytes_rate).label("tx_bytes_rate"), + func.min(UnifiWanStat.scraped_at).label("scraped_at"), + wan_bucket, + ) + .where(UnifiWanStat.scraped_at >= since) + .group_by(wan_bucket) + .order_by(wan_bucket) + ) + wan_history = result.all() + + # Latest WAN — always most recent raw row for current displayed values + result = await db.execute( + select(UnifiWanStat).order_by(UnifiWanStat.scraped_at.desc()).limit(1) + ) + latest_wan = result.scalar_one_or_none() + + # Latest client snapshot (always most recent) + result = await db.execute( + select(UnifiClientSnapshot) + .order_by(UnifiClientSnapshot.scraped_at.desc()) + .limit(1) + ) + latest_clients = result.scalar_one_or_none() + + # All devices + result = await db.execute( + select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name) + ) + devices = result.scalars().all() + + # Client count history — bucketed + cs_bucket = ( + cast(func.strftime('%s', UnifiClientSnapshot.scraped_at), Integer) / b_secs + ).label("bucket") + result = await db.execute( + select( + func.avg(UnifiClientSnapshot.total_clients).label("total_clients"), + func.min(UnifiClientSnapshot.scraped_at).label("scraped_at"), + cs_bucket, + ) + .where(UnifiClientSnapshot.scraped_at >= since) + .group_by(cs_bucket) + .order_by(cs_bucket) + ) + client_history = result.all() + + # Latest WLAN stats (always most recent per SSID) + result = await db.execute( + select(UnifiWlanStat.ssid).distinct().order_by(UnifiWlanStat.ssid) + ) + ssids = [row[0] for row in result.all()] + wlan_latest: list[UnifiWlanStat] = [] + for ssid in ssids: + result = await db.execute( + select(UnifiWlanStat) + .where(UnifiWlanStat.ssid == ssid) + .order_by(UnifiWlanStat.scraped_at.desc()) + .limit(1) + ) + w = result.scalar_one_or_none() + if w: + wlan_latest.append(w) + + # Latest speedtest + result = await db.execute( + select(UnifiSpeedtestResult) + .order_by(UnifiSpeedtestResult.run_at.desc()) + .limit(1) + ) + speedtest = result.scalar_one_or_none() + + top_clients = json.loads(latest_clients.top_clients_json) if latest_clients else [] + + sparkline_latency = _sparkline([r.latency_ms or 0 for r in wan_history]) + sparkline_rx = _sparkline([r.rx_bytes_rate or 0 for r in wan_history]) + sparkline_tx = _sparkline([r.tx_bytes_rate or 0 for r in wan_history]) + sparkline_clients = _sparkline([r.total_clients for r in client_history]) + + return await render_template( + "unifi/rows.html", + latest_wan=latest_wan, + latest_clients=latest_clients, + devices=devices, + top_clients=top_clients, + wlan_latest=wlan_latest, + speedtest=speedtest, + sparkline_latency=sparkline_latency, + sparkline_rx=sparkline_rx, + sparkline_tx=sparkline_tx, + sparkline_clients=sparkline_clients, + range_key=range_key, + ) + + +@unifi_bp.get("/events") +@require_role(UserRole.viewer) +async def events(): + """HTMX fragment: site events within selected range.""" + since, range_key = parse_range(request.args.get("range")) + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiEvent) + .where(UnifiEvent.event_time >= since) + .order_by(UnifiEvent.event_time.desc()) + .limit(500) + ) + event_rows = result.scalars().all() + + return await render_template( + "unifi/events.html", events=event_rows, range_key=range_key + ) + + +@unifi_bp.get("/dpi") +@require_role(UserRole.viewer) +async def dpi(): + """HTMX fragment: DPI traffic breakdown (latest snapshot).""" + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiDpiSnapshot) + .order_by(UnifiDpiSnapshot.scraped_at.desc()) + .limit(1) + ) + snapshot = result.scalar_one_or_none() + + categories = [] + top_clients = [] + if snapshot: + categories = json.loads(snapshot.by_category_json) + top_clients = json.loads(snapshot.by_client_json) + + return await render_template( + "unifi/dpi.html", categories=categories, top_clients=top_clients, snapshot=snapshot + ) + + +@unifi_bp.get("/security") +@require_role(UserRole.viewer) +async def security(): + """HTMX fragment: active and archived alarms.""" + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiAlarm) + .where(UnifiAlarm.archived == False) # noqa: E712 + .order_by(UnifiAlarm.alarm_time.desc()) + ) + active_alarms = result.scalars().all() + + result = await db.execute( + select(UnifiAlarm) + .where(UnifiAlarm.archived == True) # noqa: E712 + .order_by(UnifiAlarm.alarm_time.desc()) + .limit(20) + ) + archived_alarms = result.scalars().all() + + return await render_template( + "unifi/security.html", + active_alarms=active_alarms, + archived_alarms=archived_alarms, + ) + + +@unifi_bp.get("/inventory") +@require_role(UserRole.viewer) +async def inventory(): + """HTMX fragment: known clients, networks, port forwards, firewall rules.""" + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiKnownClient) + .order_by(UnifiKnownClient.last_seen.desc().nullslast()) + ) + known_clients = result.scalars().all() + + result = await db.execute( + select(UnifiNetwork).order_by(UnifiNetwork.name) + ) + networks = result.scalars().all() + + result = await db.execute( + select(UnifiPortForward).order_by(UnifiPortForward.name) + ) + port_forwards = result.scalars().all() + + result = await db.execute( + select(UnifiFwRule) + .order_by(UnifiFwRule.ruleset, UnifiFwRule.rule_index) + ) + fw_rules = result.scalars().all() + + return await render_template( + "unifi/inventory.html", + known_clients=known_clients, + networks=networks, + port_forwards=port_forwards, + fw_rules=fw_rules, + ) + + +@unifi_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """HTMX dashboard widget fragment.""" + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiWanStat).order_by(UnifiWanStat.scraped_at.desc()).limit(1) + ) + latest_wan = result.scalar_one_or_none() + + result = await db.execute( + select(UnifiClientSnapshot).order_by(UnifiClientSnapshot.scraped_at.desc()).limit(1) + ) + latest_clients = result.scalar_one_or_none() + + result = await db.execute(select(UnifiDevice)) + devices = result.scalars().all() + + result = await db.execute( + select(UnifiAlarm).where(UnifiAlarm.archived == False) # noqa: E712 + ) + active_alarms = result.scalars().all() + + offline_devices = [d for d in devices if d.state != 1] + top_clients = json.loads(latest_clients.top_clients_json)[:5] if latest_clients else [] + + return await render_template( + "unifi/widget.html", + latest_wan=latest_wan, + latest_clients=latest_clients, + offline_devices=offline_devices, + active_alarms=active_alarms, + top_clients=top_clients, + ) + + +@unifi_bp.get("/widget/clients") +@require_role(UserRole.viewer) +async def widget_clients(): + """HTMX dashboard widget: top active clients.""" + limit = max(1, min(20, int(request.args.get("limit", 5) or 5))) + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiClientSnapshot) + .order_by(UnifiClientSnapshot.scraped_at.desc()) + .limit(1) + ) + snapshot = result.scalar_one_or_none() + + clients = json.loads(snapshot.top_clients_json)[:limit] if snapshot else [] + return await render_template( + "unifi/widget_clients.html", + clients=clients, + snapshot=snapshot, + widget_id=widget_id, + ) + + +@unifi_bp.get("/widget/devices") +@require_role(UserRole.viewer) +async def widget_devices(): + """HTMX dashboard widget: infrastructure devices with optional type filter.""" + type_filter = request.args.get("type_filter", "all") + widget_id = request.args.get("wid", "0") + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name) + ) + all_devices = result.scalars().all() + + if type_filter == "wireless": + devices = [d for d in all_devices if d.device_type == "uap"] + elif type_filter == "wired": + devices = [d for d in all_devices if d.device_type != "uap"] + elif type_filter == "offline": + devices = [d for d in all_devices if d.state != 1] + else: + devices = list(all_devices) + + return await render_template( + "unifi/widget_devices.html", + devices=devices, + type_filter=type_filter, + widget_id=widget_id, + ) diff --git a/plugins/unifi/scheduler.py b/plugins/unifi/scheduler.py new file mode 100644 index 0000000..2a6345d --- /dev/null +++ b/plugins/unifi/scheduler.py @@ -0,0 +1,522 @@ +# plugins/unifi/scheduler.py +from __future__ import annotations +import json +import logging +from datetime import datetime, timezone +from typing import TYPE_CHECKING + +from fabledscryer.core.scheduler import ScheduledTask + +if TYPE_CHECKING: + from quart import Quart + +logger = logging.getLogger(__name__) + +_client = None # UnifiClient instance, initialised on first scrape + + +def make_poll_task(app: "Quart") -> ScheduledTask: + cfg = app.config["PLUGINS"]["unifi"] + interval = int(cfg.get("poll_interval_seconds", 60)) + + async def poll() -> None: + await _do_poll(app) + + return ScheduledTask( + name="unifi_poll", + coro_factory=poll, + interval_seconds=interval, + run_on_startup=True, + ) + + +async def _do_poll(app: "Quart") -> None: + global _client + + from .client import UnifiClient + from .models import UnifiClientSnapshot, UnifiDevice, UnifiWanStat + from fabledscryer.core.alerts import record_metric + + cfg = app.config["PLUGINS"]["unifi"] + + if _client is None: + _client = UnifiClient( + host=cfg["host"], + username=cfg["username"], + password=cfg["password"], + site=cfg.get("site", "default"), + verify_ssl=cfg.get("verify_ssl", False), + ) + try: + await _client.login() + except Exception as exc: + logger.warning("UniFi initial login failed: %s", exc) + _client = None + return + + try: + health = await _client.get_health() + clients = await _client.get_active_clients() + devices = await _client.get_devices() + except Exception as exc: + logger.warning("UniFi poll failed — will retry next tick: %s", exc) + _client = None # force re-auth on next tick + return + + scraped_at = datetime.now(timezone.utc) + top_n = int(cfg.get("top_clients", 10)) + + # ── WAN stat ────────────────────────────────────────────────────────────── + wan_data = next((h for h in health if h.get("subsystem") == "wan"), None) + wan_stat = UnifiWanStat(scraped_at=scraped_at) + if wan_data: + wan_stat.is_up = wan_data.get("status") == "ok" + wan_stat.wan_ip = wan_data.get("wan_ip") + wan_stat.latency_ms = wan_data.get("latency") + wan_stat.rx_bytes_rate = wan_data.get("rx_bytes-r") + wan_stat.tx_bytes_rate = wan_data.get("tx_bytes-r") + wan_stat.uptime_seconds = wan_data.get("uptime") + + # ── Client snapshot ─────────────────────────────────────────────────────── + wireless = [c for c in clients if not c.get("is_wired", True)] + wired = [c for c in clients if c.get("is_wired", True)] + guests = [c for c in clients if c.get("is_guest", False)] + + top_clients = sorted( + clients, + key=lambda c: (c.get("tx_bytes-r", 0) or 0) + (c.get("rx_bytes-r", 0) or 0), + reverse=True, + )[:top_n] + + top_clients_data = [ + { + "mac": c.get("mac", ""), + "hostname": c.get("hostname") or c.get("name") or c.get("mac", ""), + "ip": c.get("ip", ""), + "type": "wireless" if not c.get("is_wired", True) else "wired", + "tx_rate": c.get("tx_bytes-r", 0) or 0, + "rx_rate": c.get("rx_bytes-r", 0) or 0, + "signal": c.get("signal"), + "essid": c.get("essid"), + } + for c in top_clients + ] + + client_snapshot = UnifiClientSnapshot( + scraped_at=scraped_at, + total_clients=len(clients), + wireless_clients=len(wireless), + wired_clients=len(wired), + guest_clients=len(guests), + top_clients_json=json.dumps(top_clients_data), + ) + + async with app.db_sessionmaker() as session: + async with session.begin(): + session.add(wan_stat) + session.add(client_snapshot) + + # ── Devices (upsert by MAC) ─────────────────────────────────────── + for d in devices: + mac = d.get("mac", "") + if not mac: + continue + last_seen_ts = d.get("last_seen") + last_seen_dt = ( + datetime.fromtimestamp(last_seen_ts, tz=timezone.utc) + if last_seen_ts else None + ) + existing = await session.get(UnifiDevice, mac) + if existing: + existing.name = d.get("name") + existing.model = d.get("model") + existing.device_type = d.get("type") + existing.ip = d.get("ip") + existing.state = d.get("state", 0) + existing.uptime_seconds = d.get("uptime") + existing.last_seen = last_seen_dt + existing.version = d.get("version") + existing.scraped_at = scraped_at + else: + session.add(UnifiDevice( + mac=mac, + name=d.get("name"), + model=d.get("model"), + device_type=d.get("type"), + ip=d.get("ip"), + state=d.get("state", 0), + uptime_seconds=d.get("uptime"), + last_seen=last_seen_dt, + version=d.get("version"), + scraped_at=scraped_at, + )) + + # ── Alert metrics ───────────────────────────────────────────────── + if wan_stat.latency_ms is not None: + await record_metric( + session=session, + source_module="unifi", + resource_name="wan", + metric_name="latency_ms", + value=wan_stat.latency_ms, + ) + await record_metric( + session=session, + source_module="unifi", + resource_name="wan", + metric_name="is_up", + value=1.0 if wan_stat.is_up else 0.0, + ) + await record_metric( + session=session, + source_module="unifi", + resource_name="clients", + metric_name="total_clients", + value=float(len(clients)), + ) + + logger.debug( + "UniFi poll: wan=%s, clients=%d, devices=%d", + "up" if wan_stat.is_up else "down", + len(clients), + len(devices), + ) + + # ── Expanded data (best-effort, failures don't abort core poll) ─────────── + try: + await _do_expanded(app, scraped_at) + except Exception as exc: + logger.warning("UniFi expanded poll failed: %s", exc) + + +async def _do_expanded(app: "Quart", scraped_at: datetime) -> None: + """Fetch supplementary UniFi data: WLAN stats, events, alarms, DPI, + known clients, speedtest, networks, port forwards, firewall rules.""" + from .models import ( + UnifiAlarm, UnifiDpiSnapshot, UnifiEvent, UnifiFwRule, + UnifiKnownClient, UnifiNetwork, UnifiPortForward, UnifiSpeedtestResult, + UnifiWlanStat, + ) + + # ── Fetch from controller ───────────────────────────────────────────────── + try: + wlan_configs = await _client.get_wlan_configs() + except Exception: + wlan_configs = [] + + try: + events = await _client.get_events(limit=200) + except Exception: + events = [] + + try: + alarms = await _client.get_alarms(archived=False) + except Exception: + alarms = [] + + try: + dpi_cats = await _client.get_dpi_site_stats() + except Exception: + dpi_cats = [] + + try: + dpi_clients = await _client.get_dpi_stats() + except Exception: + dpi_clients = [] + + try: + known_clients = await _client.get_known_clients() + except Exception: + known_clients = [] + + try: + speedtest = await _client.get_speedtest_status() + except Exception: + speedtest = None + + try: + networks = await _client.get_networks() + except Exception: + networks = [] + + try: + port_forwards = await _client.get_port_forwards() + except Exception: + port_forwards = [] + + try: + fw_rules = await _client.get_firewall_rules() + except Exception: + fw_rules = [] + + # ── Derive WLAN stats from device VAP table ─────────────────────────────── + # WLAN usage stats live on the device objects' vap_table, not wlan configs. + # We get per-radio SSID data from the active clients grouped by essid. + # Store one row per SSID using wlan_configs for channel/satisfaction. + wlan_rows: list[UnifiWlanStat] = [] + for wc in wlan_configs: + ssid = wc.get("name") or wc.get("x_passphrase", "") + if not ssid: + continue + wlan_rows.append(UnifiWlanStat( + scraped_at=scraped_at, + ssid=ssid, + radio=None, + num_sta=wc.get("num_sta", 0) or 0, + tx_bytes=wc.get("tx_bytes"), + rx_bytes=wc.get("rx_bytes"), + channel=None, + satisfaction=wc.get("satisfaction"), + )) + + # ── Build DPI category summary ──────────────────────────────────────────── + cat_summary = [] + for entry in dpi_cats: + cat_id = entry.get("cat") or entry.get("cat_id") + cat_name = entry.get("cat_name") or str(cat_id) + rx = entry.get("rx_bytes", 0) or 0 + tx = entry.get("tx_bytes", 0) or 0 + if rx or tx: + cat_summary.append({"cat_id": cat_id, "cat_name": cat_name, "rx_bytes": rx, "tx_bytes": tx}) + cat_summary.sort(key=lambda x: x["rx_bytes"] + x["tx_bytes"], reverse=True) + + # Top 10 clients by DPI total + client_dpi: dict[str, dict] = {} + for entry in dpi_clients: + mac = entry.get("mac", "") + if not mac: + continue + if mac not in client_dpi: + client_dpi[mac] = {"mac": mac, "hostname": entry.get("hostname", mac), "bytes": 0, "top_cats": []} + client_dpi[mac]["bytes"] += (entry.get("rx_bytes", 0) or 0) + (entry.get("tx_bytes", 0) or 0) + top_dpi_clients = sorted(client_dpi.values(), key=lambda x: x["bytes"], reverse=True)[:10] + + # ── Write to DB ─────────────────────────────────────────────────────────── + async with app.db_sessionmaker() as session: + async with session.begin(): + # WLAN stats + for row in wlan_rows: + session.add(row) + + # DPI snapshot (only if we got data) + if cat_summary or top_dpi_clients: + session.add(UnifiDpiSnapshot( + scraped_at=scraped_at, + by_category_json=json.dumps(cat_summary), + by_client_json=json.dumps(top_dpi_clients), + )) + + # Events (upsert by _id) + _category_map = { + "wlan": "wlan", "wireless": "wlan", + "wired": "wired", + "ugw": "wan", "wan": "wan", + "ids": "ids", + "system": "system", + "ap": "ap", + } + for ev in events: + uid = ev.get("_id", "") + if not uid: + continue + ts_raw = ev.get("datetime") or ev.get("time") + try: + if isinstance(ts_raw, (int, float)): + event_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc) + else: + event_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) + except Exception: + event_time = scraped_at + subsystem = ev.get("subsystem", "system") + category = _category_map.get(subsystem, subsystem[:16]) + details = {k: v for k, v in ev.items() + if k not in ("_id", "datetime", "time", "key", "subsystem", "msg", "user", "src_ip")} + existing = await session.get(UnifiEvent, uid) + if not existing: + session.add(UnifiEvent( + unifi_id=uid, + event_time=event_time, + event_key=ev.get("key", ""), + category=category, + message=ev.get("msg", ""), + source_mac=ev.get("user") or ev.get("src_mac"), + source_ip=ev.get("src_ip"), + details_json=json.dumps(details), + )) + + # Alarms (upsert) + for al in alarms: + uid = al.get("_id", "") + if not uid: + continue + ts_raw = al.get("datetime") or al.get("time") + try: + if isinstance(ts_raw, (int, float)): + alarm_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc) + else: + alarm_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) + except Exception: + alarm_time = scraped_at + existing = await session.get(UnifiAlarm, uid) + if existing: + existing.archived = al.get("archived", False) + existing.scraped_at = scraped_at + else: + session.add(UnifiAlarm( + unifi_id=uid, + alarm_time=alarm_time, + alarm_key=al.get("key", ""), + message=al.get("msg", ""), + archived=al.get("archived", False), + scraped_at=scraped_at, + )) + + # Known clients (upsert by MAC) + for kc in known_clients: + mac = kc.get("mac", "") + if not mac: + continue + def _ts(val): + if not val: + return None + try: + if isinstance(val, (int, float)): + return datetime.fromtimestamp(val, tz=timezone.utc) + return datetime.fromisoformat(str(val).replace("Z", "+00:00")) + except Exception: + return None + existing = await session.get(UnifiKnownClient, mac) + if existing: + existing.hostname = kc.get("hostname") + existing.name = kc.get("name") + existing.ip = kc.get("ip") + existing.mac_vendor = kc.get("oui") + existing.note = kc.get("note") + existing.is_blocked = kc.get("blocked", False) + existing.first_seen = _ts(kc.get("first_seen")) + existing.last_seen = _ts(kc.get("last_seen")) + existing.scraped_at = scraped_at + else: + session.add(UnifiKnownClient( + mac=mac, + hostname=kc.get("hostname"), + name=kc.get("name"), + ip=kc.get("ip"), + mac_vendor=kc.get("oui"), + note=kc.get("note"), + is_blocked=kc.get("blocked", False), + first_seen=_ts(kc.get("first_seen")), + last_seen=_ts(kc.get("last_seen")), + scraped_at=scraped_at, + )) + + # Speedtest (upsert by run_at) + if speedtest: + ts_raw = speedtest.get("rundate") or speedtest.get("time") + try: + if isinstance(ts_raw, (int, float)): + run_at = datetime.fromtimestamp(ts_raw, tz=timezone.utc) + else: + run_at = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) + except Exception: + run_at = scraped_at + existing = await session.get(UnifiSpeedtestResult, run_at) + if not existing: + session.add(UnifiSpeedtestResult( + run_at=run_at, + download_mbps=speedtest.get("xput_download"), + upload_mbps=speedtest.get("xput_upload"), + latency_ms=speedtest.get("latency"), + server_name=speedtest.get("server_name"), + )) + + # Networks (upsert by UniFi ID) + for net in networks: + uid = net.get("_id", "") + if not uid: + continue + existing = await session.get(UnifiNetwork, uid) + if existing: + existing.name = net.get("name", "") + existing.purpose = net.get("purpose") + existing.vlan = net.get("vlan") + existing.ip_subnet = net.get("ip_subnet") + existing.dhcp_enabled = net.get("dhcpd_enabled", False) + existing.is_guest = net.get("is_guest", False) + existing.scraped_at = scraped_at + else: + session.add(UnifiNetwork( + unifi_id=uid, + name=net.get("name", ""), + purpose=net.get("purpose"), + vlan=net.get("vlan"), + ip_subnet=net.get("ip_subnet"), + dhcp_enabled=net.get("dhcpd_enabled", False), + is_guest=net.get("is_guest", False), + scraped_at=scraped_at, + )) + + # Port forwards (upsert by UniFi ID) + for pf in port_forwards: + uid = pf.get("_id", "") + if not uid: + continue + existing = await session.get(UnifiPortForward, uid) + if existing: + existing.name = pf.get("name") + existing.proto = pf.get("proto", "tcp") + existing.dst_port = pf.get("dst_port", "") + existing.fwd_ip = pf.get("fwd", "") + existing.fwd_port = pf.get("fwd_port", "") + existing.src_filter = pf.get("src") + existing.enabled = pf.get("enabled", True) + existing.scraped_at = scraped_at + else: + session.add(UnifiPortForward( + unifi_id=uid, + name=pf.get("name"), + proto=pf.get("proto", "tcp"), + dst_port=pf.get("dst_port", ""), + fwd_ip=pf.get("fwd", ""), + fwd_port=pf.get("fwd_port", ""), + src_filter=pf.get("src"), + enabled=pf.get("enabled", True), + scraped_at=scraped_at, + )) + + # Firewall rules (upsert by UniFi ID) + for rule in fw_rules: + uid = rule.get("_id", "") + if not uid: + continue + existing = await session.get(UnifiFwRule, uid) + if existing: + existing.name = rule.get("name", "") + existing.ruleset = rule.get("ruleset", "") + existing.rule_index = rule.get("rule_index") + existing.action = rule.get("action", "") + existing.protocol = rule.get("protocol") + existing.enabled = rule.get("enabled", True) + existing.src_address = rule.get("src_address") + existing.dst_address = rule.get("dst_address") + existing.scraped_at = scraped_at + else: + session.add(UnifiFwRule( + unifi_id=uid, + name=rule.get("name", ""), + ruleset=rule.get("ruleset", ""), + rule_index=rule.get("rule_index"), + action=rule.get("action", ""), + protocol=rule.get("protocol"), + enabled=rule.get("enabled", True), + src_address=rule.get("src_address"), + dst_address=rule.get("dst_address"), + scraped_at=scraped_at, + )) + + logger.debug( + "UniFi expanded: wlans=%d, events=%d, alarms=%d, known_clients=%d, " + "networks=%d, port_fwds=%d, fw_rules=%d", + len(wlan_rows), len(events), len(alarms), len(known_clients), + len(networks), len(port_forwards), len(fw_rules), + ) diff --git a/plugins/unifi/templates/unifi/dpi.html b/plugins/unifi/templates/unifi/dpi.html new file mode 100644 index 0000000..6e085b7 --- /dev/null +++ b/plugins/unifi/templates/unifi/dpi.html @@ -0,0 +1,65 @@ +{# plugins/unifi/templates/unifi/dpi.html #} +{% if not snapshot %} +
+

No DPI data yet. DPI must be enabled on the UniFi controller.

+
+{% else %} +
+ + {# Category breakdown #} + {% if categories %} +
+
+ Traffic by Category + {{ snapshot.scraped_at.strftime("%H:%M") }} +
+ {% set total_bytes = categories | sum(attribute='rx_bytes') + categories | sum(attribute='tx_bytes') %} + + {% for cat in categories[:20] %} + {% set bytes = cat.rx_bytes + cat.tx_bytes %} + {% set pct = (bytes / total_bytes * 100) if total_bytes else 0 %} + + + + + + {% endfor %} +
{{ cat.cat_name or cat.cat_id }} + {% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB + {% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB + {% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB + {% else %}{{ bytes }} B{% endif %} + +
+
+
+
+
+ {% endif %} + + {# Top clients by DPI #} + {% if top_clients %} +
+
Top Clients by Traffic
+ + {% for c in top_clients %} + + + + + {% endfor %} +
+ {{ c.hostname or c.mac }} + {{ c.mac }} + + {% set bytes = c.bytes %} + {% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB + {% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB + {% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB + {% else %}{{ bytes }} B{% endif %} +
+
+ {% endif %} + +
+{% endif %} diff --git a/plugins/unifi/templates/unifi/events.html b/plugins/unifi/templates/unifi/events.html new file mode 100644 index 0000000..79c4a21 --- /dev/null +++ b/plugins/unifi/templates/unifi/events.html @@ -0,0 +1,52 @@ +{# plugins/unifi/templates/unifi/events.html #} +{% if not events %} +
+

No events in the last {{ range_key }}.

+
+{% else %} +
+
+ Site Events + {{ events|length }} in {{ range_key }} +
+ + + + + + + + + + + + + + {% for ev in events %} + + + + + + + {% endfor %} +
TimeCategoryKeyMessage
+ {{ ev.event_time.strftime("%Y-%m-%d %H:%M:%S") }} + + + {{ ev.category }} + + + {{ ev.event_key.replace("EVT_", "") if ev.event_key else "" }} + + {{ ev.message }} + {% if ev.source_mac %} + {{ ev.source_mac }} + {% endif %} +
+
+{% endif %} diff --git a/plugins/unifi/templates/unifi/index.html b/plugins/unifi/templates/unifi/index.html new file mode 100644 index 0000000..b2113b6 --- /dev/null +++ b/plugins/unifi/templates/unifi/index.html @@ -0,0 +1,63 @@ +{# plugins/unifi/templates/unifi/index.html #} +{% extends "base.html" %} +{% block title %}UniFi — Fabled Scryer{% endblock %} +{% block content %} +
+
+

UniFi Network

+ polling every {{ poll_interval }}s +
+ {% include "_time_range.html" %} +
+ +{# Tab nav #} +
+ {% set tabs = [ + ("overview", "Overview", "/plugins/unifi/rows"), + ("events", "Events", "/plugins/unifi/events"), + ("dpi", "Traffic", "/plugins/unifi/dpi"), + ("security", "Security", "/plugins/unifi/security"), + ("inventory", "Inventory", "/plugins/unifi/inventory"), + ] %} + {% for tab_id, label, url in tabs %} + + {% endfor %} +
+ +
+
+ + +{% endblock %} diff --git a/plugins/unifi/templates/unifi/inventory.html b/plugins/unifi/templates/unifi/inventory.html new file mode 100644 index 0000000..62e99f7 --- /dev/null +++ b/plugins/unifi/templates/unifi/inventory.html @@ -0,0 +1,146 @@ +{# plugins/unifi/templates/unifi/inventory.html #} +
+ + {# Networks / VLANs #} + {% if networks %} +
+
Networks / VLANs
+ + + + + + + {% for net in networks %} + + + + + + {% endfor %} +
NameVLANSubnet
+ {{ net.name }} + {% if net.is_guest %} guest{% endif %} + + {{ net.vlan or "—" }} + + {{ net.ip_subnet or "—" }} +
+
+ {% endif %} + + {# Port forwards #} + {% if port_forwards %} +
+
Port Forwards
+ + {% for pf in port_forwards %} + + + + + {% endfor %} +
+ + {{ pf.name or "—" }} + + {% if not pf.enabled %} disabled{% endif %} + + {{ pf.proto }}:{{ pf.dst_port }} → {{ pf.fwd_ip }}:{{ pf.fwd_port }} +
+
+ {% endif %} + + {# Firewall rules, grouped by ruleset #} + {% if fw_rules %} + {% set ns = namespace(current_rs=None) %} +
+
Firewall Rules
+ + {% for rule in fw_rules %} + {% if rule.ruleset != ns.current_rs %} + {% set ns.current_rs = rule.ruleset %} + + + + {% endif %} + + + + + + {% endfor %} +
+ {{ rule.ruleset }} +
+ + {{ rule.name }} + + + + {{ rule.action }} + + + {% if rule.protocol %}{{ rule.protocol }}{% endif %} + {% if rule.src_address %} src:{{ rule.src_address }}{% endif %} + {% if rule.dst_address %} dst:{{ rule.dst_address }}{% endif %} +
+
+ {% endif %} + + {# Known clients #} + {% if known_clients %} +
+
+ Known Clients + {{ known_clients|length }} total +
+ + + + + + + {% for kc in known_clients[:50] %} + + + + + + {% endfor %} + {% if known_clients|length > 50 %} + + + + {% endif %} +
Name / MACIPLast Seen
+
+ {{ kc.name or kc.hostname or kc.mac }} + {% if kc.is_blocked %} blocked{% endif %} +
+ {% if kc.name or kc.hostname %} +
{{ kc.mac }}
+ {% endif %} + {% if kc.mac_vendor %} +
{{ kc.mac_vendor }}
+ {% endif %} + 		+ {{ kc.ip or "—" }} + + {% if kc.last_seen %}{{ kc.last_seen.strftime("%m-%d %H:%M") }}{% else %}—{% endif %} +
+ +{{ known_clients|length - 50 }} more +
+
+ {% endif %} + + {% if not networks and not port_forwards and not fw_rules and not known_clients %} +
+

No inventory data yet. This populates on the next poll cycle.

+
+ {% endif %} + +
diff --git a/plugins/unifi/templates/unifi/rows.html b/plugins/unifi/templates/unifi/rows.html new file mode 100644 index 0000000..9de46ce --- /dev/null +++ b/plugins/unifi/templates/unifi/rows.html @@ -0,0 +1,203 @@ +{# plugins/unifi/templates/unifi/rows.html #} + +{# ── WAN health bar ───────────────────────────────────────────────────────── #} +{% if latest_wan %} +
+ + WAN + + {% if latest_wan.is_up %}✓ Up{% else %}✗ Down{% endif %} + + {% if latest_wan.wan_ip %} + {{ latest_wan.wan_ip }} + {% endif %} + + + {% if latest_wan.latency_ms is not none %} + + Latency + + {{ "%.1f"|format(latest_wan.latency_ms) }}ms + + {{ sparkline_latency | safe }} + + {% endif %} + + {% if latest_wan.rx_bytes_rate is not none %} + + ↓ RX + + {% set bps = latest_wan.rx_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s + {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %} + + {{ sparkline_rx | safe }} + + + ↑ TX + + {% set bps = latest_wan.tx_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s + {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %} + + {{ sparkline_tx | safe }} + + {% endif %} + + {% if latest_wan.uptime_seconds is not none %} + + up {{ (latest_wan.uptime_seconds // 3600) }}h {{ ((latest_wan.uptime_seconds % 3600) // 60) }}m + + {% endif %} +
+{% endif %} + +{# ── Content columns ──────────────────────────────────────────────────────── #} +
+ + {# Client summary card #} + {% if latest_clients %} +
+
+ Clients + {{ sparkline_clients | safe }} +
+
+ + {{ latest_clients.total_clients }} + total + + + {{ latest_clients.wireless_clients }}↗ wireless   + {{ latest_clients.wired_clients }}↔ wired + {% if latest_clients.guest_clients %}  {{ latest_clients.guest_clients }} guest{% endif %} + +
+
+ {% endif %} + + {# Infrastructure devices card #} + {% if devices %} +
+
Infrastructure Devices
+ + {% for d in devices %} + + + + + + {% if d.state == 1 and d.uptime_seconds %} + + + + {% endif %} + {% endfor %} +
+ ● + {{ d.name or d.mac }} + {{ d.model or d.device_type or "" }}{{ d.ip or "" }}
+ up {{ (d.uptime_seconds // 86400) }}d {{ ((d.uptime_seconds % 86400) // 3600) }}h +
+
+ {% endif %} + + {# Top talkers card #} + {% if top_clients %} +
+
Top Talkers
+ + {% for c in top_clients %} + + + + + {% endfor %} +
+ {{ c.hostname }} + {% if c.type == "wireless" and c.signal %} + {{ c.signal }}dBm + {% endif %} + + {% set rx = c.rx_rate %}{% set tx = c.tx_rate %} + ↓ + {% if rx >= 1048576 %}{{ "%.1f"|format(rx/1048576) }}M + {% elif rx >= 1024 %}{{ "%.0f"|format(rx/1024) }}K + {% else %}{{ "%.0f"|format(rx) }}B{% endif %} + ↑ + {% if tx >= 1048576 %}{{ "%.1f"|format(tx/1048576) }}M + {% elif tx >= 1024 %}{{ "%.0f"|format(tx/1024) }}K + {% else %}{{ "%.0f"|format(tx) }}B{% endif %} +
+
+ {% endif %} + + {# WLAN SSIDs card #} + {% if wlan_latest %} +
+
Wireless Networks
+ + + + + + + {% for w in wlan_latest %} + + + + + + {% endfor %} +
SSIDClientsScore
{{ w.ssid }}{{ w.num_sta }} + {% if w.satisfaction is not none %}{{ w.satisfaction }}%{% else %}—{% endif %} +
+
+ {% endif %} + + {# Speedtest card #} + {% if speedtest %} +
+
Last Speed Test
+
+ {% if speedtest.download_mbps is not none %} + + ↓ DL + {{ "%.1f"|format(speedtest.download_mbps) }} + Mbps + + {% endif %} + {% if speedtest.upload_mbps is not none %} + + ↑ UL + {{ "%.1f"|format(speedtest.upload_mbps) }} + Mbps + + {% endif %} + {% if speedtest.latency_ms is not none %} + + Latency + {{ "%.0f"|format(speedtest.latency_ms) }}ms + + {% endif %} +
+ {% if speedtest.server_name %} +
via {{ speedtest.server_name }}
+ {% endif %} +
{{ speedtest.run_at.strftime("%Y-%m-%d %H:%M") }} UTC
+
+ {% endif %} + +
+ +{% if not latest_wan and not devices %} +
+

No UniFi data yet. Check your host, username, and password in the plugin config.

+
+{% endif %} diff --git a/plugins/unifi/templates/unifi/security.html b/plugins/unifi/templates/unifi/security.html new file mode 100644 index 0000000..f28d4c0 --- /dev/null +++ b/plugins/unifi/templates/unifi/security.html @@ -0,0 +1,56 @@ +{# plugins/unifi/templates/unifi/security.html #} +
+ + {# Active alarms #} +
+
+ Active Alarms + {% if active_alarms %} + {{ active_alarms|length }} + {% endif %} +
+ {% if not active_alarms %} +

✓ No active alarms

+ {% else %} + + {% for al in active_alarms %} + + + + {% endfor %} +
+
+ ⚠ {{ al.alarm_key }} +
+
{{ al.message }}
+
+ {{ al.alarm_time.strftime("%Y-%m-%d %H:%M") }} UTC +
+
+ {% endif %} +
+ + {# Recent archived alarms #} + {% if archived_alarms %} +
+
+ Recent Archived Alarms + last {{ archived_alarms|length }} +
+ + {% for al in archived_alarms %} + + + + + {% endfor %} +
+
{{ al.alarm_key }}
+
{{ al.message }}
+ 		+ {{ al.alarm_time.strftime("%m-%d %H:%M") }} +
+
+ {% endif %} + +
diff --git a/plugins/unifi/templates/unifi/widget.html b/plugins/unifi/templates/unifi/widget.html new file mode 100644 index 0000000..2993ea6 --- /dev/null +++ b/plugins/unifi/templates/unifi/widget.html @@ -0,0 +1,95 @@ +{# plugins/unifi/templates/unifi/widget.html #} + +{# Active alarms #} +{% for al in active_alarms %} +
+ ⚠ {{ al.alarm_key }} + alarm +
+{% endfor %} +{# Offline device alerts #} +{% for d in offline_devices %} +
+ ⚠ {{ d.name or d.mac }} + offline +
+{% endfor %} +{% if active_alarms or offline_devices %}
{% endif %} + +{% if latest_wan %} +
+
+ WAN +
+
+ + {% if latest_wan.is_up %}Up{% else %}Down{% endif %} + + {% if latest_wan.latency_ms is not none %} + + lat + + {{ "%.0f"|format(latest_wan.latency_ms) }}ms + + + {% endif %} + {% if latest_wan.rx_bytes_rate is not none %} + + ↓ + {% set bps = latest_wan.rx_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s + {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %} + + + ↑ + {% set bps = latest_wan.tx_bytes_rate %} + {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s + {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s + {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %} + + {% endif %} +
+
+{% endif %} + +{% if latest_clients %} +
+
+ Clients +
+
+ {{ latest_clients.total_clients }} + {{ latest_clients.wireless_clients }}w / {{ latest_clients.wired_clients }}e +
+
+{% endif %} + +{% if top_clients %} +
+ {% for c in top_clients %} +
+
+ {{ c.hostname }} +
+
+ {% set total_bps = c.tx_rate + c.rx_rate %} + ↓ + {% set rx = c.rx_rate %} + {% if rx >= 1048576 %}{{ "%.1f"|format(rx / 1048576) }}M + {% elif rx >= 1024 %}{{ "%.0f"|format(rx / 1024) }}K + {% else %}{{ "%.0f"|format(rx) }}B{% endif %} + ↑ + {% set tx = c.tx_rate %} + {% if tx >= 1048576 %}{{ "%.1f"|format(tx / 1048576) }}M + {% elif tx >= 1024 %}{{ "%.0f"|format(tx / 1024) }}K + {% else %}{{ "%.0f"|format(tx) }}B{% endif %} +
+
+ {% endfor %} +
+{% endif %} + +{% if not latest_wan and not latest_clients and not active_alarms and not offline_devices %} +

No UniFi data yet.

+{% endif %} diff --git a/plugins/unifi/templates/unifi/widget_clients.html b/plugins/unifi/templates/unifi/widget_clients.html new file mode 100644 index 0000000..99a5892 --- /dev/null +++ b/plugins/unifi/templates/unifi/widget_clients.html @@ -0,0 +1,42 @@ +{# unifi/widget_clients.html — dashboard widget: top active clients #} +{% if not snapshot %} +
No client data yet.
+{% else %} + +
+
+ {{ snapshot.total_clients }} + total +
+
+ {{ snapshot.wireless_clients }} + wireless +
+
+ {{ snapshot.wired_clients }} + wired +
+
+ +{% if clients %} +
+ {% for c in clients %} +
+ + + {{ c.hostname or c.ip or c.mac }} + + {% if c.tx_rate or c.rx_rate %} + + {% if c.tx_rate %}↑{{ "%.0f" | format(c.tx_rate / 1000) }}k{% endif %} + {% if c.rx_rate %} ↓{{ "%.0f" | format(c.rx_rate / 1000) }}k{% endif %} + + {% endif %} +
+ {% endfor %} +
+{% else %} +
No clients connected.
+{% endif %} + +{% endif %} diff --git a/plugins/unifi/templates/unifi/widget_devices.html b/plugins/unifi/templates/unifi/widget_devices.html new file mode 100644 index 0000000..90994ed --- /dev/null +++ b/plugins/unifi/templates/unifi/widget_devices.html @@ -0,0 +1,36 @@ +{# unifi/widget_devices.html — dashboard widget: infrastructure devices #} +{% if not devices %} +
+ {% if type_filter != "all" %}No {{ type_filter }} devices found.{% else %}No devices found.{% endif %} +
+{% else %} + +{% set online = devices | selectattr("state", "equalto", 1) | list %} +{% set offline = devices | rejectattr("state", "equalto", 1) | list %} + +
+
+ {{ online | length }} + online +
+ {% if offline %} +
+ {{ offline | length }} + offline +
+ {% endif %} +
+ +
+ {% for d in devices %} +
+ + + {{ d.name or d.mac }} + + {{ d.model or d.device_type or "" }} +
+ {% endfor %} +
+ +{% endif %} diff --git a/steward/app.py b/steward/app.py index 676593a..b1ad710 100644 --- a/steward/app.py +++ b/steward/app.py @@ -20,13 +20,15 @@ def create_app( bootstrap = { "database_url": "postgresql+asyncpg://test/test", "secret_key": "test-secret-key", - "plugin_dir": "plugins", + "plugin_dirs": ["plugins"], + "plugin_install_dir": "plugins", } app.config.update( SECRET_KEY=bootstrap["secret_key"], DATABASE_URL=bootstrap["database_url"], - PLUGIN_DIR=bootstrap["plugin_dir"], + PLUGIN_DIRS=bootstrap["plugin_dirs"], + PLUGIN_INSTALL_DIR=bootstrap["plugin_install_dir"], TESTING=testing, ) @@ -42,7 +44,7 @@ def create_app( from .core.migration_runner import run_core_migrations run_core_migrations( app.config["DATABASE_URL"], - plugin_dir=Path(app.config["PLUGIN_DIR"]).resolve(), + plugin_dirs=[Path(d).resolve() for d in app.config["PLUGIN_DIRS"]], ) # ── 4. Load all settings from DB → populate app.config ──────────────────── @@ -81,9 +83,9 @@ def create_app( # so this is a no-op on normal startup. We still run it with all discovered dirs so # Alembic can resolve the full revision graph regardless of which plugins are enabled. if not testing: - from .core.migration_runner import run_plugin_migrations, discover_all_plugin_migration_dirs - _plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve() - _all_plugin_dirs = discover_all_plugin_migration_dirs(_plugin_dir) + from .core.migration_runner import run_plugin_migrations, discover_all_in + _plugin_dirs = [Path(d).resolve() for d in app.config["PLUGIN_DIRS"]] + _all_plugin_dirs = discover_all_in(_plugin_dirs) run_plugin_migrations(app.config["DATABASE_URL"], _all_plugin_dirs) # ── 6. Alert pipeline ────────────────────────────────────────────────────── diff --git a/steward/config.py b/steward/config.py index b119a25..9c8f1fd 100644 --- a/steward/config.py +++ b/steward/config.py @@ -15,7 +15,7 @@ def _env(suffix: str) -> str | None: return os.environ.get(f"STEWARD_{suffix}") -def load_bootstrap(config_path: Path | str | None = None) -> dict[str, str]: +def load_bootstrap(config_path: Path | str | None = None) -> dict[str, Any]: """Return the minimum bootstrap config: database_url and secret_key. This is the only config read from files/env vars at startup. @@ -48,15 +48,29 @@ def load_bootstrap(config_path: Path | str | None = None) -> dict[str, str]: ) secret_key = _resolve_secret_key(raw) - plugin_dir = ( + + # Plugin discovery spans two roots (see load_plugins / migration_runner): + # • bundled — first-party plugins shipped inside the image at repo-root + # `plugins/`; they version atomically with core and are read-only at runtime. + # • external — operator-mounted dir for third-party plugins, persisted in the + # /data volume. Downloads/installs land here, never in the bundled dir. + # Bundled is scanned first, so on a name collision the first-party plugin wins. + bundled_plugin_dir = raw.get("plugin_dir", "plugins") + external_plugin_dir = ( _env("PLUGIN_DIR") - or raw.get("plugin_dir", "plugins") + or raw.get("external_plugin_dir") + or "/data/plugins" ) + plugin_dirs = [bundled_plugin_dir] + if external_plugin_dir and external_plugin_dir != bundled_plugin_dir: + plugin_dirs.append(external_plugin_dir) return { "database_url": database_url, "secret_key": secret_key, - "plugin_dir": plugin_dir, + "plugin_dirs": plugin_dirs, + # Installs/downloads target the external (writable, persistent) dir. + "plugin_install_dir": external_plugin_dir or bundled_plugin_dir, } diff --git a/steward/core/migration_runner.py b/steward/core/migration_runner.py index d8e513c..c382ff5 100644 --- a/steward/core/migration_runner.py +++ b/steward/core/migration_runner.py @@ -24,14 +24,26 @@ def discover_all_plugin_migration_dirs(plugin_dir: Path) -> list[Path]: return dirs -def run_core_migrations(db_url: str, plugin_dir: Path | None = None) -> None: +def discover_all_in(plugin_dirs: list[Path]) -> list[Path]: + """discover_all_plugin_migration_dirs across multiple plugin roots. + + Plugins now live in more than one root (bundled + external), so the full + revision graph is the union of every root's migration dirs. + """ + dirs: list[Path] = [] + for pd in plugin_dirs: + dirs.extend(discover_all_plugin_migration_dirs(pd)) + return dirs + + +def run_core_migrations(db_url: str, plugin_dirs: list[Path] | None = None) -> None: """Run core Alembic migrations. - Includes all discovered plugin migration dirs (if plugin_dir given) so + Includes all discovered plugin migration dirs (across every plugin root) so Alembic can resolve any previously-applied plugin revisions in the graph. Called first so the app_settings table exists before loading settings. """ - dirs = discover_all_plugin_migration_dirs(plugin_dir) if plugin_dir else [] + dirs = discover_all_in(plugin_dirs) if plugin_dirs else [] _run(db_url, plugin_migration_dirs=dirs) diff --git a/steward/core/plugin_manager.py b/steward/core/plugin_manager.py index 2bc0e2c..b0c6e6d 100644 --- a/steward/core/plugin_manager.py +++ b/steward/core/plugin_manager.py @@ -31,6 +31,19 @@ def get_plugin_failures() -> dict[str, str]: return dict(_FAILED_PLUGINS) +def resolve_plugin_path(plugin_dirs: list[Path], name: str) -> Path | None: + """Return the first plugin root that contains `name`, else None. + + Roots are searched in order, so a bundled (first-party) plugin shadows an + external plugin of the same name. Pure function — no app/IO beyond exists(). + """ + for pd in plugin_dirs: + candidate = pd / name + if candidate.exists(): + return candidate + return None + + def _import_plugin(name: str, plugin_path: Path): """Load a plugin module by file path, avoiding sys.modules stdlib collisions. @@ -80,22 +93,24 @@ def load_plugins(app: "Quart") -> None: """ import steward - plugin_dir = Path(app.config["PLUGIN_DIR"]) + plugin_dirs = [Path(d) for d in app.config["PLUGIN_DIRS"]] plugins_cfg: dict = app.config["PLUGINS"] - # Ensure plugin_dir is on sys.path so plugins are importable by name - plugin_dir_str = str(plugin_dir.resolve()) - if plugin_dir_str not in sys.path: - sys.path.insert(0, plugin_dir_str) + # Ensure every plugin root is on sys.path so plugins are importable by name + for pd in plugin_dirs: + pd_str = str(pd.resolve()) + if pd_str not in sys.path: + sys.path.insert(0, pd_str) for name, cfg in list(plugins_cfg.items()): if not cfg.get("enabled", False): continue - plugin_path = plugin_dir / name - if not plugin_path.exists(): - _FAILED_PLUGINS[name] = f"Plugin directory not found: {plugin_path}" - logger.error("Plugin %r: directory %s not found, skipping", name, plugin_path) + plugin_path = resolve_plugin_path(plugin_dirs, name) + if plugin_path is None: + roots = ", ".join(str(d) for d in plugin_dirs) + _FAILED_PLUGINS[name] = f"Plugin directory not found in: {roots}" + logger.error("Plugin %r: not found in any plugin root (%s), skipping", name, roots) continue # Load and validate plugin.yaml @@ -224,10 +239,12 @@ async def download_and_install_plugin( """ import httpx - plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve() + # Downloads always land in the external (writable, persistent) install dir, + # never the read-only bundled dir. + plugin_dir = Path(app.config["PLUGIN_INSTALL_DIR"]).resolve() - # Ensure plugin_dir is on sys.path (may not be set yet if no plugins were - # enabled at startup) + # Ensure the install dir is on sys.path (may not be set yet if no plugins + # were enabled at startup) plugin_dir_str = str(plugin_dir) if plugin_dir_str not in sys.path: sys.path.insert(0, plugin_dir_str) @@ -304,9 +321,11 @@ async def download_and_install_plugin( if mdir.exists(): from steward.core.migration_runner import ( run_plugin_migrations, - discover_all_plugin_migration_dirs, + discover_all_in, ) - all_dirs = discover_all_plugin_migration_dirs(plugin_dir) + # Span every plugin root so Alembic can resolve bundled-plugin + # revisions already stamped in alembic_version. + all_dirs = discover_all_in([Path(d).resolve() for d in app.config["PLUGIN_DIRS"]]) run_plugin_migrations(app.config["DATABASE_URL"], all_dirs) except Exception: logger.exception("Plugin %r: migration failed after install", name) @@ -330,11 +349,11 @@ def hot_reload_plugin(app: "Quart", name: str) -> tuple[bool, str]: if name in _LOADED_PLUGINS: return False, "Plugin already loaded — restart required to apply updates" - plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve() - plugin_path = plugin_dir / name + plugin_dirs = [Path(d).resolve() for d in app.config["PLUGIN_DIRS"]] + plugin_path = resolve_plugin_path(plugin_dirs, name) - if not plugin_path.exists(): - return False, f"Plugin directory {plugin_path} not found" + if plugin_path is None: + return False, f"Plugin {name!r} not found in any plugin root" yaml_path = plugin_path / "plugin.yaml" if not yaml_path.exists(): @@ -383,9 +402,9 @@ def hot_reload_plugin(app: "Quart", name: str) -> tuple[bool, str]: try: from steward.core.migration_runner import ( run_plugin_migrations, - discover_all_plugin_migration_dirs, + discover_all_in, ) - all_dirs = discover_all_plugin_migration_dirs(plugin_dir) + all_dirs = discover_all_in(plugin_dirs) run_plugin_migrations(app.config["DATABASE_URL"], all_dirs) except Exception: logger.exception("Plugin %r: migration failed during hot-reload", name) diff --git a/steward/settings/routes.py b/steward/settings/routes.py index c3e5b58..8434b70 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -19,22 +19,32 @@ logger = logging.getLogger(__name__) def _discover_plugins() -> list[dict]: - """Scan PLUGIN_DIR for plugin.yaml files.""" + """Scan every plugin root for plugin.yaml files. + + Roots are scanned in order (bundled first), and a plugin name found in an + earlier root shadows the same name in a later one — matching load_plugins. + """ import yaml - plugin_dir = Path(current_app.config.get("PLUGIN_DIR", "plugins")).resolve() - plugins = [] - if not plugin_dir.exists(): - return plugins - for entry in sorted(plugin_dir.iterdir()): - yaml_path = entry / "plugin.yaml" - if entry.is_dir() and yaml_path.exists(): - try: - with yaml_path.open() as f: - meta = yaml.safe_load(f) or {} - meta["_dir"] = entry.name - plugins.append(meta) - except Exception: - logger.warning("Could not read %s", yaml_path) + plugin_dirs = [ + Path(d).resolve() + for d in current_app.config.get("PLUGIN_DIRS", ["plugins"]) + ] + plugins: list[dict] = [] + seen: set[str] = set() + for plugin_dir in plugin_dirs: + if not plugin_dir.exists(): + continue + for entry in sorted(plugin_dir.iterdir()): + yaml_path = entry / "plugin.yaml" + if entry.is_dir() and yaml_path.exists() and entry.name not in seen: + try: + with yaml_path.open() as f: + meta = yaml.safe_load(f) or {} + meta["_dir"] = entry.name + plugins.append(meta) + seen.add(entry.name) + except Exception: + logger.warning("Could not read %s", yaml_path) return plugins diff --git a/tests/test_plugin_dirs.py b/tests/test_plugin_dirs.py new file mode 100644 index 0000000..65d7cee --- /dev/null +++ b/tests/test_plugin_dirs.py @@ -0,0 +1,84 @@ +"""Unit tests for the bundled + external multi-root plugin model. + +Pure-function coverage (no DB, no app): path resolution, multi-root migration +discovery, and the config split into bundled/external roots + install dir. +""" +from __future__ import annotations + +from pathlib import Path + +from steward.core.migration_runner import discover_all_in +from steward.core.plugin_manager import resolve_plugin_path + + +def _make_plugin(root: Path, name: str, with_migrations: bool = False) -> None: + pdir = root / name + pdir.mkdir(parents=True) + (pdir / "plugin.yaml").write_text(f"name: {name}\n") + if with_migrations: + (pdir / "migrations" / "versions").mkdir(parents=True) + + +def test_resolve_plugin_path_prefers_earlier_root(tmp_path): + bundled = tmp_path / "bundled" + external = tmp_path / "external" + _make_plugin(bundled, "dupe") + _make_plugin(external, "dupe") + _make_plugin(external, "only_external") + + # Bundled is listed first → it shadows the external copy of the same name. + assert resolve_plugin_path([bundled, external], "dupe") == bundled / "dupe" + # A name only present externally still resolves. + assert resolve_plugin_path([bundled, external], "only_external") == external / "only_external" + + +def test_resolve_plugin_path_missing_returns_none(tmp_path): + assert resolve_plugin_path([tmp_path], "nope") is None + + +def test_discover_all_in_unions_roots(tmp_path): + bundled = tmp_path / "bundled" + external = tmp_path / "external" + _make_plugin(bundled, "a", with_migrations=True) + _make_plugin(bundled, "no_mig") # no migrations dir → not discovered + _make_plugin(external, "b", with_migrations=True) + + found = discover_all_in([bundled, external]) + assert (bundled / "a" / "migrations") in found + assert (external / "b" / "migrations") in found + assert (bundled / "no_mig" / "migrations") not in found + + +def test_discover_all_in_skips_missing_root(tmp_path): + bundled = tmp_path / "bundled" + _make_plugin(bundled, "a", with_migrations=True) + # A non-existent external root must not raise. + found = discover_all_in([bundled, tmp_path / "does_not_exist"]) + assert found == [bundled / "a" / "migrations"] + + +def test_bootstrap_splits_bundled_and_external(tmp_path, monkeypatch): + from steward.config import load_bootstrap + + monkeypatch.setenv("STEWARD_DATABASE_URL", "postgresql+asyncpg://x/y") + monkeypatch.setenv("STEWARD_SECRET_KEY", "k" * 32) + monkeypatch.setenv("STEWARD_PLUGIN_DIR", "/srv/external-plugins") + + boot = load_bootstrap(config_path=tmp_path / "absent.yaml") + + assert boot["plugin_dirs"] == ["plugins", "/srv/external-plugins"] + # Installs target the external (writable) root, never the bundled one. + assert boot["plugin_install_dir"] == "/srv/external-plugins" + + +def test_bootstrap_external_defaults_to_data_plugins(tmp_path, monkeypatch): + from steward.config import load_bootstrap + + monkeypatch.setenv("STEWARD_DATABASE_URL", "postgresql+asyncpg://x/y") + monkeypatch.setenv("STEWARD_SECRET_KEY", "k" * 32) + monkeypatch.delenv("STEWARD_PLUGIN_DIR", raising=False) + + boot = load_bootstrap(config_path=tmp_path / "absent.yaml") + + assert boot["plugin_dirs"] == ["plugins", "/data/plugins"] + assert boot["plugin_install_dir"] == "/data/plugins" From d925709c77c6902c7c1f35e908e2f001c7ec8c1d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 1 Jun 2026 08:37:32 -0400 Subject: [PATCH 002/126] ci: add Forgejo CI (lint + unit + Postgres integration) + ci-requirements.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single-repo CI now that plugins are bundled in-tree. Three lanes on push to dev/main, modeled on FabledCurator's canonical ci.yml (minus frontend/Redis): - lint: ruff check steward/ plugins/ tests/ (no dep install) - unit: pytest -m 'not integration' — whole current suite (testing=True mocks DB) - integration: postgres:16-alpine service via socket-discovered bridge IP; a boot-and-migrate test creates the real app, running core + all bundled-plugin migrations, then round-trips the DB — guards the folded-in migration graph. Registers the 'integration' pytest marker; adds tests/integration/test_boot.py. Co-Authored-By: Claude Opus 4.8 (1M context) --- .forgejo/workflows/ci.yml | 92 ++++++++++++++++++++++++++++++++++ ci-requirements.md | 47 +++++++++++++++++ pyproject.toml | 3 ++ tests/integration/__init__.py | 0 tests/integration/test_boot.py | 42 ++++++++++++++++ 5 files changed, 184 insertions(+) create mode 100644 .forgejo/workflows/ci.yml create mode 100644 ci-requirements.md create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/test_boot.py diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 0000000..251d2bd --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,92 @@ +name: CI + +# CI lanes per FabledRulebook forgejo.md "CI philosophy": +# - lint: ruff only, no dep install — fast-fail for the common lint bounce. +# - unit: pytest -m "not integration"; no service containers, no DB. +# - integration: postgres service container; boots the real app + runs all +# (core + bundled-plugin) migrations. +# Single repo: first-party plugins are bundled in-tree under plugins/, so the +# unit lane imports them directly — no cross-repo checkout. + +on: + push: + branches: [dev, main] + # pull_request intentionally absent — push on [dev, main] already fires CI for + # every dev commit and dev→main PRs. Single-operator repo, no fork PRs. + +jobs: + # Fast-fail lint lane. ruff is pre-installed in the ci-python image, so this + # runs with NO dependency install and surfaces lint bounces in seconds. + lint: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + steps: + - uses: actions/checkout@v4 + - name: Ruff lint + run: ruff check steward/ plugins/ tests/ + + unit: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + steps: + - uses: actions/checkout@v4 + - name: Install Steward + test deps + # uv: 5-10x faster wheel resolve than pip on cold caches. Falls back to + # pip on uv-missing runners. Steward declares its deps in pyproject; the + # [dev] extra adds pytest + pytest-asyncio. + run: | + if command -v uv >/dev/null 2>&1; then + uv pip install --system -e '.[dev]' + else + pip install -e '.[dev]' + fi + - name: Pytest (unit only — integration runs in its own lane) + run: pytest tests/ -v -m "not integration" + + integration: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + env: + STEWARD_SECRET_KEY: ci_integration_placeholder + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: steward + POSTGRES_PASSWORD: steward + POSTGRES_DB: steward + options: >- + --health-cmd "pg_isready -U steward" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + - name: Boot + migrate against Postgres + # act_runner (swarm-runner) puts service containers on the default bridge + # with no embedded DNS, so the hostname `postgres` won't resolve — we + # discover the service container's bridge IP via the docker socket. The + # job name `integration` (no underscore) is the docker-ps name filter. + run: | + set -euxo pipefail + echo "=== container landscape (diagnostic for filter scoping) ===" + docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}' + echo "=== end landscape ===" + PG=$(docker ps --filter "name=integration" --filter "ancestor=postgres:16-alpine" -q | head -n1) + test -n "$PG" + PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") + test -n "$PG_IP" + export STEWARD_DATABASE_URL="postgresql+asyncpg://steward:steward@$PG_IP:5432/steward" + for i in $(seq 1 60); do + (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break + sleep 2 + done + if command -v uv >/dev/null 2>&1; then + uv pip install --system -e '.[dev]' + else + pip install -e '.[dev]' + fi + pytest tests/integration -v -m integration --durations=10 diff --git a/ci-requirements.md b/ci-requirements.md new file mode 100644 index 0000000..01a73e4 --- /dev/null +++ b/ci-requirements.md @@ -0,0 +1,47 @@ +# CI Requirements — Steward + +> Spec: https://git.fabledsword.com/bvandeusen/CI-runner/src/branch/main/docs/process.md + +## Runtime image + +git.fabledsword.com/bvandeusen/ci-python:3.14 + +## Image deps used + +- python 3.14 +- ruff (analyzer for `steward/`, `plugins/`, `tests/`) +- docker CLI (integration lane: bridge-IP discovery of the Postgres service container) + +## Per-job tool installs + +- `uv pip install --system -e '.[dev]'` (pip fallback) — in `unit` and `integration` + jobs. Steward declares its deps in `pyproject.toml`; the `[dev]` extra adds + `pytest` + `pytest-asyncio`. No `requirements.txt` is tracked. + +## Lanes + +- **lint** — `ruff check steward/ plugins/ tests/`, no dep install. +- **unit** — `pytest -m "not integration"`. The whole current suite runs here: + it builds the app with `testing=True`, which mocks `db_sessionmaker`, so no DB + is touched. First-party plugins are bundled in-tree under `plugins/`, so plugin + unit tests import them directly — no cross-repo checkout. +- **integration** — `pytest tests/integration -m integration` against a + `postgres:16-alpine` service. The single test boots the real app + (`create_app(testing=False)`), which runs core + every bundled plugin's + migrations, then round-trips the DB. This is the guard that the folded-in + plugin migration graph resolves. + +## Notes + +- Steward has **no frontend and no Redis/Celery** (no external workers), so there + is no frontend-build lane and the only service container is Postgres. +- Integration uses Forgejo Actions `services:` + a socket-discovered bridge IP + because `act_runner` (swarm-runner v0.6+) puts services on the default bridge + with no embedded DNS. FabledCurator's `ci.yml` is the canonical example of the + pattern; Steward's is the same shape minus Redis and minus sharding. +- The job name `integration` has no underscore — `act_runner` strips underscores + from job names when building service-container labels, so the `docker ps` + name filter uses the bare job name. +- Migrations run via the app itself (`create_app` → `run_core_migrations`, async + through `asyncpg`), so the integration lane needs no separate `alembic upgrade` + step and no psycopg/sync driver. diff --git a/pyproject.toml b/pyproject.toml index 465001e..fe4179c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,9 @@ steward = "steward.cli:main" [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = ["tests"] +markers = [ + "integration: requires a live Postgres; runs only in the integration CI lane", +] [tool.hatch.build.targets.wheel] packages = ["steward"] diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/test_boot.py b/tests/integration/test_boot.py new file mode 100644 index 0000000..5ef5bad --- /dev/null +++ b/tests/integration/test_boot.py @@ -0,0 +1,42 @@ +"""Integration: boot the real app against a live Postgres. + +Creating the app with testing=False runs the core Alembic migrations AND every +bundled plugin's migrations (discovered across all plugin roots). This is the +canonical guard that the folded-in plugin migration graph resolves end-to-end — +a pure-unit test can't catch a broken plugin revision chain. + +Requires STEWARD_DATABASE_URL pointing at a live Postgres; the integration CI +lane provides it. Skipped automatically when unset so the unit lane stays green. +""" +from __future__ import annotations + +import asyncio +import os + +import pytest + +pytestmark = pytest.mark.integration + + +@pytest.mark.skipif( + not os.environ.get("STEWARD_DATABASE_URL"), + reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)", +) +def test_real_boot_runs_all_migrations(): + from sqlalchemy import text + + from steward.app import create_app + + # testing=False → init_db + run_core_migrations (core + all bundled-plugin + # migrations) + load_settings_sync + load_plugins all execute here. + app = create_app(testing=False) + assert app is not None + + async def _roundtrip(): + async with app.db_sessionmaker() as session: + # app_settings exists only if core migrations actually applied. + result = await session.execute(text("SELECT COUNT(*) FROM app_settings")) + return result.scalar() + + count = asyncio.run(_roundtrip()) + assert count is not None From af60ca446dcd1f31956b56b401b591c3a4955726 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 1 Jun 2026 11:22:20 -0400 Subject: [PATCH 003/126] fix(plugins): complete steward rename across bundled plugins; clear lint debt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fabledscryer->steward rename had only ever reached host_agent. The other five bundled plugins (http, snmp, traefik, unifi, docker) still imported `from fabledscryer.*` (package no longer exists) and read FABLEDSCRYER_* env vars — so every one of them was broken at import since the original rebrand. CI stayed green only because none are enabled by default and migrations don't import plugin modules. Now that they version in-tree, complete the rename: - fabledscryer.* -> steward.* imports across all five plugins - FABLEDSCRYER_* -> STEWARD_* in plugin migration env.py files - author/repository/homepage + user-facing 'Fabled Scryer' strings -> Steward - snmp/scheduler.py: also drop dead `now`/datetime; record_metric from steward Adds tests/test_no_legacy_names.py — fails if 'scryer'/'roundtable' ever reappear in shipped code (the drift bit twice; this stops a third time). Also clears pre-existing ruff lint debt (unused imports, semicolon statements, mid-file import) surfaced by the new lint lane. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/docker/migrations/env.py | 8 ++-- plugins/docker/models.py | 2 +- plugins/docker/plugin.yaml | 6 +-- plugins/docker/routes.py | 7 ++- plugins/docker/scheduler.py | 4 +- plugins/docker/scraper.py | 1 - plugins/docker/templates/docker/index.html | 2 +- plugins/http/checker.py | 1 - plugins/http/migrations/env.py | 8 ++-- plugins/http/models.py | 2 +- plugins/http/plugin.yaml | 6 +-- plugins/http/routes.py | 7 ++- plugins/http/scheduler.py | 6 +-- plugins/http/templates/http/index.html | 2 +- plugins/index.yaml | 44 +++++++++---------- plugins/snmp/plugin.yaml | 6 +-- plugins/snmp/poller.py | 4 +- plugins/snmp/routes.py | 6 +-- plugins/snmp/scheduler.py | 6 +-- plugins/snmp/templates/snmp/device.html | 2 +- plugins/snmp/templates/snmp/index.html | 2 +- plugins/traefik/migrations/env.py | 10 ++--- plugins/traefik/models.py | 2 +- plugins/traefik/plugin.yaml | 6 +-- plugins/traefik/routes.py | 6 +-- plugins/traefik/scheduler.py | 5 +-- plugins/traefik/templates/traefik/index.html | 2 +- plugins/unifi/migrations/env.py | 8 ++-- plugins/unifi/models.py | 2 +- plugins/unifi/plugin.yaml | 6 +-- plugins/unifi/routes.py | 7 ++- plugins/unifi/scheduler.py | 4 +- plugins/unifi/templates/unifi/index.html | 2 +- steward/ansible/routes.py | 7 ++- steward/auth/oidc.py | 1 - steward/core/plugin_manager.py | 1 - steward/core/reports.py | 2 - steward/migrations/env.py | 3 +- steward/settings/routes.py | 3 +- steward/templates/settings/auth.html | 4 +- steward/templates/settings/plugins.html | 2 +- tests/conftest.py | 1 - .../host_agent/test_agent_build_payload.py | 5 +-- .../host_agent/test_agent_ring_buffer.py | 7 ++- .../host_agent/test_install_template.py | 1 - tests/test_config.py | 1 - tests/test_models.py | 1 - tests/test_no_legacy_names.py | 35 +++++++++++++++ 48 files changed, 142 insertions(+), 124 deletions(-) create mode 100644 tests/test_no_legacy_names.py diff --git a/plugins/docker/migrations/env.py b/plugins/docker/migrations/env.py index 5c20755..3878d67 100644 --- a/plugins/docker/migrations/env.py +++ b/plugins/docker/migrations/env.py @@ -13,8 +13,8 @@ from alembic import context sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) -from fabledscryer.models.base import Base -import fabledscryer.models # noqa: F401 +from steward.models.base import Base +import steward.models # noqa: F401 from plugins.docker.models import DockerContainer, DockerMetric # noqa: F401 config = context.config @@ -26,14 +26,14 @@ target_metadata = Base.metadata def _get_url() -> str: import yaml - cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml") + cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml") try: with open(cfg_path) as f: cfg = yaml.safe_load(f) or {} url = cfg.get("database", {}).get("url", "") except FileNotFoundError: url = "" - return os.environ.get("FABLEDSCRYER_DATABASE__URL", url) + return os.environ.get("STEWARD_DATABASE__URL", url) def run_migrations_offline() -> None: diff --git a/plugins/docker/models.py b/plugins/docker/models.py index 09c1cf3..bec356b 100644 --- a/plugins/docker/models.py +++ b/plugins/docker/models.py @@ -4,7 +4,7 @@ import uuid from datetime import datetime, timezone from sqlalchemy import DateTime, Float, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column -from fabledscryer.models.base import Base +from steward.models.base import Base class DockerContainer(Base): diff --git a/plugins/docker/plugin.yaml b/plugins/docker/plugin.yaml index 8745302..9d38f24 100644 --- a/plugins/docker/plugin.yaml +++ b/plugins/docker/plugin.yaml @@ -1,11 +1,11 @@ name: docker version: "1.0.0" description: "Docker container status, resource usage, restart tracking via Docker socket" -author: "FabledScryer" +author: "Steward" license: "MIT" min_app_version: "0.1.0" -repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" -homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/docker" +repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker" tags: - containers - docker diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index 12d0383..41b123a 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -1,13 +1,12 @@ # plugins/docker/routes.py from __future__ import annotations import json -from datetime import datetime, timezone from quart import Blueprint, current_app, render_template, request from sqlalchemy import Integer, cast, func, select -from fabledscryer.auth.middleware import require_role -from fabledscryer.models.users import UserRole -from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds +from steward.auth.middleware import require_role +from steward.models.users import UserRole +from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds from .models import DockerContainer, DockerMetric docker_bp = Blueprint("docker", __name__, template_folder="templates") diff --git a/plugins/docker/scheduler.py b/plugins/docker/scheduler.py index eef8c61..827cf5f 100644 --- a/plugins/docker/scheduler.py +++ b/plugins/docker/scheduler.py @@ -4,8 +4,8 @@ import json import logging from datetime import datetime, timezone -from fabledscryer.core.scheduler import ScheduledTask -from fabledscryer.core.alerts import record_metric +from steward.core.scheduler import ScheduledTask +from steward.core.alerts import record_metric logger = logging.getLogger(__name__) diff --git a/plugins/docker/scraper.py b/plugins/docker/scraper.py index 96f601b..f4133ed 100644 --- a/plugins/docker/scraper.py +++ b/plugins/docker/scraper.py @@ -106,7 +106,6 @@ async def scrape_docker(socket_path: str, include_stopped: bool) -> list[dict]: ) from exc results = [] - now = datetime.now(timezone.utc) for container, stats in pairs: names = container.get("Names") or [] name = names[0].lstrip("/") if names else _short_id(container.get("Id", "")) diff --git a/plugins/docker/templates/docker/index.html b/plugins/docker/templates/docker/index.html index 42c9bb5..9b33918 100644 --- a/plugins/docker/templates/docker/index.html +++ b/plugins/docker/templates/docker/index.html @@ -1,5 +1,5 @@ {% extends "base.html" %} -{% block title %}Docker — Fabled Scryer{% endblock %} +{% block title %}Docker — Steward{% endblock %} {% block content %}

Docker

diff --git a/plugins/http/checker.py b/plugins/http/checker.py index 7eec743..c898ad6 100644 --- a/plugins/http/checker.py +++ b/plugins/http/checker.py @@ -2,7 +2,6 @@ """Core HTTP check logic — runs a single monitor check and returns a result dict.""" from __future__ import annotations import asyncio -import json import logging import ssl import time diff --git a/plugins/http/migrations/env.py b/plugins/http/migrations/env.py index fd308a2..02082a7 100644 --- a/plugins/http/migrations/env.py +++ b/plugins/http/migrations/env.py @@ -13,8 +13,8 @@ from alembic import context sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) -from fabledscryer.models.base import Base -import fabledscryer.models # noqa: F401 +from steward.models.base import Base +import steward.models # noqa: F401 from plugins.http.models import HttpMonitor, HttpResult # noqa: F401 config = context.config @@ -26,14 +26,14 @@ target_metadata = Base.metadata def _get_url() -> str: import yaml - cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml") + cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml") try: with open(cfg_path) as f: cfg = yaml.safe_load(f) or {} url = cfg.get("database", {}).get("url", "") except FileNotFoundError: url = "" - return os.environ.get("FABLEDSCRYER_DATABASE__URL", url) + return os.environ.get("STEWARD_DATABASE__URL", url) def run_migrations_offline() -> None: diff --git a/plugins/http/models.py b/plugins/http/models.py index 180effd..d11cb0c 100644 --- a/plugins/http/models.py +++ b/plugins/http/models.py @@ -4,7 +4,7 @@ import uuid from datetime import datetime, timezone from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column -from fabledscryer.models.base import Base +from steward.models.base import Base class HttpMonitor(Base): diff --git a/plugins/http/plugin.yaml b/plugins/http/plugin.yaml index 9aa1475..a232c4d 100644 --- a/plugins/http/plugin.yaml +++ b/plugins/http/plugin.yaml @@ -1,11 +1,11 @@ name: http version: "1.0.0" description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry" -author: "FabledScryer" +author: "Steward" license: "MIT" min_app_version: "0.1.0" -repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" -homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/http" +repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/http" tags: - monitoring - http diff --git a/plugins/http/routes.py b/plugins/http/routes.py index eea5ce7..24bede7 100644 --- a/plugins/http/routes.py +++ b/plugins/http/routes.py @@ -1,14 +1,13 @@ # plugins/http/routes.py from __future__ import annotations -import json import uuid from datetime import datetime, timezone from quart import Blueprint, current_app, render_template, request, redirect, url_for from sqlalchemy import Integer, cast, func, select -from fabledscryer.auth.middleware import require_role -from fabledscryer.models.users import UserRole -from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds +from steward.auth.middleware import require_role +from steward.models.users import UserRole +from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds from .models import HttpMonitor, HttpResult http_bp = Blueprint("http", __name__, template_folder="templates") diff --git a/plugins/http/scheduler.py b/plugins/http/scheduler.py index f9edd67..424836c 100644 --- a/plugins/http/scheduler.py +++ b/plugins/http/scheduler.py @@ -3,10 +3,10 @@ from __future__ import annotations import asyncio import json import logging -from datetime import datetime, timezone, timedelta +from datetime import datetime, timezone -from fabledscryer.core.scheduler import ScheduledTask -from fabledscryer.core.alerts import record_metric +from steward.core.scheduler import ScheduledTask +from steward.core.alerts import record_metric logger = logging.getLogger(__name__) diff --git a/plugins/http/templates/http/index.html b/plugins/http/templates/http/index.html index 6d1ffd0..a2c7771 100644 --- a/plugins/http/templates/http/index.html +++ b/plugins/http/templates/http/index.html @@ -1,5 +1,5 @@ {% extends "base.html" %} -{% block title %}HTTP Monitors — Fabled Scryer{% endblock %} +{% block title %}HTTP Monitors — Steward{% endblock %} {% block content %}

HTTP Monitors

diff --git a/plugins/index.yaml b/plugins/index.yaml index 0192afe..276306d 100644 --- a/plugins/index.yaml +++ b/plugins/index.yaml @@ -1,6 +1,6 @@ -# fabledscryer-plugins / index.yaml +# steward-plugins / index.yaml # -# Plugin catalog for Fabled Scryer. +# Plugin catalog for Steward. # Fetched by the app at: Settings → Plugins → Plugin Catalog # # After updating an entry, commit and push — changes are live within 5 minutes @@ -16,12 +16,12 @@ plugins: - name: http version: "1.0.0" description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry" - author: "FabledScryer" + author: "Steward" license: "MIT" min_app_version: "0.1.0" - repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" - homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/http" - download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/http-v1.0.0/http.zip" + repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/http" + download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/http-v1.0.0/http.zip" checksum_sha256: "" tags: - monitoring @@ -32,12 +32,12 @@ plugins: - name: docker version: "1.0.0" description: "Docker container status, resource usage, and restart tracking via Docker socket" - author: "FabledScryer" + author: "Steward" license: "MIT" min_app_version: "0.1.0" - repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" - homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/docker" - download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/docker-v1.0.0/docker.zip" + repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker" + download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v1.0.0/docker.zip" checksum_sha256: "" tags: - containers @@ -47,12 +47,12 @@ plugins: - name: traefik version: "1.0.0" description: "Traefik reverse proxy metrics and access log integration" - author: "FabledScryer" + author: "Steward" license: "MIT" min_app_version: "0.1.0" - repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" - homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/traefik" - download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/traefik-v1.0.0/traefik.zip" + repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik" + download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip" checksum_sha256: "" tags: - proxy @@ -62,12 +62,12 @@ plugins: - name: unifi version: "1.0.0" description: "UniFi Network controller integration — WAN health, devices, clients, DPI" - author: "FabledScryer" + author: "Steward" license: "MIT" min_app_version: "0.1.0" - repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" - homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/unifi" - download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/unifi-v1.0.0/unifi.zip" + repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi" + download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/unifi-v1.0.0/unifi.zip" checksum_sha256: "" tags: - network @@ -77,12 +77,12 @@ plugins: - name: ups version: "1.0.0" description: "UPS monitoring via NUT (Network UPS Tools) with Ansible shutdown automation" - author: "FabledScryer" + author: "Steward" license: "MIT" min_app_version: "0.1.0" - repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" - homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/ups" - download_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/releases/download/ups-v1.0.0/ups.zip" + repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" + homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/ups" + download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/ups-v1.0.0/ups.zip" checksum_sha256: "" tags: - ups diff --git a/plugins/snmp/plugin.yaml b/plugins/snmp/plugin.yaml index 5e98fce..ba8d8a6 100644 --- a/plugins/snmp/plugin.yaml +++ b/plugins/snmp/plugin.yaml @@ -2,11 +2,11 @@ name: snmp version: "1.0.0" description: "SNMP polling for network devices — switches, routers, printers, UPSes, etc." -author: "FabledScryer" +author: "Steward" license: "MIT" min_app_version: "0.1.0" -repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" -homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/snmp" +repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/snmp" tags: - snmp - network diff --git a/plugins/snmp/poller.py b/plugins/snmp/poller.py index 2f7e86f..6a837f6 100644 --- a/plugins/snmp/poller.py +++ b/plugins/snmp/poller.py @@ -3,7 +3,7 @@ Synchronous SNMP GET helper, run via executor. Requires pysnmp-lextudio (maintained pysnmp fork): - pip install 'fabledscryer[snmp]' + pip install 'steward[snmp]' If pysnmp is not installed, poll_device() returns an empty dict and logs a warning. """ @@ -40,7 +40,7 @@ def poll_device_sync( """ if not _pysnmp_available(): logger.warning("pysnmp not installed — SNMP polling disabled. " - "Install with: pip install 'fabledscryer[snmp]'") + "Install with: pip install 'steward[snmp]'") return {} from pysnmp.hlapi import ( diff --git a/plugins/snmp/routes.py b/plugins/snmp/routes.py index cca9697..94bffbc 100644 --- a/plugins/snmp/routes.py +++ b/plugins/snmp/routes.py @@ -5,9 +5,9 @@ from datetime import datetime, timedelta, timezone from quart import Blueprint, current_app, render_template, request from sqlalchemy import func, select -from fabledscryer.auth.middleware import require_role -from fabledscryer.models.users import UserRole -from fabledscryer.models.metrics import PluginMetric +from steward.auth.middleware import require_role +from steward.models.users import UserRole +from steward.models.metrics import PluginMetric snmp_bp = Blueprint("snmp", __name__, template_folder="templates") diff --git a/plugins/snmp/scheduler.py b/plugins/snmp/scheduler.py index b8809c3..d1a1d85 100644 --- a/plugins/snmp/scheduler.py +++ b/plugins/snmp/scheduler.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: from quart import Quart -from fabledscryer.core.scheduler import ScheduledTask +from steward.core.scheduler import ScheduledTask logger = logging.getLogger(__name__) @@ -27,16 +27,14 @@ def make_poll_task(app: "Quart") -> ScheduledTask: async def _do_poll(app: "Quart") -> None: - from datetime import datetime, timezone from .poller import poll_device_sync - from fabledscryer.core.alerts import record_metric + from steward.core.alerts import record_metric devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", []) if not devices: return loop = asyncio.get_event_loop() - now = datetime.now(timezone.utc) async with app.db_sessionmaker() as session: async with session.begin(): diff --git a/plugins/snmp/templates/snmp/device.html b/plugins/snmp/templates/snmp/device.html index 82022a9..b8fe625 100644 --- a/plugins/snmp/templates/snmp/device.html +++ b/plugins/snmp/templates/snmp/device.html @@ -1,6 +1,6 @@ {# plugins/snmp/templates/snmp/device.html #} {% extends "base.html" %} -{% block title %}SNMP — {{ device_name }} — Fabled Scryer{% endblock %} +{% block title %}SNMP — {{ device_name }} — Steward{% endblock %} {% block content %}
← SNMP diff --git a/plugins/snmp/templates/snmp/index.html b/plugins/snmp/templates/snmp/index.html index 86a1485..ef5979c 100644 --- a/plugins/snmp/templates/snmp/index.html +++ b/plugins/snmp/templates/snmp/index.html @@ -1,6 +1,6 @@ {# plugins/snmp/templates/snmp/index.html #} {% extends "base.html" %} -{% block title %}SNMP — Fabled Scryer{% endblock %} +{% block title %}SNMP — Steward{% endblock %} {% block content %}

SNMP Devices

diff --git a/plugins/traefik/migrations/env.py b/plugins/traefik/migrations/env.py index 1919fec..a523e27 100644 --- a/plugins/traefik/migrations/env.py +++ b/plugins/traefik/migrations/env.py @@ -15,11 +15,11 @@ from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config from alembic import context -# Make fabledscryer importable +# Make steward importable sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) -from fabledscryer.models.base import Base -import fabledscryer.models # noqa: F401 — core models +from steward.models.base import Base +import steward.models # noqa: F401 — core models from plugins.traefik.models import TraefikMetric # noqa: F401 — plugin model config = context.config @@ -31,14 +31,14 @@ target_metadata = Base.metadata def _get_url() -> str: import yaml - cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml") + cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml") try: with open(cfg_path) as f: cfg = yaml.safe_load(f) or {} url = cfg.get("database", {}).get("url", "") except FileNotFoundError: url = "" - return os.environ.get("FABLEDSCRYER_DATABASE__URL", url) + return os.environ.get("STEWARD_DATABASE__URL", url) def run_migrations_offline() -> None: diff --git a/plugins/traefik/models.py b/plugins/traefik/models.py index 1ba9e50..d177d10 100644 --- a/plugins/traefik/models.py +++ b/plugins/traefik/models.py @@ -4,7 +4,7 @@ import uuid from datetime import datetime, timezone from sqlalchemy import DateTime, Float, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column -from fabledscryer.models.base import Base +from steward.models.base import Base class TraefikMetric(Base): diff --git a/plugins/traefik/plugin.yaml b/plugins/traefik/plugin.yaml index 392713b..3f09ff4 100644 --- a/plugins/traefik/plugin.yaml +++ b/plugins/traefik/plugin.yaml @@ -2,11 +2,11 @@ name: traefik version: "1.0.0" description: "Traefik reverse proxy metrics and access log integration" -author: "FabledScryer" +author: "Steward" license: "MIT" min_app_version: "0.1.0" -repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" -homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/traefik" +repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik" tags: - proxy - metrics diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index f7fe3ce..80e6c93 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -6,9 +6,9 @@ from quart import Blueprint, current_app, render_template, request from sqlalchemy import Integer, cast, func, select import json -from fabledscryer.auth.middleware import require_role -from fabledscryer.models.users import UserRole -from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample +from steward.auth.middleware import require_role +from steward.models.users import UserRole +from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric traefik_bp = Blueprint("traefik", __name__, template_folder="templates") diff --git a/plugins/traefik/scheduler.py b/plugins/traefik/scheduler.py index b7eb050..678e788 100644 --- a/plugins/traefik/scheduler.py +++ b/plugins/traefik/scheduler.py @@ -5,7 +5,7 @@ import time from typing import TYPE_CHECKING from datetime import datetime -from fabledscryer.core.scheduler import ScheduledTask +from steward.core.scheduler import ScheduledTask if TYPE_CHECKING: from quart import Quart @@ -41,7 +41,6 @@ async def _do_scrape(app: "Quart") -> None: global _prev_metrics, _prev_time from datetime import datetime, timezone - from sqlalchemy import select from .models import TraefikCert, TraefikGlobalStat, TraefikMetric from .scraper import ( compute_global_stats, @@ -49,7 +48,7 @@ async def _do_scrape(app: "Quart") -> None: extract_certs, fetch_metrics, ) - from fabledscryer.core.alerts import record_metric + from steward.core.alerts import record_metric metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"] diff --git a/plugins/traefik/templates/traefik/index.html b/plugins/traefik/templates/traefik/index.html index da66fc7..18f50f0 100644 --- a/plugins/traefik/templates/traefik/index.html +++ b/plugins/traefik/templates/traefik/index.html @@ -1,6 +1,6 @@ {# plugins/traefik/templates/traefik/index.html #} {% extends "base.html" %} -{% block title %}Traefik — Fabled Scryer{% endblock %} +{% block title %}Traefik — Steward{% endblock %} {% block content %}
diff --git a/plugins/unifi/migrations/env.py b/plugins/unifi/migrations/env.py index 56f15d5..7fa8252 100644 --- a/plugins/unifi/migrations/env.py +++ b/plugins/unifi/migrations/env.py @@ -15,8 +15,8 @@ from alembic import context sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) -from fabledscryer.models.base import Base -import fabledscryer.models # noqa: F401 +from steward.models.base import Base +import steward.models # noqa: F401 from plugins.unifi.models import UnifiWanStat, UnifiClientSnapshot, UnifiDevice # noqa: F401 config = context.config @@ -28,14 +28,14 @@ target_metadata = Base.metadata def _get_url() -> str: import yaml - cfg_path = os.environ.get("FABLEDSCRYER_CONFIG", "config.yaml") + cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml") try: with open(cfg_path) as f: cfg = yaml.safe_load(f) or {} url = cfg.get("database", {}).get("url", "") except FileNotFoundError: url = "" - return os.environ.get("FABLEDSCRYER_DATABASE__URL", url) + return os.environ.get("STEWARD_DATABASE__URL", url) def run_migrations_offline() -> None: diff --git a/plugins/unifi/models.py b/plugins/unifi/models.py index 2ad90e9..bb36410 100644 --- a/plugins/unifi/models.py +++ b/plugins/unifi/models.py @@ -4,7 +4,7 @@ import uuid from datetime import datetime, timezone from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column -from fabledscryer.models.base import Base +from steward.models.base import Base # ── Time-series (one row per scrape) ────────────────────────────────────────── diff --git a/plugins/unifi/plugin.yaml b/plugins/unifi/plugin.yaml index 208ace5..76b64ed 100644 --- a/plugins/unifi/plugin.yaml +++ b/plugins/unifi/plugin.yaml @@ -1,11 +1,11 @@ name: unifi version: "1.0.0" description: "UniFi Network controller integration — WAN health, devices, clients, DPI" -author: "FabledScryer" +author: "Steward" license: "MIT" min_app_version: "0.1.0" -repository_url: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins" -homepage: "https://git.fabledsword.com/bvandeusen/FabledScryer-plugins/src/branch/main/unifi" +repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" +homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi" tags: - network - unifi diff --git a/plugins/unifi/routes.py b/plugins/unifi/routes.py index 2a57330..e5e00a6 100644 --- a/plugins/unifi/routes.py +++ b/plugins/unifi/routes.py @@ -1,13 +1,12 @@ # plugins/unifi/routes.py from __future__ import annotations import json -from datetime import datetime, timezone from quart import Blueprint, current_app, render_template, request from sqlalchemy import Integer, cast, func, select -from fabledscryer.auth.middleware import require_role -from fabledscryer.models.users import UserRole -from fabledscryer.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample +from steward.auth.middleware import require_role +from steward.models.users import UserRole +from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds from .models import ( UnifiAlarm, UnifiClientSnapshot, UnifiDevice, UnifiDpiSnapshot, UnifiEvent, UnifiFwRule, UnifiKnownClient, UnifiNetwork, diff --git a/plugins/unifi/scheduler.py b/plugins/unifi/scheduler.py index 2a6345d..e050c40 100644 --- a/plugins/unifi/scheduler.py +++ b/plugins/unifi/scheduler.py @@ -5,7 +5,7 @@ import logging from datetime import datetime, timezone from typing import TYPE_CHECKING -from fabledscryer.core.scheduler import ScheduledTask +from steward.core.scheduler import ScheduledTask if TYPE_CHECKING: from quart import Quart @@ -35,7 +35,7 @@ async def _do_poll(app: "Quart") -> None: from .client import UnifiClient from .models import UnifiClientSnapshot, UnifiDevice, UnifiWanStat - from fabledscryer.core.alerts import record_metric + from steward.core.alerts import record_metric cfg = app.config["PLUGINS"]["unifi"] diff --git a/plugins/unifi/templates/unifi/index.html b/plugins/unifi/templates/unifi/index.html index b2113b6..517c030 100644 --- a/plugins/unifi/templates/unifi/index.html +++ b/plugins/unifi/templates/unifi/index.html @@ -1,6 +1,6 @@ {# plugins/unifi/templates/unifi/index.html #} {% extends "base.html" %} -{% block title %}UniFi — Fabled Scryer{% endblock %} +{% block title %}UniFi — Steward{% endblock %} {% block content %}
diff --git a/steward/ansible/routes.py b/steward/ansible/routes.py index d01f860..1dceb26 100644 --- a/steward/ansible/routes.py +++ b/steward/ansible/routes.py @@ -3,13 +3,12 @@ from __future__ import annotations import asyncio import uuid from datetime import datetime, timezone -from pathlib import Path from quart import ( - Blueprint, Response, current_app, redirect, render_template, - request, session, url_for, + Blueprint, Response, current_app, render_template, + request, session, ) -from sqlalchemy import select, update +from sqlalchemy import select from steward.ansible import executor, sources as src_module from steward.auth.middleware import require_role diff --git a/steward/auth/oidc.py b/steward/auth/oidc.py index 934e3a8..9f8697e 100644 --- a/steward/auth/oidc.py +++ b/steward/auth/oidc.py @@ -8,7 +8,6 @@ we trust the HTTPS connection to the discovery-documented token/userinfo endpoin """ from __future__ import annotations import base64 -import hashlib import json import logging import os diff --git a/steward/core/plugin_manager.py b/steward/core/plugin_manager.py index b0c6e6d..680235c 100644 --- a/steward/core/plugin_manager.py +++ b/steward/core/plugin_manager.py @@ -1,7 +1,6 @@ # steward/core/plugin_manager.py from __future__ import annotations import hashlib -import importlib import logging import os import sys diff --git a/steward/core/reports.py b/steward/core/reports.py index 39f0d7a..7c376d0 100644 --- a/steward/core/reports.py +++ b/steward/core/reports.py @@ -6,8 +6,6 @@ Called by the scheduler (hourly check) or triggered manually from Settings. from __future__ import annotations import asyncio import logging -import smtplib -import ssl from datetime import datetime, timedelta, timezone from email.message import EmailMessage diff --git a/steward/migrations/env.py b/steward/migrations/env.py index 3da677f..0c35e5e 100644 --- a/steward/migrations/env.py +++ b/steward/migrations/env.py @@ -15,7 +15,8 @@ target_metadata = Base.metadata def get_url() -> str: - import os, yaml + import os + import yaml def _env(suffix: str) -> str | None: return os.environ.get(f"STEWARD_{suffix}") diff --git a/steward/settings/routes.py b/steward/settings/routes.py index 8434b70..f2740a4 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -1,6 +1,5 @@ # steward/settings/routes.py from __future__ import annotations -import json import logging from pathlib import Path from quart import Blueprint, current_app, render_template, request, redirect, url_for, session @@ -9,7 +8,7 @@ from steward.auth.middleware import require_role from steward.core.audit import log_audit from steward.models.users import UserRole from steward.core.settings import ( - DEFAULTS, get_all_settings, set_setting, + get_all_settings, set_setting, to_smtp_cfg, to_webhook_cfg, to_ansible_cfg, to_plugins_cfg, to_oidc_cfg, to_ldap_cfg, ) diff --git a/steward/templates/settings/auth.html b/steward/templates/settings/auth.html index 655fd08..41b69c4 100644 --- a/steward/templates/settings/auth.html +++ b/steward/templates/settings/auth.html @@ -144,12 +144,12 @@
+ placeholder="cn=steward-admins,ou=groups,dc=example,dc=com">
+ placeholder="cn=steward-operators,ou=groups,dc=example,dc=com">
diff --git a/steward/templates/settings/plugins.html b/steward/templates/settings/plugins.html index 31d1a7c..2b2a5c6 100644 --- a/steward/templates/settings/plugins.html +++ b/steward/templates/settings/plugins.html @@ -81,7 +81,7 @@
Plugin Repositories

- Repositories are remote index.yaml files that Scryer checks for available + Repositories are remote index.yaml files that Steward checks for available and updated plugins. Add repos from other developers to expand the plugin catalog.

{% include "settings/_plugin_repos.html" %} diff --git a/tests/conftest.py b/tests/conftest.py index 1031522..8b4edc0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ import pytest import pytest_asyncio -from pathlib import Path import textwrap from steward.app import create_app diff --git a/tests/plugins/host_agent/test_agent_build_payload.py b/tests/plugins/host_agent/test_agent_build_payload.py index f38f567..2d7f26a 100644 --- a/tests/plugins/host_agent/test_agent_build_payload.py +++ b/tests/plugins/host_agent/test_agent_build_payload.py @@ -1,3 +1,4 @@ +import urllib.error from unittest.mock import patch from plugins.host_agent import agent as a @@ -55,10 +56,6 @@ def test_build_payload_wraps_samples(): assert payload["samples"][0]["cpu_pct"] == 5.0 -import urllib.error -from unittest.mock import MagicMock, patch - - def test_post_payload_success(): from plugins.host_agent import agent as a captured = {} diff --git a/tests/plugins/host_agent/test_agent_ring_buffer.py b/tests/plugins/host_agent/test_agent_ring_buffer.py index f0d7a46..2482e6d 100644 --- a/tests/plugins/host_agent/test_agent_ring_buffer.py +++ b/tests/plugins/host_agent/test_agent_ring_buffer.py @@ -3,7 +3,9 @@ from plugins.host_agent.agent import RingBuffer def test_ring_buffer_preserves_order(): rb = RingBuffer(maxlen=3) - rb.push(1); rb.push(2); rb.push(3) + rb.push(1) + rb.push(2) + rb.push(3) assert list(rb.drain()) == [1, 2, 3] assert len(rb) == 0 @@ -17,7 +19,8 @@ def test_ring_buffer_drops_oldest_when_full(): def test_ring_buffer_drain_clears_and_is_atomic(): rb = RingBuffer(maxlen=5) - rb.push("a"); rb.push("b") + rb.push("a") + rb.push("b") out = list(rb.drain()) rb.push("c") assert out == ["a", "b"] diff --git a/tests/plugins/host_agent/test_install_template.py b/tests/plugins/host_agent/test_install_template.py index dc3f928..efaed79 100644 --- a/tests/plugins/host_agent/test_install_template.py +++ b/tests/plugins/host_agent/test_install_template.py @@ -2,7 +2,6 @@ import subprocess from pathlib import Path -import pytest from jinja2 import Environment, FileSystemLoader from plugins.host_agent.routes import _agent_version, AGENT_SOURCE_PATH diff --git a/tests/test_config.py b/tests/test_config.py index d4151bd..40e9203 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,3 @@ -import os import textwrap import pytest from steward.config import load_bootstrap diff --git a/tests/test_models.py b/tests/test_models.py index 2bce1c7..50a074b 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,4 +1,3 @@ -import pytest from steward.models.users import User, UserRole diff --git a/tests/test_no_legacy_names.py b/tests/test_no_legacy_names.py new file mode 100644 index 0000000..0e75198 --- /dev/null +++ b/tests/test_no_legacy_names.py @@ -0,0 +1,35 @@ +"""Regression guard: shipped code must not reference old project names. + +The FabledScryer -> Roundtable -> Steward renames repeatedly left stale package +imports behind in the (then separate) plugins (e.g. `from fabledscryer.models...`), +silently breaking every non-host_agent plugin. Now that plugins are folded in-tree +and version with core, this test fails loudly if either legacy name reappears in +the shipped Python / templates / plugin manifests. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).resolve().parent.parent +_TREES = [_ROOT / "steward", _ROOT / "plugins"] +_EXTENSIONS = {".py", ".yaml", ".yml", ".html", ".j2"} +_FORBIDDEN = ("scryer", "roundtable") # case-insensitive; old project names + + +def _shipped_files(): + for tree in _TREES: + for path in tree.rglob("*"): + if path.suffix in _EXTENSIONS and "__pycache__" not in path.parts: + yield path + + +@pytest.mark.parametrize("name", _FORBIDDEN) +def test_no_legacy_project_name(name): + offenders = [ + str(p.relative_to(_ROOT)) + for p in _shipped_files() + if name in p.read_text(encoding="utf-8", errors="ignore").lower() + ] + assert not offenders, f"legacy name {name!r} found in shipped code: {offenders}" From 3b2146bc7a1903cd9f56db0515d3761e90e66d36 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 1 Jun 2026 12:11:41 -0400 Subject: [PATCH 004/126] feat(host_agent): agent reports its primary IP; mirror into Host.address (task 274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent now detects its own primary non-loopback IP (UDP-socket trick, no packets sent) and sends it in the metadata bag; bumps AGENT_VERSION 1.0.0->1.1.0 and re-detects on SIGHUP. The server persists it on HostAgentRegistration.host_ip and mirrors it into Host.address ONLY when that field is blank — an admin-typed address is never overwritten. The reported IP shows in the settings table so admins can see drift regardless. - agent.py: detect_primary_ip(); host_ip in collect_metadata(); refresh on reload - routes.py: pure pick_host_address() helper; ingest persists host_ip + mirrors - models.py + migration host_agent_002_host_ip: new String(45) column - settings_list.html: Reported IP column - plugin.yaml: 1.0.0 -> 1.1.0 - tests: pick_host_address (fill-when-blank / never-clobber / no-op), detect_primary_ip (never raises, non-loopback), metadata carries host_ip Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/agent.py | 33 ++++++++++++++- .../versions/host_agent_002_host_ip.py | 24 +++++++++++ plugins/host_agent/models.py | 2 + plugins/host_agent/plugin.yaml | 2 +- plugins/host_agent/routes.py | 21 ++++++++++ .../host_agent/templates/settings_list.html | 5 ++- tests/plugins/host_agent/test_host_ip.py | 41 +++++++++++++++++++ 7 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 plugins/host_agent/migrations/versions/host_agent_002_host_ip.py create mode 100644 tests/plugins/host_agent/test_host_ip.py diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index 34f52d6..2051c60 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -18,7 +18,7 @@ import urllib.request from collections import deque from datetime import datetime, timezone -AGENT_VERSION = "1.0.0" +AGENT_VERSION = "1.1.0" class ConfigError(Exception): @@ -148,6 +148,29 @@ def collect_uptime() -> int: return int(float(_read_file(UPTIME_PATH).split()[0])) +def detect_primary_ip() -> str | None: + """Best-effort primary non-loopback IPv4 of this host. + + connect() on a SOCK_DGRAM socket only sets the default peer and selects the + outbound interface — no packets are sent — so getsockname() reveals the + source IP the kernel would use to reach the internet. This is the host's own + view of its address, so it survives reverse proxies / NAT (unlike the + server reading request.remote_addr). Returns None on any error (offline / no + route) and skips loopback. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect(("8.8.8.8", 80)) + ip = s.getsockname()[0] + except OSError: + return None + finally: + s.close() + if not ip or ip.startswith("127."): + return None + return ip + + def collect_metadata() -> dict: u = os.uname() distro = "unknown" @@ -158,7 +181,12 @@ def collect_metadata() -> dict: break except (OSError, FileNotFoundError): pass - return {"kernel": u.release, "distro": distro, "arch": u.machine} + return { + "kernel": u.release, + "distro": distro, + "arch": u.machine, + "host_ip": detect_primary_ip(), + } def default_mounts() -> list[str]: @@ -319,6 +347,7 @@ def main_loop(conf_path: str) -> int: try: cfg = read_config(conf_path) mounts = cfg.get("mounts") or default_mounts() + metadata = collect_metadata() # refresh host_ip/distro on reload _log("INFO", "config reloaded") except ConfigError as e: _log("ERROR", f"reload failed: {e}") diff --git a/plugins/host_agent/migrations/versions/host_agent_002_host_ip.py b/plugins/host_agent/migrations/versions/host_agent_002_host_ip.py new file mode 100644 index 0000000..197dd2f --- /dev/null +++ b/plugins/host_agent/migrations/versions/host_agent_002_host_ip.py @@ -0,0 +1,24 @@ +# plugins/host_agent/migrations/versions/host_agent_002_host_ip.py +"""host_agent: add agent-reported host_ip column + +Revision ID: host_agent_002_host_ip +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "host_agent_002_host_ip" +down_revision: Union[str, None] = "host_agent_001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "host_agent_registrations", + sa.Column("host_ip", sa.String(45), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("host_agent_registrations", "host_ip") diff --git a/plugins/host_agent/models.py b/plugins/host_agent/models.py index b93d725..e8d87d0 100644 --- a/plugins/host_agent/models.py +++ b/plugins/host_agent/models.py @@ -26,6 +26,8 @@ class HostAgentRegistration(Base): kernel = Column(String(128), nullable=True) distro = Column(String(128), nullable=True) arch = Column(String(32), nullable=True) + # Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix). + host_ip = Column(String(45), nullable=True) last_seen_at = Column(DateTime(timezone=True), nullable=True) created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) updated_at = Column(DateTime(timezone=True), nullable=False, diff --git a/plugins/host_agent/plugin.yaml b/plugins/host_agent/plugin.yaml index 09b9022..d6716e8 100644 --- a/plugins/host_agent/plugin.yaml +++ b/plugins/host_agent/plugin.yaml @@ -1,6 +1,6 @@ # plugins/host_agent/plugin.yaml name: host_agent -version: "1.0.0" +version: "1.1.0" description: "Remote Linux host resource monitoring via a lightweight Python push agent (CPU, memory, storage, load, uptime)" author: "Steward" license: "MIT" diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index f023811..584db94 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -34,6 +34,19 @@ def _parse_ts(ts: str) -> datetime: return datetime.fromisoformat(ts) +def pick_host_address(current: str | None, reported: str | None) -> str | None: + """Return the address to store on the Host, or None for no change. + + The agent-reported IP fills Host.address only when the current value is + blank — an admin-typed address (DNS name, management IP) is never + overwritten. The reported IP is always kept on the registration regardless, + so admins can still see drift. + """ + if reported and not (current or "").strip(): + return reported + return None + + def _expand_sample_to_metrics( sample: dict, host_name: str, recorded_at: datetime ) -> list[PluginMetric]: @@ -157,6 +170,14 @@ async def ingest(): if field in md and getattr(reg, field) != md[field]: setattr(reg, field, md[field]) changed = True + host_ip = md.get("host_ip") + if host_ip and reg.host_ip != host_ip: + reg.host_ip = host_ip + changed = True + # Mirror the reported IP into Host.address only when it's blank. + new_address = pick_host_address(host.address, host_ip) + if new_address is not None: + host.address = new_address if changed: reg.updated_at = datetime.now(timezone.utc) diff --git a/plugins/host_agent/templates/settings_list.html b/plugins/host_agent/templates/settings_list.html index 7e4ca96..49684f7 100644 --- a/plugins/host_agent/templates/settings_list.html +++ b/plugins/host_agent/templates/settings_list.html @@ -32,7 +32,7 @@ - + @@ -40,6 +40,7 @@ {% for item in registrations %} + @@ -57,7 +58,7 @@ {% else %} - + {% endfor %}
HostAgent versionDistroHostReported IPAgent versionDistro Last seenActions
{{ item.host.name if item.host else item.reg.host_id }}{{ item.reg.host_ip or "—" }} {{ item.reg.agent_version or "—" }} {{ item.reg.distro or "—" }} {{ item.reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") if item.reg.last_seen_at else "never" }}
No hosts registered. Add one above.
No hosts registered. Add one above.
diff --git a/tests/plugins/host_agent/test_host_ip.py b/tests/plugins/host_agent/test_host_ip.py new file mode 100644 index 0000000..ee07c9a --- /dev/null +++ b/tests/plugins/host_agent/test_host_ip.py @@ -0,0 +1,41 @@ +"""Tests for agent-reported primary IP (task 274).""" +from plugins.host_agent.routes import pick_host_address +from plugins.host_agent import agent as a + + +# ── pick_host_address: fill only when blank, never clobber ──────────────────── + +def test_pick_fills_when_current_blank(): + assert pick_host_address("", "10.0.0.5") == "10.0.0.5" + assert pick_host_address(None, "10.0.0.5") == "10.0.0.5" + assert pick_host_address(" ", "10.0.0.5") == "10.0.0.5" + + +def test_pick_keeps_admin_value(): + # A non-blank current address is never overwritten. + assert pick_host_address("monitor.example.com", "10.0.0.5") is None + assert pick_host_address("192.168.1.9", "10.0.0.5") is None + + +def test_pick_no_reported_ip_is_noop(): + assert pick_host_address("", None) is None + assert pick_host_address("", "") is None + assert pick_host_address(None, None) is None + + +# ── detect_primary_ip: never raises; None or a non-loopback string ──────────── + +def test_detect_primary_ip_never_raises_and_is_sane(): + ip = a.detect_primary_ip() + assert ip is None or isinstance(ip, str) + if ip is not None: + assert not ip.startswith("127.") + assert ip # non-empty + + +# ── host_ip rides in the metadata bag the agent sends ───────────────────────── + +def test_collect_metadata_includes_host_ip_key(): + md = a.collect_metadata() + assert "host_ip" in md # present even if None when offline + assert set(md) >= {"kernel", "distro", "arch", "host_ip"} From 712aec60c157ace4e7e777c410ffd794e6055877 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 1 Jun 2026 12:14:53 -0400 Subject: [PATCH 005/126] test(host_agent): assert parsed agent version matches AGENT_VERSION constant The on-disk-version test hardcoded '1.0.0' and broke on the 1.1.0 bump. Compare against agent.AGENT_VERSION so future bumps don't require touching the test. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/plugins/host_agent/test_install_template.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/plugins/host_agent/test_install_template.py b/tests/plugins/host_agent/test_install_template.py index efaed79..835242f 100644 --- a/tests/plugins/host_agent/test_install_template.py +++ b/tests/plugins/host_agent/test_install_template.py @@ -69,6 +69,9 @@ def test_rendered_script_passes_sh_n(tmp_path): def test_agent_version_parses_on_disk_agent(): + # Version-agnostic: the parsed value must match the agent module's constant, + # so a future AGENT_VERSION bump doesn't require touching this test. + from plugins.host_agent import agent assert AGENT_SOURCE_PATH.exists() v = _agent_version() - assert v == "1.0.0" + assert v == agent.AGENT_VERSION From ccc7601182f759503c26483a2e5b5814a494ecc6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 2 Jun 2026 10:35:58 -0400 Subject: [PATCH 006/126] fix(docker): stop ignoring plugins/ in image build; dev-friendly compose .dockerignore excluded plugins/ (leftover from the separate-repo era), so the Dockerfile's COPY plugins/ found nothing and the build failed. Track it now that first-party plugins ship in the image. docker-compose.yml reworked for local dev: live-mounted ./steward + ./plugins with PYTHONPATH=/app, --debug auto-reload, fixed dev SECRET_KEY, Postgres port exposed, and the host-specific traefik/docker.sock mounts commented out. Co-Authored-By: Claude Opus 4.8 (1M context) --- .dockerignore | 5 +++-- docker-compose.yml | 22 +++++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.dockerignore b/.dockerignore index c1dc287..28ed355 100644 --- a/.dockerignore +++ b/.dockerignore @@ -26,8 +26,9 @@ config.yaml # Runtime data (mounted at runtime via volume) playbook_cache/ -# Plugins (mounted at runtime via volume) -plugins/ +# NOTE: first-party plugins under plugins/ now ship IN the image +# (Dockerfile `COPY plugins/`), so plugins/ must NOT be ignored here. +# Third-party plugins are mounted at runtime into /data/plugins instead. # Deployment files not needed in image docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml index 3e1ce35..5d57fd8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,17 +3,26 @@ services: build: . container_name: steward image: steward:latest + # Dev: --debug turns on Quart's auto-reloader so edits to the mounted source + # below take effect without a rebuild. + command: ["steward", "--host", "0.0.0.0", "--port", "5000", "--debug"] ports: - "5000:5000" volumes: - # First-party plugins are baked into the image at /app/plugins. - # Third-party plugins persist in the external root /data/plugins - # (inside app_data); STEWARD_PLUGIN_DIR defaults there, no extra mount needed. - app_data:/data - - /mnt/Data/traefik/log:/var/log/traefik:ro - - /var/run/docker.sock:/var/run/docker.sock:ro + # Live-edit core + first-party plugins without rebuilding. PYTHONPATH=/app + # (below) makes this mounted source win over the image-installed package. + - ./steward:/app/steward + - ./plugins:/app/plugins + # Optional host mounts — enable the matching plugin in Settings first, then + # uncomment (the traefik path must exist on this host): + # - /mnt/Data/traefik/log:/var/log/traefik:ro # traefik plugin + # - /var/run/docker.sock:/var/run/docker.sock:ro # docker plugin environment: - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@db/steward + # Dev-only fixed key so sessions survive restarts. Replace in production. + - STEWARD_SECRET_KEY=dev-secret-key-change-me + - PYTHONPATH=/app depends_on: db: condition: service_healthy @@ -25,6 +34,9 @@ services: POSTGRES_USER: steward POSTGRES_PASSWORD: steward POSTGRES_DB: steward + ports: + # Exposed for dev convenience (psql from the host). Drop for prod. + - "5432:5432" volumes: - pgdata:/var/lib/postgresql/data healthcheck: From fb579bcf974a6010dcb0aea75947b2f4a7a38d17 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 2 Jun 2026 11:31:43 -0400 Subject: [PATCH 007/126] feat(ansible): ship ansible in image + allow system-triggered runs (task 545) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundation for the Ansible automation milestone (#37) — makes the existing manual playbook runner actually executable and the schema automation-ready. - pyproject: [ansible] extra (full ansible package, batteries-included, pinned) - Dockerfile: pip install .[ansible]; add openssh-client for remote runs - models/ansible.py + migration 0012: AnsibleRun.triggered_by now nullable so automated (alert/schedule) runs need no human actor - ansible/routes.py + run_detail.html: show 'Triggered by' (username or 'system') - CI: integration lane installs .[dev,ansible]; new tests/integration/ test_ansible_foundation.py runs a real connection:local playbook end-to-end, asserts success+output, and round-trips a NULL-triggered run No extra-vars/limit/credentials/scheduling here — those are their own #37 tasks. Co-Authored-By: Claude Opus 4.8 (1M context) --- .forgejo/workflows/ci.yml | 4 +- Dockerfile | 5 +- pyproject.toml | 5 + steward/ansible/routes.py | 15 ++- .../0012_ansible_run_triggered_by_nullable.py | 31 ++++++ steward/models/ansible.py | 6 +- steward/templates/ansible/run_detail.html | 1 + tests/integration/test_ansible_foundation.py | 101 ++++++++++++++++++ 8 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 steward/migrations/versions/0012_ansible_run_triggered_by_nullable.py create mode 100644 tests/integration/test_ansible_foundation.py diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 251d2bd..ab6dea8 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -85,8 +85,8 @@ jobs: sleep 2 done if command -v uv >/dev/null 2>&1; then - uv pip install --system -e '.[dev]' + uv pip install --system -e '.[dev,ansible]' else - pip install -e '.[dev]' + pip install -e '.[dev,ansible]' fi pytest tests/integration -v -m integration --durations=10 diff --git a/Dockerfile b/Dockerfile index 4172e1d..c74d42d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \ iputils-ping \ # gosu — minimal privilege-drop tool used by entrypoint.sh gosu \ + # openssh-client — ansible-playbook reaches remote hosts over ssh + openssh-client \ && rm -rf /var/lib/apt/lists/* # Upgrade pip to suppress the version notice @@ -14,7 +16,8 @@ WORKDIR /app COPY pyproject.toml . COPY steward/ steward/ -RUN pip install --no-cache-dir . +# .[ansible] pulls the full Ansible package so the playbook runner works in-image. +RUN pip install --no-cache-dir '.[ansible]' COPY alembic.ini . # First-party plugins ship inside the image (bundled root at /app/plugins). diff --git a/pyproject.toml b/pyproject.toml index fe4179c..1a4aecc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,11 @@ ldap = [ snmp = [ "pysnmp-lextudio>=6.2", ] +# Batteries-included Ansible (bundles the common community collections) for the +# playbook runner. Installed into the image via Dockerfile `pip install .[ansible]`. +ansible = [ + "ansible>=10,<13", +] dev = [ "pytest>=8.0", "pytest-asyncio>=0.23", diff --git a/steward/ansible/routes.py b/steward/ansible/routes.py index 1dceb26..11022dc 100644 --- a/steward/ansible/routes.py +++ b/steward/ansible/routes.py @@ -13,7 +13,7 @@ from sqlalchemy import select from steward.ansible import executor, sources as src_module from steward.auth.middleware import require_role from steward.models.ansible import AnsibleRun, AnsibleRunStatus -from steward.models.users import UserRole +from steward.models.users import User, UserRole ansible_bp = Blueprint("ansible", __name__, url_prefix="/ansible") @@ -163,6 +163,13 @@ async def run_detail(run_id: str): async with current_app.db_sessionmaker() as db: result = await db.execute(select(AnsibleRun).where(AnsibleRun.id == run_id)) run = result.scalar_one_or_none() - if run is None: - return "Not found", 404 - return await render_template("ansible/run_detail.html", run=run) + if run is None: + return "Not found", 404 + # NULL triggered_by = an automated (system) run; otherwise resolve the user. + triggered_label = "system" + if run.triggered_by: + res = await db.execute( + select(User.username).where(User.id == run.triggered_by)) + triggered_label = res.scalar_one_or_none() or "(deleted user)" + return await render_template( + "ansible/run_detail.html", run=run, triggered_label=triggered_label) diff --git a/steward/migrations/versions/0012_ansible_run_triggered_by_nullable.py b/steward/migrations/versions/0012_ansible_run_triggered_by_nullable.py new file mode 100644 index 0000000..26ddc9d --- /dev/null +++ b/steward/migrations/versions/0012_ansible_run_triggered_by_nullable.py @@ -0,0 +1,31 @@ +"""Make ansible_runs.triggered_by nullable (automated runs have no user) + +Revision ID: 0012_ansible_run_triggered_by_nullable +Revises: 0011_audit_log +Create Date: 2026-06-02 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "0012_ansible_run_triggered_by_nullable" +down_revision: Union[str, None] = "0011_audit_log" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column( + "ansible_runs", "triggered_by", + existing_type=sa.String(36), + nullable=True, + ) + + +def downgrade() -> None: + # Re-imposes NOT NULL; will fail if any system-triggered (NULL) rows exist. + op.alter_column( + "ansible_runs", "triggered_by", + existing_type=sa.String(36), + nullable=False, + ) diff --git a/steward/models/ansible.py b/steward/models/ansible.py index 8524fca..82d18bc 100644 --- a/steward/models/ansible.py +++ b/steward/models/ansible.py @@ -21,8 +21,10 @@ class AnsibleRun(Base): playbook_path: Mapped[str] = mapped_column(String(512), nullable=False) inventory_path: Mapped[str] = mapped_column(String(512), nullable=False) source_name: Mapped[str] = mapped_column(String(128), nullable=False) - triggered_by: Mapped[str] = mapped_column( - String(36), ForeignKey("users.id", ondelete="RESTRICT"), nullable=False + # Nullable: automated runs (alert actions, schedules) have no human actor — + # NULL renders as "system". Manual runs still record the triggering user. + triggered_by: Mapped[str | None] = mapped_column( + String(36), ForeignKey("users.id", ondelete="RESTRICT"), nullable=True ) status: Mapped[AnsibleRunStatus] = mapped_column( Enum(AnsibleRunStatus), nullable=False, default=AnsibleRunStatus.running diff --git a/steward/templates/ansible/run_detail.html b/steward/templates/ansible/run_detail.html index 236ad81..d6fdf9f 100644 --- a/steward/templates/ansible/run_detail.html +++ b/steward/templates/ansible/run_detail.html @@ -13,6 +13,7 @@ Playbook{{ run.playbook_path }} Inventory{{ run.inventory_path }} Source{{ run.source_name }} + Triggered by{{ triggered_label }} Status{{ run.status.value.upper() }} Started{{ run.started_at.strftime("%Y-%m-%d %H:%M:%S UTC") }} {% if run.finished_at %} diff --git a/tests/integration/test_ansible_foundation.py b/tests/integration/test_ansible_foundation.py new file mode 100644 index 0000000..341755d --- /dev/null +++ b/tests/integration/test_ansible_foundation.py @@ -0,0 +1,101 @@ +"""Integration: the Ansible foundation (task 545). + +Proves the two foundation pieces against a live Postgres + a real ansible install: + 1. ansible-playbook is on PATH (ships in the image / [ansible] extra). + 2. executor.start_run actually runs a trivial connection:local playbook end to + end and records success + output on the AnsibleRun row. + 3. An AnsibleRun with triggered_by=NULL inserts and round-trips (migration 0012). + +Requires STEWARD_DATABASE_URL (the integration CI lane provides it). A fresh app +is built per test so each test's async work runs on its own event loop with its +own engine (create_app's internal asyncio.run can't be nested under a running +loop, so these tests are sync and drive async work via asyncio.run). +""" +from __future__ import annotations + +import asyncio +import os +import shutil +import textwrap +import uuid + +import pytest + +pytestmark = pytest.mark.integration + +_NEEDS_DB = pytest.mark.skipif( + not os.environ.get("STEWARD_DATABASE_URL"), + reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)", +) + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + # testing=False → runs all migrations (incl. 0012) and gives a real engine. + return create_app(testing=False) + + +def test_ansible_playbook_on_path(): + assert shutil.which("ansible-playbook") is not None + + +@_NEEDS_DB +def test_null_triggered_run_inserts(app): + from sqlalchemy import select + from steward.models.ansible import AnsibleRun, AnsibleRunStatus + + run_id = str(uuid.uuid4()) + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(AnsibleRun( + id=run_id, playbook_path="p.yml", inventory_path="i.ini", + source_name="t", triggered_by=None, + status=AnsibleRunStatus.success, + )) + async with app.db_sessionmaker() as s: + row = (await s.execute( + select(AnsibleRun).where(AnsibleRun.id == run_id))).scalar_one() + return row.triggered_by + + assert asyncio.run(_go()) is None + + +@_NEEDS_DB +def test_executor_runs_local_playbook(app, tmp_path): + from sqlalchemy import select + from steward.ansible import executor + from steward.models.ansible import AnsibleRun, AnsibleRunStatus + + (tmp_path / "test.yml").write_text(textwrap.dedent("""\ + - hosts: localhost + connection: local + gather_facts: false + tasks: + - debug: + msg: STEWARD_ANSIBLE_OK + """)) + (tmp_path / "inv.ini").write_text("localhost ansible_connection=local\n") + + run_id = str(uuid.uuid4()) + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(AnsibleRun( + id=run_id, playbook_path="test.yml", inventory_path="inv.ini", + source_name="t", triggered_by=None, + status=AnsibleRunStatus.running, + )) + await executor.start_run(app, run_id, "test.yml", "inv.ini", str(tmp_path)) + async with app.db_sessionmaker() as s: + return (await s.execute( + select(AnsibleRun).where(AnsibleRun.id == run_id))).scalar_one() + + run = asyncio.run(_go()) + assert run.status == AnsibleRunStatus.success, run.output + assert "STEWARD_ANSIBLE_OK" in (run.output or "") From 5af7312a721adc315876f9164bf663ad1a737131 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 2 Jun 2026 11:35:55 -0400 Subject: [PATCH 008/126] fix(ansible): shorten migration 0012 revision id to fit alembic_version(32) The id 0012_ansible_run_triggered_by_nullable was 38 chars; alembic_version. version_num is VARCHAR(32), so stamping it raised StringDataRightTruncationError and the boot/migrate integration lane failed. Renamed to 0012_ansible_run_nullable. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...riggered_by_nullable.py => 0012_ansible_run_nullable.py} | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) rename steward/migrations/versions/{0012_ansible_run_triggered_by_nullable.py => 0012_ansible_run_nullable.py} (82%) diff --git a/steward/migrations/versions/0012_ansible_run_triggered_by_nullable.py b/steward/migrations/versions/0012_ansible_run_nullable.py similarity index 82% rename from steward/migrations/versions/0012_ansible_run_triggered_by_nullable.py rename to steward/migrations/versions/0012_ansible_run_nullable.py index 26ddc9d..3c700e1 100644 --- a/steward/migrations/versions/0012_ansible_run_triggered_by_nullable.py +++ b/steward/migrations/versions/0012_ansible_run_nullable.py @@ -1,14 +1,16 @@ """Make ansible_runs.triggered_by nullable (automated runs have no user) -Revision ID: 0012_ansible_run_triggered_by_nullable +Revision ID: 0012_ansible_run_nullable Revises: 0011_audit_log Create Date: 2026-06-02 + +Note: the revision id is kept <=32 chars to fit alembic_version.version_num. """ from typing import Sequence, Union from alembic import op import sqlalchemy as sa -revision: str = "0012_ansible_run_triggered_by_nullable" +revision: str = "0012_ansible_run_nullable" down_revision: Union[str, None] = "0011_audit_log" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None From 8b62eb2ca3e040f93f771cefe1d5f2320edf9eaf Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 2 Jun 2026 13:31:16 -0400 Subject: [PATCH 009/126] =?UTF-8?q?feat(ansible):=20parameterized=20runs?= =?UTF-8?q?=20=E2=80=94=20extra-vars,=20limit,=20tags,=20dry-run=20(task?= =?UTF-8?q?=20546)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run form + executor now support optional params, passed as argv (never shell-interpolated): - extra_vars: one key=value per line -> repeated -e - limit -> --limit; tags -> --tags - dry-run checkbox -> --check --diff - executor.py: pure build_ansible_command(playbook, inventory, params); start_run gains an optional params arg (backward-compatible) - models/ansible.py + migration 0013: nullable params JSON column - routes.py: create_run parses + validates (extra-var lines need '='), stores params on the run, passes to the executor - browse.html run form: Extra vars / Limit / Tags / Dry-run fields - run_detail.html: shows the params used - tests/test_ansible_command.py: unit coverage of the builder; integration test runs a real playbook with extra-var + limit + check and asserts substitution Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/ansible/executor.py | 27 +++++++++++- steward/ansible/routes.py | 26 ++++++++++++ .../versions/0013_ansible_run_params.py | 22 ++++++++++ steward/models/ansible.py | 4 +- steward/templates/ansible/browse.html | 19 +++++++++ steward/templates/ansible/run_detail.html | 14 +++++++ tests/integration/test_ansible_foundation.py | 39 +++++++++++++++++ tests/test_ansible_command.py | 42 +++++++++++++++++++ 8 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 steward/migrations/versions/0013_ansible_run_params.py create mode 100644 tests/test_ansible_command.py diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py index 22e62c9..92ee4ab 100644 --- a/steward/ansible/executor.py +++ b/steward/ansible/executor.py @@ -59,12 +59,37 @@ def _broadcast(run_id: str, item: str | _Done) -> None: _run_listeners.pop(run_id, None) +def build_ansible_command( + playbook_path: str, + inventory_path: str, + params: dict | None = None, +) -> list[str]: + """Build the ansible-playbook argv from a run's optional params. + + Pure (no IO). Everything is passed as argv — never shell-interpolated — so + extra-var values and limits can contain arbitrary characters safely. + params keys: extra_vars (list of "key=value"), limit, tags, check (bool). + """ + cmd = ["ansible-playbook", playbook_path, "-i", inventory_path] + params = params or {} + for ev in params.get("extra_vars") or []: + cmd += ["-e", ev] + if params.get("limit"): + cmd += ["--limit", params["limit"]] + if params.get("tags"): + cmd += ["--tags", params["tags"]] + if params.get("check"): + cmd += ["--check", "--diff"] + return cmd + + async def start_run( app: "Quart", run_id: str, playbook_path: str, inventory_path: str, source_path: str, + params: dict | None = None, ) -> None: """Execute ansible-playbook as a subprocess and update the DB run row.""" _run_lines[run_id] = [] @@ -92,7 +117,7 @@ async def start_run( lines_since_flush = 0 last_flush_at = time.monotonic() - cmd = ["ansible-playbook", playbook_path, "-i", inventory_path] + cmd = build_ansible_command(playbook_path, inventory_path, params) cwd = source_path if Path(source_path).exists() else None try: diff --git a/steward/ansible/routes.py b/steward/ansible/routes.py index 11022dc..b304bd5 100644 --- a/steward/ansible/routes.py +++ b/steward/ansible/routes.py @@ -85,6 +85,30 @@ async def create_run(): if source is None: return "Source not found", 404 + # Optional run parameters (all passed as argv by the executor — no shell). + extra_vars: list[str] = [] + for line in (form.get("extra_vars", "") or "").splitlines(): + line = line.strip() + if not line: + continue + if "=" not in line: + return f"Invalid extra var (expected key=value): {line!r}", 400 + extra_vars.append(line) + limit = (form.get("limit", "") or "").strip() + tags = (form.get("tags", "") or "").strip() + check = "check" in form + + params: dict = {} + if extra_vars: + params["extra_vars"] = extra_vars + if limit: + params["limit"] = limit + if tags: + params["tags"] = tags + if check: + params["check"] = True + params_or_none = params or None + run_id = str(uuid.uuid4()) now = datetime.now(timezone.utc) @@ -96,6 +120,7 @@ async def create_run(): triggered_by=session["user_id"], status=AnsibleRunStatus.running, started_at=now, + params=params_or_none, ) async with current_app.db_sessionmaker() as db: async with db.begin(): @@ -108,6 +133,7 @@ async def create_run(): playbook_path, inventory_path, source["path"], + params_or_none, ) ) task.add_done_callback( diff --git a/steward/migrations/versions/0013_ansible_run_params.py b/steward/migrations/versions/0013_ansible_run_params.py new file mode 100644 index 0000000..f70811b --- /dev/null +++ b/steward/migrations/versions/0013_ansible_run_params.py @@ -0,0 +1,22 @@ +"""Add ansible_runs.params (optional run parameters) + +Revision ID: 0013_ansible_run_params +Revises: 0012_ansible_run_nullable +Create Date: 2026-06-02 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "0013_ansible_run_params" +down_revision: Union[str, None] = "0012_ansible_run_nullable" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("ansible_runs", sa.Column("params", sa.JSON(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("ansible_runs", "params") diff --git a/steward/models/ansible.py b/steward/models/ansible.py index 82d18bc..2adf06e 100644 --- a/steward/models/ansible.py +++ b/steward/models/ansible.py @@ -2,7 +2,7 @@ from __future__ import annotations import enum import uuid from datetime import datetime, timezone -from sqlalchemy import DateTime, Enum, ForeignKey, String, Text +from sqlalchemy import JSON, DateTime, Enum, ForeignKey, String, Text from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -34,3 +34,5 @@ class AnsibleRun(Base): ) finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) output: Mapped[str | None] = mapped_column(Text, nullable=True) + # Optional run parameters: {extra_vars: [..], limit: str, tags: str, check: bool}. + params: Mapped[dict | None] = mapped_column(JSON, nullable=True) diff --git a/steward/templates/ansible/browse.html b/steward/templates/ansible/browse.html index 93dae22..e0b95b8 100644 --- a/steward/templates/ansible/browse.html +++ b/steward/templates/ansible/browse.html @@ -83,6 +83,25 @@
+
+ + +
+
+ + +
+
+ + +
+
+ +
diff --git a/tests/integration/test_ansible_foundation.py b/tests/integration/test_ansible_foundation.py index 341755d..d15c54e 100644 --- a/tests/integration/test_ansible_foundation.py +++ b/tests/integration/test_ansible_foundation.py @@ -99,3 +99,42 @@ def test_executor_runs_local_playbook(app, tmp_path): run = asyncio.run(_go()) assert run.status == AnsibleRunStatus.success, run.output assert "STEWARD_ANSIBLE_OK" in (run.output or "") + + +@_NEEDS_DB +def test_executor_runs_parameterized(app, tmp_path): + """extra_vars + limit + check all flow through to a real run (task 546).""" + from sqlalchemy import select + from steward.ansible import executor + from steward.models.ansible import AnsibleRun, AnsibleRunStatus + + (tmp_path / "test.yml").write_text(textwrap.dedent("""\ + - hosts: localhost + connection: local + gather_facts: false + tasks: + - debug: + msg: "GOT-{{ myvar }}" + """)) + (tmp_path / "inv.ini").write_text("localhost ansible_connection=local\n") + + run_id = str(uuid.uuid4()) + params = {"extra_vars": ["myvar=HELLO"], "limit": "localhost", "check": True} + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(AnsibleRun( + id=run_id, playbook_path="test.yml", inventory_path="inv.ini", + source_name="t", triggered_by=None, + status=AnsibleRunStatus.running, params=params, + )) + await executor.start_run(app, run_id, "test.yml", "inv.ini", str(tmp_path), params) + async with app.db_sessionmaker() as s: + return (await s.execute( + select(AnsibleRun).where(AnsibleRun.id == run_id))).scalar_one() + + run = asyncio.run(_go()) + assert run.status == AnsibleRunStatus.success, run.output + assert "GOT-HELLO" in (run.output or "") # extra var substituted + assert run.params == params # params round-trip through the DB diff --git a/tests/test_ansible_command.py b/tests/test_ansible_command.py new file mode 100644 index 0000000..2c2e240 --- /dev/null +++ b/tests/test_ansible_command.py @@ -0,0 +1,42 @@ +"""Unit tests for the ansible-playbook command builder (task 546).""" +from steward.ansible.executor import build_ansible_command + +BASE = ["ansible-playbook", "site.yml", "-i", "inv.ini"] + + +def test_base_no_params(): + assert build_ansible_command("site.yml", "inv.ini") == BASE + assert build_ansible_command("site.yml", "inv.ini", None) == BASE + assert build_ansible_command("site.yml", "inv.ini", {}) == BASE + + +def test_extra_vars_each_passed_as_argv(): + cmd = build_ansible_command("site.yml", "inv.ini", {"extra_vars": ["a=1", "b=two words"]}) + assert cmd == BASE + ["-e", "a=1", "-e", "b=two words"] + + +def test_limit_and_tags(): + cmd = build_ansible_command("site.yml", "inv.ini", {"limit": "web*", "tags": "deploy,cfg"}) + assert cmd[cmd.index("--limit") + 1] == "web*" + assert cmd[cmd.index("--tags") + 1] == "deploy,cfg" + + +def test_check_adds_check_and_diff(): + cmd = build_ansible_command("site.yml", "inv.ini", {"check": True}) + assert cmd == BASE + ["--check", "--diff"] + + +def test_falsey_values_are_skipped(): + cmd = build_ansible_command( + "site.yml", "inv.ini", + {"extra_vars": [], "limit": "", "tags": "", "check": False}, + ) + assert cmd == BASE + + +def test_all_combined_order(): + cmd = build_ansible_command( + "site.yml", "inv.ini", + {"extra_vars": ["v=1"], "limit": "h1", "tags": "t1", "check": True}, + ) + assert cmd == BASE + ["-e", "v=1", "--limit", "h1", "--tags", "t1", "--check", "--diff"] From 4771d17f6d2a3e90d88d7698dd0387007c113a1e Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 2 Jun 2026 14:22:29 -0400 Subject: [PATCH 010/126] feat(alerts): run an Ansible playbook on alert firing (task 250) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebuilds the deleted NUT/UPS automation as a general alert action: any metric can drive a playbook run on transition-to-firing. - models/alerts.py + migration 0014: AlertRule.ansible_action (JSON, admin-only, reuses the #546 param shape); AlertEvent.ansible_run_id links a firing event to the run it triggered - core/alerts.py: pure alert_extra_vars() injects steward_alert_* context; on ('firing', event) with an action set, schedule _run_ansible_action (deferred after commit, same pattern as notifications) — fires once per transition, consecutive_failures_required is the debounce; system-triggered AnsibleRun - alerts/routes.py: admin-only parse/validate of the action (source must be configured, playbook must exist); operators keep editing rules, action preserved - rules_form.html: admin-only 'On firing -> run a playbook' section - tests: unit for alert_extra_vars; integration drives record_metric to firing and asserts a system AnsibleRun ran the playbook with the injected var and the event linked to it Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/alerts/routes.py | 74 ++++++++++++++ steward/core/alerts.py | 86 ++++++++++++++++ .../versions/0014_alert_ansible_action.py | 31 ++++++ steward/models/alerts.py | 9 +- steward/templates/alerts/rules_form.html | 51 ++++++++++ .../integration/test_alert_ansible_action.py | 99 +++++++++++++++++++ tests/test_alert_ansible.py | 30 ++++++ 7 files changed, 379 insertions(+), 1 deletion(-) create mode 100644 steward/migrations/versions/0014_alert_ansible_action.py create mode 100644 tests/integration/test_alert_ansible_action.py create mode 100644 tests/test_alert_ansible.py diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py index 9c71119..fd26249 100644 --- a/steward/alerts/routes.py +++ b/steward/alerts/routes.py @@ -4,6 +4,7 @@ from quart import Blueprint, render_template, request, redirect, url_for, curren from steward.core.audit import log_audit from sqlalchemy import select from steward.auth.middleware import require_role +from steward.ansible import sources as src_module from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertOperator, MaintenanceWindow from steward.models.hosts import Host from steward.models.metrics import PluginMetric @@ -56,6 +57,61 @@ async def _get_resources(db, source_module: str) -> list[str]: return sorted(resources) +def _is_admin() -> bool: + return session.get("user_role") == UserRole.admin.value + + +def _ansible_source_names() -> list[str]: + try: + return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))] + except Exception: + return [] + + +def _parse_ansible_action(form) -> tuple[dict | None, str | None]: + """Parse + validate the admin-only Ansible action from a rule form. + + Returns (action_or_None, error_or_None). action is None when the action + checkbox is unchecked (i.e. the action is being cleared). + """ + if "ansible_enabled" not in form: + return None, None + source_name = (form.get("ansible_source", "") or "").strip() + playbook = (form.get("ansible_playbook", "") or "").strip() + inventory = (form.get("ansible_inventory", "") or "").strip() + if not (source_name and playbook and inventory): + return None, "Ansible action requires source, playbook, and inventory" + try: + srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {})) + except Exception: + return None, "Ansible sources are misconfigured" + source = next((s for s in srcs if s["name"] == source_name), None) + if source is None: + return None, f"Ansible source '{source_name}' not found" + if playbook not in src_module.discover_playbooks(source["path"]): + return None, f"Playbook '{playbook}' not found in source '{source_name}'" + extra_vars: list[str] = [] + for line in (form.get("ansible_extra_vars", "") or "").splitlines(): + line = line.strip() + if not line: + continue + if "=" not in line: + return None, f"Invalid extra var (expected key=value): {line!r}" + extra_vars.append(line) + action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory} + if extra_vars: + action["extra_vars"] = extra_vars + limit = (form.get("ansible_limit", "") or "").strip() + tags = (form.get("ansible_tags", "") or "").strip() + if limit: + action["limit"] = limit + if tags: + action["tags"] = tags + if "ansible_check" in form: + action["check"] = True + return action, None + + @alerts_bp.get("/") @require_role(UserRole.viewer) async def list_alerts(): @@ -120,6 +176,7 @@ async def new_rule(): initial_source=first_source, resources=resources, metrics=METRIC_CATALOG.get(first_source, []), + ansible_sources=_ansible_source_names(), ) @@ -140,6 +197,7 @@ async def edit_rule(rule_id: str): initial_source=rule.source_module, resources=resources, metrics=METRIC_CATALOG.get(rule.source_module, []), + ansible_sources=_ansible_source_names(), ) @@ -149,6 +207,12 @@ async def create_rule(): form = await request.form user_id = session.get("user_id") now = datetime.now(timezone.utc) + # Ansible action is admin-only; non-admins simply can't set one. + ansible_action = None + if _is_admin(): + ansible_action, err = _parse_ansible_action(form) + if err: + return err, 400 rule = AlertRule( name=form["name"].strip(), source_module=form["source_module"].strip(), @@ -158,6 +222,7 @@ async def create_rule(): threshold=float(form["threshold"]), consecutive_failures_required=int(form.get("consecutive_failures_required") or 1), enabled=True, + ansible_action=ansible_action, created_by=user_id, ) state = AlertState( @@ -182,6 +247,13 @@ async def create_rule(): @require_role(UserRole.operator) async def update_rule(rule_id: str): form = await request.form + # Validate the admin-only action before opening the transaction. Non-admins + # leave the existing action untouched (they can still edit everything else). + apply_action = _is_admin() + if apply_action: + ansible_action, err = _parse_ansible_action(form) + if err: + return err, 400 async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id)) @@ -195,6 +267,8 @@ async def update_rule(rule_id: str): rule.operator = AlertOperator(form["operator"]) rule.threshold = float(form["threshold"]) rule.consecutive_failures_required = int(form.get("consecutive_failures_required") or 1) + if apply_action: + rule.ansible_action = ansible_action await log_audit(current_app, session.get("user_id"), session.get("username", ""), "alert_rule.updated", entity_type="alert_rule", entity_id=rule.name) return redirect(url_for("alerts.list_alerts")) diff --git a/steward/core/alerts.py b/steward/core/alerts.py index eebd55c..40089cb 100644 --- a/steward/core/alerts.py +++ b/steward/core/alerts.py @@ -32,6 +32,23 @@ def init_alerts(app: "Quart") -> None: _app = app +def alert_extra_vars(rule: AlertRule, value: float) -> list[str]: + """Ansible extra-vars describing the alert that fired, as `key=value` strings. + + Injected into every alert-triggered playbook run so the playbook can target + the exact resource/metric that fired without templating. Pure function. + """ + return [ + f"steward_alert_rule={rule.name}", + f"steward_alert_source={rule.source_module}", + f"steward_alert_resource={rule.resource_name}", + f"steward_alert_metric={rule.metric_name}", + f"steward_alert_value={value}", + f"steward_alert_threshold={rule.threshold}", + f"steward_alert_operator={rule.operator.value}", + ] + + async def record_metric( session: AsyncSession, source_module: str, @@ -110,6 +127,11 @@ async def record_metric( _app, notif_type, rule, value, event_id ) ) + # Admin-configured Ansible action runs only on transition-to-firing. + if notif_type == "firing" and rule.ansible_action: + asyncio.create_task( + _run_ansible_action(_app, rule, value, event_id) + ) def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool: @@ -259,3 +281,67 @@ async def _dispatch_notification( if webhook_r: event.webhook_sent = webhook_r["sent"] event.webhook_error = webhook_r.get("error") + + +async def _run_ansible_action( + app: "Quart", + rule: AlertRule, + value: float, + event_id: str, +) -> None: + """Run the rule's configured Ansible playbook (system-triggered). + + Runs after the evaluation transaction commits (scheduled via create_task). + Creates a system-triggered AnsibleRun, links it to the firing AlertEvent, + then executes it with the static params plus injected alert context vars. + """ + from steward.ansible import executor, sources as src_module + from steward.models.ansible import AnsibleRun, AnsibleRunStatus + + action = rule.ansible_action or {} + playbook = action.get("playbook") + inventory = action.get("inventory") + if not playbook or not inventory: + logger.error("Ansible action for rule %r missing playbook/inventory", rule.name) + return + + try: + srcs = src_module.get_sources(app.config.get("ANSIBLE", {})) + except Exception: + logger.exception("Ansible action for rule %r: could not load sources", rule.name) + return + source = next((s for s in srcs if s["name"] == action.get("source")), None) + if source is None: + logger.error("Ansible action for rule %r: source %r not configured", + rule.name, action.get("source")) + return + + # Static params + auto-injected alert context (all argv, never shell). + params: dict = { + "extra_vars": list(action.get("extra_vars") or []) + alert_extra_vars(rule, value), + } + if action.get("limit"): + params["limit"] = action["limit"] + if action.get("tags"): + params["tags"] = action["tags"] + if action.get("check"): + params["check"] = True + + run_id = str(uuid.uuid4()) + async with app.db_sessionmaker() as db: + async with db.begin(): + db.add(AnsibleRun( + id=run_id, + playbook_path=playbook, + inventory_path=inventory, + source_name=source["name"], + triggered_by=None, + status=AnsibleRunStatus.running, + params=params, + )) + event = (await db.execute( + select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none() + if event: + event.ansible_run_id = run_id + + await executor.start_run(app, run_id, playbook, inventory, source["path"], params) diff --git a/steward/migrations/versions/0014_alert_ansible_action.py b/steward/migrations/versions/0014_alert_ansible_action.py new file mode 100644 index 0000000..6048301 --- /dev/null +++ b/steward/migrations/versions/0014_alert_ansible_action.py @@ -0,0 +1,31 @@ +"""Alert Ansible action: alert_rules.ansible_action + alert_events.ansible_run_id + +Revision ID: 0014_alert_ansible_action +Revises: 0013_ansible_run_params +Create Date: 2026-06-02 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "0014_alert_ansible_action" +down_revision: Union[str, None] = "0013_ansible_run_params" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("alert_rules", sa.Column("ansible_action", sa.JSON(), nullable=True)) + op.add_column( + "alert_events", + sa.Column( + "ansible_run_id", sa.String(36), + sa.ForeignKey("ansible_runs.id", ondelete="SET NULL"), + nullable=True, + ), + ) + + +def downgrade() -> None: + op.drop_column("alert_events", "ansible_run_id") + op.drop_column("alert_rules", "ansible_action") diff --git a/steward/models/alerts.py b/steward/models/alerts.py index 88e85e6..c69b3cf 100644 --- a/steward/models/alerts.py +++ b/steward/models/alerts.py @@ -2,7 +2,7 @@ from __future__ import annotations import enum import uuid from datetime import datetime, timezone -from sqlalchemy import Boolean, DateTime, Enum, Float, ForeignKey, Integer, String, Text +from sqlalchemy import JSON, Boolean, DateTime, Enum, Float, ForeignKey, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -36,6 +36,9 @@ class AlertRule(Base): threshold: Mapped[float] = mapped_column(Float, nullable=False) consecutive_failures_required: Mapped[int] = mapped_column(Integer, nullable=False, default=1) enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + # Optional admin-only action: run an Ansible playbook on transition-to-firing. + # Shape: {source, playbook, inventory, extra_vars, limit, tags, check}. NULL = none. + ansible_action: Mapped[dict | None] = mapped_column(JSON, nullable=True) created_by: Mapped[str] = mapped_column(String(36), ForeignKey("users.id", ondelete="RESTRICT"), nullable=False) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) @@ -97,3 +100,7 @@ class AlertEvent(Base): email_error: Mapped[str | None] = mapped_column(Text, nullable=True) webhook_sent: Mapped[bool | None] = mapped_column(Boolean, nullable=True) webhook_error: Mapped[str | None] = mapped_column(Text, nullable=True) + # Set when this firing event triggered an Ansible action run (audit link). + ansible_run_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("ansible_runs.id", ondelete="SET NULL"), nullable=True + ) diff --git a/steward/templates/alerts/rules_form.html b/steward/templates/alerts/rules_form.html index 99ded85..6805827 100644 --- a/steward/templates/alerts/rules_form.html +++ b/steward/templates/alerts/rules_form.html @@ -128,6 +128,57 @@
+ {# ── Ansible action (admin only) ─────────────────────────────────────── #} + {% if session.get("user_role") == "admin" %} + {% set act = rule.ansible_action if rule and rule.ansible_action else None %} +
+ +

+ Admin only. Runs once when the alert transitions into FIRING. + steward_alert_* context vars (resource, metric, value…) are injected automatically. +

+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+ {% endif %} +
Cancel diff --git a/tests/integration/test_alert_ansible_action.py b/tests/integration/test_alert_ansible_action.py new file mode 100644 index 0000000..e1d2ca2 --- /dev/null +++ b/tests/integration/test_alert_ansible_action.py @@ -0,0 +1,99 @@ +"""Integration: an alert transition-to-firing triggers its Ansible action (task 250). + +Drives the full chain against a live Postgres + real Ansible: a breaching metric +fires the rule, which schedules a system-triggered AnsibleRun that runs a real +connection:local playbook receiving the injected steward_alert_* context, and the +firing AlertEvent links to the run. +""" +from __future__ import annotations + +import asyncio +import os +import textwrap +import uuid + +import pytest + +pytestmark = pytest.mark.integration + +_NEEDS_DB = pytest.mark.skipif( + not os.environ.get("STEWARD_DATABASE_URL"), + reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)", +) + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +@_NEEDS_DB +def test_firing_triggers_ansible_action(app, tmp_path): + from sqlalchemy import select + from steward.core.alerts import record_metric + from steward.models.alerts import ( + AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, + ) + from steward.models.ansible import AnsibleRun, AnsibleRunStatus + from steward.models.users import User, UserRole + + (tmp_path / "fire.yml").write_text(textwrap.dedent("""\ + - hosts: localhost + connection: local + gather_facts: false + tasks: + - debug: + msg: "FIRED-{{ steward_alert_metric }}" + """)) + (tmp_path / "inv.ini").write_text("localhost ansible_connection=local\n") + + # An ansible source the action resolves against. + app.config["ANSIBLE"] = { + "sources": [{"name": "t", "type": "local", "path": str(tmp_path)}], + } + + uid = str(uuid.uuid4()) + rule_id = str(uuid.uuid4()) + action = {"source": "t", "playbook": "fire.yml", "inventory": "inv.ini"} + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(User( + id=uid, username=f"u{uid[:8]}", email=f"{uid[:8]}@e.test", + password_hash="x", role=UserRole.admin, is_active=True, + )) + s.add(AlertRule( + id=rule_id, name="cpu hot", source_module="host_agent", + resource_name="srv1", metric_name="cpu_pct", + operator=AlertOperator.gt, threshold=90.0, + consecutive_failures_required=1, enabled=True, + ansible_action=action, created_by=uid, + )) + s.add(AlertState(rule_id=rule_id, state=AlertStateEnum.inactive)) + + # Breaching value → transition to firing → schedules the action task. + async with app.db_sessionmaker() as s: + async with s.begin(): + await record_metric(s, "host_agent", "srv1", "cpu_pct", 95.0) + + # Let the fire-and-forget dispatch + action tasks complete. + pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] + await asyncio.gather(*pending, return_exceptions=True) + + async with app.db_sessionmaker() as s: + run = (await s.execute( + select(AnsibleRun).where(AnsibleRun.source_name == "t"))).scalars().first() + event = (await s.execute( + select(AlertEvent).where(AlertEvent.rule_id == rule_id))).scalar_one() + return run, event + + run, event = asyncio.run(_go()) + assert run is not None, "no AnsibleRun was created by the firing rule" + assert run.triggered_by is None # system-triggered + assert run.status == AnsibleRunStatus.success, run.output + assert "FIRED-cpu_pct" in (run.output or "") # injected context var reached the play + assert event.ansible_run_id == run.id # firing event linked to the run diff --git a/tests/test_alert_ansible.py b/tests/test_alert_ansible.py new file mode 100644 index 0000000..d7beb9d --- /dev/null +++ b/tests/test_alert_ansible.py @@ -0,0 +1,30 @@ +"""Unit tests for alert -> Ansible action context vars (task 250).""" +from steward.core.alerts import alert_extra_vars +from steward.models.alerts import AlertOperator, AlertRule + + +def test_alert_extra_vars_describes_the_firing(): + rule = AlertRule( + name="UPS low battery", source_module="snmp", resource_name="ups", + metric_name="battery_status", operator=AlertOperator.lt, threshold=3.0, + created_by="u", + ) + ev = alert_extra_vars(rule, 2.0) + assert "steward_alert_rule=UPS low battery" in ev + assert "steward_alert_source=snmp" in ev + assert "steward_alert_resource=ups" in ev + assert "steward_alert_metric=battery_status" in ev + assert "steward_alert_value=2.0" in ev + assert "steward_alert_threshold=3.0" in ev + assert "steward_alert_operator=<" in ev + + +def test_alert_extra_vars_are_key_value_strings(): + rule = AlertRule( + name="r", source_module="host_agent", resource_name="srv1", + metric_name="cpu_pct", operator=AlertOperator.gt, threshold=90.0, + created_by="u", + ) + ev = alert_extra_vars(rule, 95.5) + assert all("=" in item for item in ev) + assert all(item.startswith("steward_alert_") for item in ev) From 43a5325e69c6a83af7c73dcce1c08aac67b3e6d1 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 2 Jun 2026 14:27:25 -0400 Subject: [PATCH 011/126] fix(alerts): persist AlertOperator by value to match the DB enum MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The alertoperator Postgres enum (migration 0002) was created with the operator VALUES as labels ('>', '<', …), but the model's Enum(AlertOperator) defaulted to persisting enum-member NAMES ('gt', 'lt', …) — so inserting any AlertRule against real Postgres raised 'invalid input value for enum alertoperator: "gt"'. Never caught because unit tests mock the DB and no integration test inserted a rule until task 250's. Add values_callable so the ORM round-trips the values. Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/models/alerts.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/steward/models/alerts.py b/steward/models/alerts.py index c69b3cf..3e86b8e 100644 --- a/steward/models/alerts.py +++ b/steward/models/alerts.py @@ -32,7 +32,13 @@ class AlertRule(Base): source_module: Mapped[str] = mapped_column(String(64), nullable=False) resource_name: Mapped[str] = mapped_column(String(255), nullable=False) metric_name: Mapped[str] = mapped_column(String(128), nullable=False) - operator: Mapped[AlertOperator] = mapped_column(Enum(AlertOperator), nullable=False) + # The DB enum (migration 0002) stores the operator VALUES (">", "<", …), so + # persist values rather than SQLAlchemy's default enum-member NAMES ("gt"…). + operator: Mapped[AlertOperator] = mapped_column( + Enum(AlertOperator, name="alertoperator", + values_callable=lambda e: [m.value for m in e]), + nullable=False, + ) threshold: Mapped[float] = mapped_column(Float, nullable=False) consecutive_failures_required: Mapped[int] = mapped_column(Integer, nullable=False, default=1) enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) From 534ed030b898549cf04d2aec0bf1e7a547815179 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 2 Jun 2026 14:31:25 -0400 Subject: [PATCH 012/126] test(alerts): commit the test user before the rule's created_by FK Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/integration/test_alert_ansible_action.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_alert_ansible_action.py b/tests/integration/test_alert_ansible_action.py index e1d2ca2..a72f5d1 100644 --- a/tests/integration/test_alert_ansible_action.py +++ b/tests/integration/test_alert_ansible_action.py @@ -61,11 +61,13 @@ def test_firing_triggers_ansible_action(app, tmp_path): async def _go(): async with app.db_sessionmaker() as s: + # User must be committed before the rule's created_by FK can reference it. async with s.begin(): s.add(User( id=uid, username=f"u{uid[:8]}", email=f"{uid[:8]}@e.test", password_hash="x", role=UserRole.admin, is_active=True, )) + async with s.begin(): s.add(AlertRule( id=rule_id, name="cpu hot", source_module="host_agent", resource_name="srv1", metric_name="cpu_pct", From 0a36a57901dbe432075e59da396bcd122fc0dae1 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 2 Jun 2026 17:58:18 -0400 Subject: [PATCH 013/126] chore(dev): mount ./ansible-dev as a local Ansible source at /srv/ansible Dev convenience so you can author playbooks on the host and have Steward's local-source discovery pick them up. ./ansible-dev/ is gitignored (operator content); a safe demo playbook lives there locally. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 3 +++ docker-compose.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index b4b5ccd..2678717 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,6 @@ playbook_cache/ # Superpowers brainstorm scratch (visual companion mockups, state) .superpowers/ + +# Local dev Ansible playbooks (operator content, mounted into the container) +/ansible-dev/ diff --git a/docker-compose.yml b/docker-compose.yml index 5d57fd8..9f7cf81 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,9 @@ services: # (below) makes this mounted source win over the image-installed package. - ./steward:/app/steward - ./plugins:/app/plugins + # Dev playbooks: edit on the host under ./ansible-dev/, register a *local* + # Ansible source pointing at /srv/ansible in Settings → Ansible. + - ./ansible-dev:/srv/ansible # Optional host mounts — enable the matching plugin in Settings first, then # uncomment (the traefik path must exist on this host): # - /mnt/Data/traefik/log:/var/log/traefik:ro # traefik plugin From 0bf007173bffb806f0877f785d4570c4c92e0a90 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 2 Jun 2026 18:41:58 -0400 Subject: [PATCH 014/126] =?UTF-8?q?feat(ansible):=20credentials=20?= =?UTF-8?q?=E2=80=94=20SSH=20key,=20become,=20vault,=20host-key=20checking?= =?UTF-8?q?=20(task=20548)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Global Ansible credentials, applied to every run (manual + alert-triggered): - core/settings.py: ansible.ssh_private_key / become_password / vault_password (plaintext at rest, masked in UI — encryption tracked in #580) + host_key_checking (default off); surfaced via to_ansible_cfg into app.config[ANSIBLE] - executor.py: pure build_credentials() materializes creds into a 0600 temp dir (--private-key, --vault-password-file, become via -e @vars-file so the password never hits argv) cleaned up in finally; pure ansible_env() sets ANSIBLE_HOST_KEY_CHECKING. build_ansible_command stays param-only - settings/routes.py + ansible.html: admin-only Credentials section, masked-update (blank keeps current, explicit Clear checkbox), reload app config on save - tests: unit (build_credentials per cred + none; ansible_env toggle); integration vault round-trip (ansible-vault encrypt a vars file, run via executor with the vault password, assert it decrypted) Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/ansible/executor.py | 59 ++++++++++++- steward/core/settings.py | 14 +++- steward/settings/routes.py | 38 ++++++++- steward/templates/settings/ansible.html | 48 +++++++++++ tests/integration/test_ansible_credentials.py | 82 +++++++++++++++++++ tests/test_ansible_credentials.py | 57 +++++++++++++ 6 files changed, 295 insertions(+), 3 deletions(-) create mode 100644 tests/integration/test_ansible_credentials.py create mode 100644 tests/test_ansible_credentials.py diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py index 92ee4ab..e27eeb5 100644 --- a/steward/ansible/executor.py +++ b/steward/ansible/executor.py @@ -1,7 +1,11 @@ # steward/ansible/executor.py from __future__ import annotations import asyncio +import json import logging +import os +import shutil +import tempfile import time from dataclasses import dataclass from datetime import datetime, timezone @@ -83,6 +87,46 @@ def build_ansible_command( return cmd +def build_credentials(creds: dict | None, tmpdir: str) -> tuple[list[str], list[tuple[str, str]]]: + """Given global Ansible creds + a temp dir, return (extra argv, files to write). + + Pure: decides flags + file contents; the caller writes the files (mode 0600). + Secrets are always passed via files — never on argv / the process list. + creds keys: ssh_private_key, vault_password, become_password. + """ + args: list[str] = [] + files: list[tuple[str, str]] = [] + creds = creds or {} + + key = creds.get("ssh_private_key") + if key: + path = os.path.join(tmpdir, "id_key") + files.append((path, key if key.endswith("\n") else key + "\n")) + args += ["--private-key", path] + + vault = creds.get("vault_password") + if vault: + path = os.path.join(tmpdir, "vault_pass") + files.append((path, vault + "\n")) + args += ["--vault-password-file", path] + + become = creds.get("become_password") + if become: + path = os.path.join(tmpdir, "become.yml") + # A JSON string is valid YAML; keeps the password out of argv. + files.append((path, "ansible_become_password: " + json.dumps(become) + "\n")) + args += ["-e", "@" + path] + + return args, files + + +def ansible_env(creds: dict | None, base_env) -> dict: + """Return a copy of base_env with ANSIBLE_HOST_KEY_CHECKING set from creds.""" + env = dict(base_env) + env["ANSIBLE_HOST_KEY_CHECKING"] = "True" if (creds or {}).get("host_key_checking") else "False" + return env + + async def start_run( app: "Quart", run_id: str, @@ -119,13 +163,24 @@ async def start_run( cmd = build_ansible_command(playbook_path, inventory_path, params) cwd = source_path if Path(source_path).exists() else None + creds = app.config.get("ANSIBLE", {}) + # Materialize credentials into a private temp dir; removed in finally. + tmpdir = tempfile.mkdtemp(prefix="steward-ansible-") + os.chmod(tmpdir, 0o700) try: + cred_args, cred_files = build_credentials(creds, tmpdir) + for cred_path, content in cred_files: + with open(cred_path, "w", encoding="utf-8") as cf: + cf.write(content) + os.chmod(cred_path, 0o600) + proc = await asyncio.create_subprocess_exec( - *cmd, + *cmd, *cred_args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, cwd=cwd, + env=ansible_env(creds, os.environ), ) assert proc.stdout is not None @@ -152,6 +207,8 @@ async def start_run( except Exception: logger.exception("Ansible run %s failed with exception", run_id) final_status = "failed" + finally: + shutil.rmtree(tmpdir, ignore_errors=True) await _flush(final_status) _broadcast(run_id, _Done(status=final_status)) diff --git a/steward/core/settings.py b/steward/core/settings.py index 5acc0d1..a1e055a 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -51,6 +51,12 @@ DEFAULTS: dict[str, Any] = { "webhook.url": "", "webhook.template": _DEFAULT_WEBHOOK_TEMPLATE, "ansible.sources": [], + # Ansible credentials — global, used by every run (manual + alert-triggered). + # Plaintext at rest, masked in the UI (encryption-at-rest tracked separately). + "ansible.ssh_private_key": "", + "ansible.become_password": "", + "ansible.vault_password": "", + "ansible.host_key_checking": False, "ping.threshold.good_ms": 50, "ping.threshold.warn_ms": 200, "plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml", @@ -152,7 +158,13 @@ def to_webhook_cfg(settings: dict[str, Any]) -> dict: def to_ansible_cfg(settings: dict[str, Any]) -> dict: - return {"sources": settings.get("ansible.sources", [])} + return { + "sources": settings.get("ansible.sources", []), + "ssh_private_key": settings.get("ansible.ssh_private_key", ""), + "become_password": settings.get("ansible.become_password", ""), + "vault_password": settings.get("ansible.vault_password", ""), + "host_key_checking": settings.get("ansible.host_key_checking", False), + } def to_oidc_cfg(settings: dict[str, Any]) -> dict: diff --git a/steward/settings/routes.py b/steward/settings/routes.py index f2740a4..4414031 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -170,7 +170,43 @@ async def _ansible_sources_partial(sources: list[dict]): @require_role(UserRole.admin) async def ansible(): sources = await _get_ansible_sources() - return await render_template("settings/ansible.html", sources=sources) + async with current_app.db_sessionmaker() as db: + settings = await get_all_settings(db) + return await render_template( + "settings/ansible.html", + sources=sources, + ssh_key_set=bool(settings.get("ansible.ssh_private_key")), + become_set=bool(settings.get("ansible.become_password")), + vault_set=bool(settings.get("ansible.vault_password")), + host_key_checking=settings.get("ansible.host_key_checking", False), + ) + + +@settings_bp.post("/ansible/credentials") +@require_role(UserRole.admin) +async def ansible_save_credentials(): + """Save global Ansible credentials (masked-update: blank keeps the current value).""" + form = await request.form + async with current_app.db_sessionmaker() as db: + async with db.begin(): + for field, key in ( + ("ssh_private_key", "ansible.ssh_private_key"), + ("become_password", "ansible.become_password"), + ("vault_password", "ansible.vault_password"), + ): + if f"clear_{field}" in form: + await set_setting(db, key, "") + continue + raw = form.get(field, "") or "" + # Preserve PEM whitespace for the key; trim passwords. + val = raw.strip("\n") if field == "ssh_private_key" else raw.strip() + if val: + await set_setting(db, key, val) + await set_setting(db, "ansible.host_key_checking", "host_key_checking" in form) + await _reload_app_config() + await log_audit(current_app, session.get("user_id"), session.get("username", ""), + "settings.saved", detail={"section": "ansible.credentials"}) + return redirect(url_for("settings.ansible")) @settings_bp.post("/ansible/sources/add") diff --git a/steward/templates/settings/ansible.html b/steward/templates/settings/ansible.html index f0b1d4a..64aa5a2 100644 --- a/steward/templates/settings/ansible.html +++ b/steward/templates/settings/ansible.html @@ -15,5 +15,53 @@ cloned automatically. Add as many as you need; each can be browsed and run independently.

{% include "settings/_ansible_sources.html" %} + + {# ── Credentials ──────────────────────────────────────────────────────────── #} + {% set cfg_badge = '(configured)' %} + {% set not_set_badge = '(not set)' %} +
+
Credentials
+

+ Used by every playbook run (manual and alert-triggered). Stored on the server and + never shown again after saving — leave a field blank to keep the current value. +

+
+
+ + + {% if ssh_key_set %} + + {% endif %} +
+
+ + + {% if become_set %} + + {% endif %} +
+
+ + + {% if vault_set %} + + {% endif %} +
+
+ +
+ +
+
{% endblock %} diff --git a/tests/integration/test_ansible_credentials.py b/tests/integration/test_ansible_credentials.py new file mode 100644 index 0000000..c24f2e6 --- /dev/null +++ b/tests/integration/test_ansible_credentials.py @@ -0,0 +1,82 @@ +"""Integration: the executor wires a configured vault password into a real run (task 548). + +Encrypts a vars file with ansible-vault, then runs a playbook through +executor.start_run with the matching vault_password in app config, and asserts +the executor's --vault-password-file decrypted it. (SSH-key/become auth need a +remote host, so they're covered by the unit-level flag construction.) +""" +from __future__ import annotations + +import asyncio +import os +import subprocess +import textwrap +import uuid + +import pytest + +pytestmark = pytest.mark.integration + +_NEEDS_DB = pytest.mark.skipif( + not os.environ.get("STEWARD_DATABASE_URL"), + reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)", +) + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +@_NEEDS_DB +def test_vault_password_decrypts_a_run(app, tmp_path): + from sqlalchemy import select + from steward.ansible import executor + from steward.models.ansible import AnsibleRun, AnsibleRunStatus + + vault_pw = "test-vault-pw" + pwfile = tmp_path / "pw" + pwfile.write_text(vault_pw + "\n") + + secret = tmp_path / "secret.yml" + secret.write_text("vaulted_value: hello-from-vault\n") + subprocess.run( + ["ansible-vault", "encrypt", str(secret), "--vault-password-file", str(pwfile)], + check=True, capture_output=True, + ) + + (tmp_path / "play.yml").write_text(textwrap.dedent("""\ + - hosts: localhost + connection: local + gather_facts: false + vars_files: + - secret.yml + tasks: + - debug: + msg: "VAULT-{{ vaulted_value }}" + """)) + (tmp_path / "inv.ini").write_text("localhost ansible_connection=local\n") + + # Executor reads credentials from app.config["ANSIBLE"]. + app.config["ANSIBLE"] = {"vault_password": vault_pw, "host_key_checking": False} + + run_id = str(uuid.uuid4()) + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(AnsibleRun( + id=run_id, playbook_path="play.yml", inventory_path="inv.ini", + source_name="t", triggered_by=None, status=AnsibleRunStatus.running, + )) + await executor.start_run(app, run_id, "play.yml", "inv.ini", str(tmp_path)) + async with app.db_sessionmaker() as s: + return (await s.execute( + select(AnsibleRun).where(AnsibleRun.id == run_id))).scalar_one() + + run = asyncio.run(_go()) + assert run.status == AnsibleRunStatus.success, run.output + assert "VAULT-hello-from-vault" in (run.output or "") # vault password decrypted it diff --git a/tests/test_ansible_credentials.py b/tests/test_ansible_credentials.py new file mode 100644 index 0000000..5481f67 --- /dev/null +++ b/tests/test_ansible_credentials.py @@ -0,0 +1,57 @@ +"""Unit tests for Ansible credential argv/env construction (task 548).""" +import json + +from steward.ansible.executor import ansible_env, build_credentials + + +def test_no_creds_is_empty(): + assert build_credentials({}, "/td") == ([], []) + assert build_credentials(None, "/td") == ([], []) + + +def test_ssh_key_via_file(): + args, files = build_credentials({"ssh_private_key": "KEYDATA"}, "/td") + assert args == ["--private-key", "/td/id_key"] + assert files == [("/td/id_key", "KEYDATA\n")] + + +def test_ssh_key_keeps_existing_trailing_newline(): + _, files = build_credentials({"ssh_private_key": "KEY\n"}, "/td") + assert files == [("/td/id_key", "KEY\n")] + + +def test_vault_password_via_file(): + args, files = build_credentials({"vault_password": "vpw"}, "/td") + assert args == ["--vault-password-file", "/td/vault_pass"] + assert files == [("/td/vault_pass", "vpw\n")] + + +def test_become_password_uses_vars_file_not_argv(): + pw = "p@ss word" + args, files = build_credentials({"become_password": pw}, "/td") + assert args == ["-e", "@/td/become.yml"] + assert files[0][0] == "/td/become.yml" + assert files[0][1] == "ansible_become_password: " + json.dumps(pw) + "\n" + # The password must never appear on the command line. + assert pw not in " ".join(args) + + +def test_all_three_combined(): + args, files = build_credentials( + {"ssh_private_key": "K", "vault_password": "V", "become_password": "B"}, "/td") + assert "--private-key" in args + assert "--vault-password-file" in args + assert "-e" in args + assert len(files) == 3 + + +def test_ansible_env_host_key_checking_toggle(): + assert ansible_env({"host_key_checking": True}, {})["ANSIBLE_HOST_KEY_CHECKING"] == "True" + assert ansible_env({"host_key_checking": False}, {})["ANSIBLE_HOST_KEY_CHECKING"] == "False" + assert ansible_env({}, {})["ANSIBLE_HOST_KEY_CHECKING"] == "False" + assert ansible_env(None, {})["ANSIBLE_HOST_KEY_CHECKING"] == "False" + + +def test_ansible_env_preserves_base(): + env = ansible_env({}, {"FOO": "bar"}) + assert env["FOO"] == "bar" From 38f61b71c1619cef6aaaead8bf7f111f293d36ab Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 2 Jun 2026 19:26:58 -0400 Subject: [PATCH 015/126] feat(ansible): run a playbook against a single Steward host (task 547) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From a host's edit page, an operator can run a playbook against just that host via an ephemeral one-host inventory — no need for the host to exist in a source inventory. - ansible/sources.py: pure host_inventory_content(host) -> ' ansible_host=' - executor.start_run: optional inventory_content written to the temp dir and used as -i (overrides inventory_path); cleaned up with the creds dir - hosts/routes.py: POST /hosts//run-playbook (operator) — validates source + playbook, builds the ephemeral inventory, starts a user-triggered run, redirects to the live run detail; edit page gets the ansible source list - hosts/form.html: 'Run Ansible playbook against this host' panel (shown when editing an existing host and sources exist) — source + playbook + extra-vars/ tags/dry-run (no --limit; single host) - tests: unit host_inventory_content; integration runs a hosts:all/connection:local playbook against the ephemeral inventory and asserts inventory_hostname Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/ansible/executor.py | 18 ++++- steward/ansible/sources.py | 12 +++ steward/hosts/routes.py | 90 ++++++++++++++++++++- steward/templates/hosts/form.html | 38 +++++++++ tests/integration/test_host_targeted_run.py | 66 +++++++++++++++ tests/test_host_inventory.py | 13 +++ 6 files changed, 233 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_host_targeted_run.py create mode 100644 tests/test_host_inventory.py diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py index e27eeb5..d194b67 100644 --- a/steward/ansible/executor.py +++ b/steward/ansible/executor.py @@ -134,8 +134,13 @@ async def start_run( inventory_path: str, source_path: str, params: dict | None = None, + inventory_content: str | None = None, ) -> None: - """Execute ansible-playbook as a subprocess and update the DB run row.""" + """Execute ansible-playbook as a subprocess and update the DB run row. + + If inventory_content is given (host-targeted runs), it's written to a temp + file and used as the inventory instead of inventory_path. + """ _run_lines[run_id] = [] db_output = "" @@ -161,14 +166,21 @@ async def start_run( lines_since_flush = 0 last_flush_at = time.monotonic() - cmd = build_ansible_command(playbook_path, inventory_path, params) cwd = source_path if Path(source_path).exists() else None creds = app.config.get("ANSIBLE", {}) - # Materialize credentials into a private temp dir; removed in finally. + # Temp dir holds the ephemeral inventory (if any) + credential files; removed in finally. tmpdir = tempfile.mkdtemp(prefix="steward-ansible-") os.chmod(tmpdir, 0o700) try: + if inventory_content: + inv_target = os.path.join(tmpdir, "inventory") + with open(inv_target, "w", encoding="utf-8") as invf: + invf.write(inventory_content) + else: + inv_target = inventory_path + cmd = build_ansible_command(playbook_path, inv_target, params) + cred_args, cred_files = build_credentials(creds, tmpdir) for cred_path, content in cred_files: with open(cred_path, "w", encoding="utf-8") as cf: diff --git a/steward/ansible/sources.py b/steward/ansible/sources.py index f6e6135..a457d4d 100644 --- a/steward/ansible/sources.py +++ b/steward/ansible/sources.py @@ -72,6 +72,18 @@ def discover_inventories(source_path: str) -> list[str]: return sorted(name for name in INVENTORY_NAMES if (root / name).exists()) +def host_inventory_content(host) -> str: + """An ephemeral single-host inventory line for a Steward Host. + + Targets the host's address when set (`ansible_host=`), else just the name + (resolved via DNS). Playbooks run against this should use `hosts: all` (or + the host's name). Pure — `host` only needs `.name` and `.address`. + """ + if getattr(host, "address", ""): + return f"{host.name} ansible_host={host.address}\n" + return f"{host.name}\n" + + def read_playbook(source_path: str, relative_path: str) -> str | None: """Return contents of a playbook file, or None if not found / path escape.""" root = Path(source_path).resolve() diff --git a/steward/hosts/routes.py b/steward/hosts/routes.py index 33afa1a..c48e795 100644 --- a/steward/hosts/routes.py +++ b/steward/hosts/routes.py @@ -1,9 +1,13 @@ from __future__ import annotations +import asyncio +import uuid from datetime import datetime, timedelta, timezone from quart import Blueprint, render_template, request, redirect, url_for, current_app, session from sqlalchemy import and_, case, select, func +from steward.ansible import executor, sources as ansible_src from steward.auth.middleware import require_role from steward.core.audit import log_audit +from steward.models.ansible import AnsibleRun, AnsibleRunStatus from steward.models.hosts import Host, ProbeType from steward.models.monitors import PingResult, DnsResult, PingStatus from steward.models.users import UserRole @@ -11,6 +15,13 @@ from steward.models.users import UserRole hosts_bp = Blueprint("hosts", __name__, url_prefix="/hosts") +def _ansible_source_names() -> list[str]: + try: + return [s["name"] for s in ansible_src.get_sources(current_app.config.get("ANSIBLE", {}))] + except Exception: + return [] + + async def _compute_uptime(db) -> dict[str, dict]: """Return per-host uptime % for 24h, 7d, 30d windows. @@ -141,7 +152,9 @@ async def edit_host(host_id: str): host = result.scalar_one_or_none() if host is None: return "Not found", 404 - return await render_template("hosts/form.html", host=host, probe_types=list(ProbeType)) + return await render_template( + "hosts/form.html", host=host, probe_types=list(ProbeType), + ansible_sources=_ansible_source_names()) @hosts_bp.post("/") @@ -165,6 +178,81 @@ async def update_host(host_id: str): return redirect(url_for("hosts.list_hosts")) +@hosts_bp.post("//run-playbook") +@require_role(UserRole.operator) +async def run_playbook(host_id: str): + """Run an Ansible playbook against just this host (ephemeral one-host inventory).""" + form = await request.form + async with current_app.db_sessionmaker() as db: + host = (await db.execute(select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return "Not found", 404 + + source_name = (form.get("source_name", "") or "").strip() + playbook = (form.get("playbook_path", "") or "").strip() + if not source_name or not playbook: + return "source and playbook are required", 400 + try: + srcs = ansible_src.get_sources(current_app.config.get("ANSIBLE", {})) + except Exception: + return "Ansible sources are misconfigured", 400 + source = next((s for s in srcs if s["name"] == source_name), None) + if source is None: + return "Source not found", 404 + if playbook not in ansible_src.discover_playbooks(source["path"]): + return f"Playbook '{playbook}' not found in source '{source_name}'", 404 + + # Optional params (no --limit — there's exactly one host). + extra_vars: list[str] = [] + for line in (form.get("extra_vars", "") or "").splitlines(): + line = line.strip() + if not line: + continue + if "=" not in line: + return f"Invalid extra var (expected key=value): {line!r}", 400 + extra_vars.append(line) + params: dict = {} + if extra_vars: + params["extra_vars"] = extra_vars + tags = (form.get("tags", "") or "").strip() + if tags: + params["tags"] = tags + if "check" in form: + params["check"] = True + params_or_none = params or None + + run_id = str(uuid.uuid4()) + inv_content = ansible_src.host_inventory_content(host) + run = AnsibleRun( + id=run_id, + playbook_path=playbook, + inventory_path=f"host: {host.name}", + source_name=source_name, + triggered_by=session.get("user_id"), + status=AnsibleRunStatus.running, + params=params_or_none, + ) + async with current_app.db_sessionmaker() as db: + async with db.begin(): + db.add(run) + + task = asyncio.create_task( + executor.start_run( + current_app._get_current_object(), # type: ignore[attr-defined] + run_id, playbook, f"host: {host.name}", source["path"], + params_or_none, inv_content, + ) + ) + task.add_done_callback( + lambda t: t.exception() and current_app.logger.error( + "Host playbook run %s raised: %s", run_id, t.exception())) + + await log_audit(current_app, session.get("user_id"), session.get("username", ""), + "ansible.host_run", entity_type="host", entity_id=host.name, + detail={"playbook": playbook, "source": source_name}) + return redirect(url_for("ansible.run_detail", run_id=run_id)) + + @hosts_bp.post("//delete") @require_role(UserRole.admin) async def delete_host(host_id: str): diff --git a/steward/templates/hosts/form.html b/steward/templates/hosts/form.html index ef2cd82..3963be2 100644 --- a/steward/templates/hosts/form.html +++ b/steward/templates/hosts/form.html @@ -49,5 +49,43 @@
+ + {% if host and ansible_sources %} +
+

Run Ansible playbook against this host

+

+ Runs against an ephemeral one-host inventory for + {{ host.name }} ({{ host.address or "no address — resolves by name" }}). + Use a playbook with hosts: all. +

+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ +
+
+ {% endif %}
{% endblock %} diff --git a/tests/integration/test_host_targeted_run.py b/tests/integration/test_host_targeted_run.py new file mode 100644 index 0000000..bdd6ad7 --- /dev/null +++ b/tests/integration/test_host_targeted_run.py @@ -0,0 +1,66 @@ +"""Integration: a host-targeted run uses the ephemeral one-host inventory (task 547). + +Runs a `hosts: all` / `connection: local` playbook through executor.start_run with +an ephemeral inventory built from a Host, and asserts the play executed against +that host's inventory_hostname — proving the inventory_content plumbing. +""" +from __future__ import annotations + +import asyncio +import os +import textwrap +import uuid + +import pytest + +pytestmark = pytest.mark.integration + +_NEEDS_DB = pytest.mark.skipif( + not os.environ.get("STEWARD_DATABASE_URL"), + reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)", +) + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +@_NEEDS_DB +def test_ephemeral_inventory_targets_host(app, tmp_path): + from sqlalchemy import select + from steward.ansible import executor, sources + from steward.models.ansible import AnsibleRun, AnsibleRunStatus + from steward.models.hosts import Host + + (tmp_path / "play.yml").write_text(textwrap.dedent("""\ + - hosts: all + connection: local + gather_facts: false + tasks: + - debug: + msg: "HOST-{{ inventory_hostname }}" + """)) + inv_content = sources.host_inventory_content(Host(name="demo-host", address="127.0.0.1")) + + run_id = str(uuid.uuid4()) + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(AnsibleRun( + id=run_id, playbook_path="play.yml", inventory_path="host: demo-host", + source_name="t", triggered_by=None, status=AnsibleRunStatus.running, + )) + await executor.start_run( + app, run_id, "play.yml", "host: demo-host", str(tmp_path), None, inv_content) + async with app.db_sessionmaker() as s: + return (await s.execute( + select(AnsibleRun).where(AnsibleRun.id == run_id))).scalar_one() + + run = asyncio.run(_go()) + assert run.status == AnsibleRunStatus.success, run.output + assert "HOST-demo-host" in (run.output or "") # ran against the ephemeral inventory host diff --git a/tests/test_host_inventory.py b/tests/test_host_inventory.py new file mode 100644 index 0000000..db6b601 --- /dev/null +++ b/tests/test_host_inventory.py @@ -0,0 +1,13 @@ +"""Unit tests for the ephemeral one-host inventory helper (task 547).""" +from steward.ansible.sources import host_inventory_content +from steward.models.hosts import Host + + +def test_with_address_sets_ansible_host(): + inv = host_inventory_content(Host(name="web1", address="10.0.0.5")) + assert inv == "web1 ansible_host=10.0.0.5\n" + + +def test_without_address_falls_back_to_name(): + inv = host_inventory_content(Host(name="web1", address="")) + assert inv == "web1\n" From fe57dde57ba9d3949a2d9dbf4850cbc508d03bb2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 11:08:43 -0400 Subject: [PATCH 016/126] feat(ci): publish steward image to the registry + remote-deploy compose Add the rule-46 image-publish lane: a `publish` job gated on [lint, unit, integration] that builds the runtime Dockerfile in ci-builder and pushes git.fabledsword.com/bvandeusen/steward: always, :dev on dev and :latest on main. Authenticates with the REGISTRY_TOKEN repo secret (the injected GITHUB_TOKEN lacks write:package). Steward is the first family app to implement the publish lane; mechanics mirror CI-runner's build-ci-python.yml. Add compose.deploy.yml for a persistent remote instance that pulls :dev (no source bind-mounts, persistent app_data+pgdata volumes, bundled Postgres), plus a README "Remote dev instance" runbook and the POSTGRES_PASSWORD env key. Fix .env.example's database URL var to the canonical STEWARD_DATABASE_URL (single underscore) and document the publish lane in ci-requirements.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 8 +++++- .forgejo/workflows/ci.yml | 37 +++++++++++++++++++++++++++ README.md | 36 ++++++++++++++++++++++++++ ci-requirements.md | 15 +++++++++-- compose.deploy.yml | 53 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 compose.deploy.yml diff --git a/.env.example b/.env.example index 8d36b78..84f9d65 100644 --- a/.env.example +++ b/.env.example @@ -2,5 +2,11 @@ # These override values in config.yaml STEWARD_SECRET_KEY=change-me -STEWARD_DATABASE__URL=postgresql+asyncpg://steward:password@localhost/steward +STEWARD_DATABASE_URL=postgresql+asyncpg://steward:password@localhost/steward STEWARD_SMTP__PASSWORD= + +# --- compose.deploy.yml (remote instance) only --- +# Password for the bundled Postgres; the steward service builds its +# STEWARD_DATABASE_URL from it. Leave STEWARD_SECRET_KEY above unset to let the +# app generate + persist one on the /data volume. +POSTGRES_PASSWORD=change-me diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index ab6dea8..800746e 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -90,3 +90,40 @@ jobs: pip install -e '.[dev,ansible]' fi pytest tests/integration -v -m integration --durations=10 + + # Image-publish lane — FabledRulebook rule 46: every push to dev/main publishes + # an immutable commit-SHA image (the rollback unit); dev moves :dev, main moves + # :latest. Gated on the three test lanes via `needs:` so a red commit never + # produces a :dev image. Mechanics mirror CI-runner's build-ci-python.yml. + publish: + needs: [lint, unit, integration] + runs-on: python-ci + container: + # ci-builder carries docker + buildx + git. The docker socket is + # auto-mounted by act_runner (valid_volumes) — do NOT add a + # container.options "-v /var/run/docker.sock:..." mount; that duplicate + # mount fails the job at "Set up job". + image: git.fabledsword.com/bvandeusen/ci-builder:latest + steps: + - uses: actions/checkout@v4 + - name: Registry login + # The injected GITHUB_TOKEN lacks write:package scope, so docker push + # 401s with it. REGISTRY_TOKEN is a repo Actions secret holding a + # Forgejo token scoped read:package + write:package. + run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.fabledsword.com -u bvandeusen --password-stdin + - name: Build + push (commit-SHA always; :dev on dev, :latest on main) + run: | + set -euxo pipefail + IMAGE=git.fabledsword.com/bvandeusen/steward + docker build -t "$IMAGE:${{ github.sha }}" . + docker push "$IMAGE:${{ github.sha }}" + if [ "${{ github.ref }}" = "refs/heads/dev" ]; then + docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:dev" + docker push "$IMAGE:dev" + elif [ "${{ github.ref }}" = "refs/heads/main" ]; then + docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:latest" + docker push "$IMAGE:latest" + fi + - name: Prune dangling layers + if: always() + run: docker image prune -f diff --git a/README.md b/README.md index b7a6239..d7e2999 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,42 @@ docker compose up -d Open `http://localhost:5000`. On first run you'll be prompted to create the admin account. +This compose file builds the image locally and bind-mounts the source for live editing. To run a persistent instance off the published CI image instead, see **Remote dev instance** below. + +--- + +## Remote dev instance + +For a long-lived instance on a remote server that tracks the `dev` CI line, use `compose.deploy.yml`. It pulls the published image (`git.fabledsword.com/bvandeusen/steward:dev`) rather than building, runs no source bind-mounts, and persists data in named volumes. + +**One-time prerequisite — registry push token (on the Git host).** CI publishes the image, so the FabledSteward repo needs a push credential. The injected `GITHUB_TOKEN` lacks `write:package`, so create a Forgejo token scoped **`read:package` + `write:package`** and add it as the repo Actions secret **`REGISTRY_TOKEN`**. Until this exists, the `publish` CI job 401s and no `:dev` image is produced. + +**Standup (on the server):** + +```bash +# 1. Authenticate to the registry (read:package token is enough to pull) +docker login git.fabledsword.com -u + +# 2. Configure secrets +cp .env.example .env +# Set POSTGRES_PASSWORD. Leave STEWARD_SECRET_KEY unset to let the app +# generate + persist one on the /data volume. + +# 3. Pull + boot +docker compose -f compose.deploy.yml up -d +``` + +Open `http://:5000` and create the admin account on first run. + +**Update to the latest dev build:** + +```bash +docker compose -f compose.deploy.yml pull +docker compose -f compose.deploy.yml up -d +``` + +Every push to `dev` that goes CI-green publishes a fresh `:dev` (and an immutable `:` for rollback). To pin a specific commit, change the `image:` tag in `compose.deploy.yml` to `:`. + --- ## Quick Start — Bare Metal diff --git a/ci-requirements.md b/ci-requirements.md index 01a73e4..86c04bd 100644 --- a/ci-requirements.md +++ b/ci-requirements.md @@ -14,8 +14,12 @@ git.fabledsword.com/bvandeusen/ci-python:3.14 ## Per-job tool installs -- `uv pip install --system -e '.[dev]'` (pip fallback) — in `unit` and `integration` - jobs. Steward declares its deps in `pyproject.toml`; the `[dev]` extra adds +- `uv pip install --system -e '.[dev]'` (pip fallback) — in the `unit` job. +- `uv pip install --system -e '.[dev,ansible]'` (pip fallback) — in the + `integration` job, which boots the real app and exercises the Ansible runner, + so it needs the `ansible` extra (`ansible>=10,<13`) at import time. + + Steward declares its deps in `pyproject.toml`; the `[dev]` extra adds `pytest` + `pytest-asyncio`. No `requirements.txt` is tracked. ## Lanes @@ -30,6 +34,13 @@ git.fabledsword.com/bvandeusen/ci-python:3.14 (`create_app(testing=False)`), which runs core + every bundled plugin's migrations, then round-trips the DB. This is the guard that the folded-in plugin migration graph resolves. +- **publish** — builds the runtime `Dockerfile` and pushes to the package + registry. Runs in `ci-builder:latest` (docker + buildx, socket auto-mounted), + gated on `needs: [lint, unit, integration]` so a red commit never ships an + image. Publishes `git.fabledsword.com/bvandeusen/steward:` always, plus + `:dev` on `dev` and `:latest` on `main` (FabledRulebook rule 46). Needs the + `REGISTRY_TOKEN` repo Actions secret (`read:package` + `write:package`) — + the injected `GITHUB_TOKEN` can't push packages. ## Notes diff --git a/compose.deploy.yml b/compose.deploy.yml new file mode 100644 index 0000000..2e19a38 --- /dev/null +++ b/compose.deploy.yml @@ -0,0 +1,53 @@ +# Remote deployment — pulls the published :dev image instead of building locally. +# +# This is NOT the local-dev compose. `docker-compose.yml` bind-mounts ./steward +# + ./plugins and runs --debug for live editing; THIS file runs the image as the +# source of truth (no bind-mounts, no reloader) for a persistent remote instance +# that tracks the dev CI line. +# +# Standup + update runbook: see README → "Remote dev instance". + +services: + steward: + image: git.fabledsword.com/bvandeusen/steward:dev + container_name: steward + # Re-pull :dev on every `up` so restarting the stack picks up the latest + # green dev build without a manual `docker pull`. + pull_policy: always + ports: + - "5000:5000" + volumes: + # /data holds the auto-generated secret key, third-party plugins, and the + # Ansible playbook cache — all of which must survive image updates. No + # source bind-mounts: core + first-party plugins live inside the image. + - app_data:/data + environment: + - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:${POSTGRES_PASSWORD:-steward}@db/steward + # Optional: if unset, the app generates a key and persists it to + # /data/secret.key, so sessions survive restarts via the app_data volume. + # Set STEWARD_SECRET_KEY in .env to pin it explicitly across volume resets. + - STEWARD_SECRET_KEY=${STEWARD_SECRET_KEY:-} + depends_on: + db: + condition: service_healthy + restart: unless-stopped + + db: + image: postgres:16-alpine + environment: + POSTGRES_USER: steward + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-steward} + POSTGRES_DB: steward + volumes: + - pgdata:/var/lib/postgresql/data + # No host port published — Postgres is reachable only on the compose network. + healthcheck: + test: ["CMD-SHELL", "pg_isready -U steward"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + +volumes: + pgdata: + app_data: From 42d9a7ba07c04d1e0ac1bc90c9548da6cf070ffc Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 11:24:29 -0400 Subject: [PATCH 017/126] fix(deploy): move POSTGRES_PASSWORD default into .env.example Plain ${POSTGRES_PASSWORD} in compose; default 'steward' lives in .env.example where it's visible and editable rather than buried in compose YAML. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 2 +- compose.deploy.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 84f9d65..95dcf31 100644 --- a/.env.example +++ b/.env.example @@ -9,4 +9,4 @@ STEWARD_SMTP__PASSWORD= # Password for the bundled Postgres; the steward service builds its # STEWARD_DATABASE_URL from it. Leave STEWARD_SECRET_KEY above unset to let the # app generate + persist one on the /data volume. -POSTGRES_PASSWORD=change-me +POSTGRES_PASSWORD=steward diff --git a/compose.deploy.yml b/compose.deploy.yml index 2e19a38..06109f1 100644 --- a/compose.deploy.yml +++ b/compose.deploy.yml @@ -22,7 +22,7 @@ services: # source bind-mounts: core + first-party plugins live inside the image. - app_data:/data environment: - - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:${POSTGRES_PASSWORD:-steward}@db/steward + - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:${POSTGRES_PASSWORD}@db/steward # Optional: if unset, the app generates a key and persists it to # /data/secret.key, so sessions survive restarts via the app_data volume. # Set STEWARD_SECRET_KEY in .env to pin it explicitly across volume resets. @@ -36,7 +36,7 @@ services: image: postgres:16-alpine environment: POSTGRES_USER: steward - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-steward} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_DB: steward volumes: - pgdata:/var/lib/postgresql/data From ef6c90c022704fa16b4ca086ee909b81ad28695b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 11:25:31 -0400 Subject: [PATCH 018/126] Revert "fix(deploy): move POSTGRES_PASSWORD default into .env.example" This reverts commit 42d9a7ba07c04d1e0ac1bc90c9548da6cf070ffc. --- .env.example | 2 +- compose.deploy.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 95dcf31..84f9d65 100644 --- a/.env.example +++ b/.env.example @@ -9,4 +9,4 @@ STEWARD_SMTP__PASSWORD= # Password for the bundled Postgres; the steward service builds its # STEWARD_DATABASE_URL from it. Leave STEWARD_SECRET_KEY above unset to let the # app generate + persist one on the /data volume. -POSTGRES_PASSWORD=steward +POSTGRES_PASSWORD=change-me diff --git a/compose.deploy.yml b/compose.deploy.yml index 06109f1..2e19a38 100644 --- a/compose.deploy.yml +++ b/compose.deploy.yml @@ -22,7 +22,7 @@ services: # source bind-mounts: core + first-party plugins live inside the image. - app_data:/data environment: - - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:${POSTGRES_PASSWORD}@db/steward + - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:${POSTGRES_PASSWORD:-steward}@db/steward # Optional: if unset, the app generates a key and persists it to # /data/secret.key, so sessions survive restarts via the app_data volume. # Set STEWARD_SECRET_KEY in .env to pin it explicitly across volume resets. @@ -36,7 +36,7 @@ services: image: postgres:16-alpine environment: POSTGRES_USER: steward - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-steward} POSTGRES_DB: steward volumes: - pgdata:/var/lib/postgresql/data From 98edb12abbc5eee7cdf9b4d9fe61cfdd146d72dd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 11:28:53 -0400 Subject: [PATCH 019/126] =?UTF-8?q?fix(deploy):=20let=20app=20own=20secret?= =?UTF-8?q?=20key=20=E2=80=94=20remove=20STEWARD=5FSECRET=5FKEY=20from=20d?= =?UTF-8?q?eploy=20compose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The app auto-generates a key on first boot and persists it to /data/secret.key, which the app_data volume keeps across image updates. No reason to surface this as a user-facing env var in the deploy compose. Removed from .env.example too. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 1 - compose.deploy.yml | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 84f9d65..f5201bf 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,6 @@ # Copy to .env for Docker / local development secrets # These override values in config.yaml -STEWARD_SECRET_KEY=change-me STEWARD_DATABASE_URL=postgresql+asyncpg://steward:password@localhost/steward STEWARD_SMTP__PASSWORD= diff --git a/compose.deploy.yml b/compose.deploy.yml index 2e19a38..175e94a 100644 --- a/compose.deploy.yml +++ b/compose.deploy.yml @@ -23,10 +23,9 @@ services: - app_data:/data environment: - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:${POSTGRES_PASSWORD:-steward}@db/steward - # Optional: if unset, the app generates a key and persists it to - # /data/secret.key, so sessions survive restarts via the app_data volume. - # Set STEWARD_SECRET_KEY in .env to pin it explicitly across volume resets. - - STEWARD_SECRET_KEY=${STEWARD_SECRET_KEY:-} + # STEWARD_SECRET_KEY is intentionally absent — the app generates a random + # key on first boot and persists it to /data/secret.key, which the app_data + # volume keeps across image updates. depends_on: db: condition: service_healthy From 06dc5073ca0f162d089fbb64bab9e6bf3646cfdc Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 11:37:10 -0400 Subject: [PATCH 020/126] =?UTF-8?q?fix(deploy):=20explicit=20backend=20net?= =?UTF-8?q?work=20for=20steward=E2=86=94db=20traffic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- compose.deploy.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/compose.deploy.yml b/compose.deploy.yml index 175e94a..bb08e55 100644 --- a/compose.deploy.yml +++ b/compose.deploy.yml @@ -16,6 +16,8 @@ services: pull_policy: always ports: - "5000:5000" + networks: + - backend volumes: # /data holds the auto-generated secret key, third-party plugins, and the # Ansible playbook cache — all of which must survive image updates. No @@ -33,13 +35,15 @@ services: db: image: postgres:16-alpine + networks: + - backend environment: POSTGRES_USER: steward POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-steward} POSTGRES_DB: steward volumes: - pgdata:/var/lib/postgresql/data - # No host port published — Postgres is reachable only on the compose network. + # No host port published — db is reachable only on the backend network. healthcheck: test: ["CMD-SHELL", "pg_isready -U steward"] interval: 5s @@ -47,6 +51,9 @@ services: retries: 5 restart: unless-stopped +networks: + backend: + volumes: pgdata: app_data: From eb319d715e08e036deec6f36ceb0da78df87a0a5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 18:19:17 -0400 Subject: [PATCH 021/126] docs: Ansible inventory infrastructure implementation plan --- .../plans/2026-06-05-ansible-inventory.md | 2071 +++++++++++++++++ 1 file changed, 2071 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-05-ansible-inventory.md diff --git a/docs/superpowers/plans/2026-06-05-ansible-inventory.md b/docs/superpowers/plans/2026-06-05-ansible-inventory.md new file mode 100644 index 0000000..e0aed90 --- /dev/null +++ b/docs/superpowers/plans/2026-06-05-ansible-inventory.md @@ -0,0 +1,2071 @@ +# Ansible Inventory Infrastructure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add AnsibleTarget + AnsibleGroup models with M2M membership, DB-generated `--list` inventory, per-source PAT auth via GIT_ASKPASS, and a full CRUD UI so Ansible runs are driven by Steward's own inventory records rather than git-repo inventory files. + +**Architecture:** Three new DB tables (`ansible_targets`, `ansible_groups`, `ansible_target_groups`) plus one new column (`ansible_runs.inventory_scope`) added via Alembic migrations. A pure `generate_inventory()` function in `inventory_gen.py` converts ORM objects to the Ansible `--list` JSON format. The existing `browse.html` run form gains a scope picker that replaces the repo-inventory select; `create_run()` fetches scope targets and passes the generated JSON as `inventory_content` to the existing executor. A new `inventory_routes.py` blueprint serves `/ansible/inventory/` CRUD pages for groups and targets. + +**Tech Stack:** Python 3.11+, SQLAlchemy (async/asyncpg), Alembic migrations, Quart, Jinja2 templates (no JS framework beyond existing HTMX), PyYAML (already in project via Ansible dependency). + +--- + +## File Map + +| Path | Action | Responsibility | +|------|--------|----------------| +| `steward/migrations/versions/0015_ansible_inventory_targets.py` | Create | Alembic migration: `ansible_targets` table | +| `steward/migrations/versions/0016_ansible_inventory_groups.py` | Create | Alembic migration: `ansible_groups` + `ansible_target_groups` tables | +| `steward/migrations/versions/0017_ansible_run_scope.py` | Create | Alembic migration: `inventory_scope` column + make `inventory_path` nullable | +| `steward/models/ansible_inventory.py` | Create | `AnsibleTarget`, `AnsibleGroup`, `ansible_target_groups` association table | +| `steward/models/__init__.py` | Modify | Import/export new models so Alembic discovers them | +| `steward/models/ansible.py` | Modify | Add `inventory_scope` field + make `inventory_path` optional on `AnsibleRun` | +| `steward/ansible/inventory_gen.py` | Create | Pure `generate_inventory(targets)` + async `fetch_scope_targets(db, scope)` | +| `steward/ansible/sources.py` | Modify | Add `http_token` to returned source dict + GIT_ASKPASS in `git_pull()` | +| `steward/ansible/routes.py` | Modify | Update `create_run()` scope handling; update `index()` context | +| `steward/ansible/inventory_routes.py` | Create | Blueprint: CRUD for `/ansible/inventory/groups` + `/ansible/inventory/targets` | +| `steward/app.py` | Modify | Register `inventory_bp` | +| `steward/settings/routes.py` | Modify | Include `http_token` in `ansible_save_source()` + `ansible_add_source()` | +| `steward/templates/ansible/browse.html` | Modify | Replace inventory select with scope picker ` +
+``` + +Inside the `ansible-add-git-fields` div, after the `pull_interval_seconds` form-group, add: + +```html +
+ + +
+``` + +- [ ] **Step 5.8: Commit** + +```bash +git add steward/ansible/sources.py \ + steward/settings/routes.py \ + steward/templates/settings/_ansible_sources.html \ + tests/core/test_git_askpass.py +git commit -m "feat(ansible): GIT_ASKPASS per-source HTTP token auth + settings UI" +``` + +--- + +## Task 6: Wire Inventory Generation into create_run() + +**Files:** +- Modify: `steward/ansible/routes.py` +- Modify: `steward/templates/ansible/browse.html` +- Modify: `steward/templates/ansible/run_detail.html` + +- [ ] **Step 6.1: Update `index()` in `steward/ansible/routes.py`** + +The `index()` route needs to pass `inventory_scope` label context to the run history table (for display). No test needed — it's a display change. + +Change the `index()` route to also fetch target/group counts for displaying in the nav: + +```python +@ansible_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + from sqlalchemy import func + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleRun).order_by(AnsibleRun.started_at.desc()).limit(50) + ) + runs = result.scalars().all() + target_count = (await db.execute(select(func.count()).select_from(AnsibleTarget))).scalar() + return await render_template("ansible/index.html", runs=runs, target_count=target_count) +``` + +- [ ] **Step 6.2: Update `create_run()` to use inventory_scope** + +Replace the existing `create_run()` implementation. The key changes are: +- Accept `inventory_scope` from the form instead of `inventory_path` +- Generate `inventory_content` from DB for `steward:*` scopes +- Store `inventory_scope` on the `AnsibleRun` row + +Full updated function (replaces the existing one at line ~108): + +```python +@ansible_bp.post("/runs") +@require_role(UserRole.operator) +async def create_run(): + import json as _json + from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory + + form = await request.form + playbook_path = form.get("playbook_path", "").strip() + source_name = form.get("source_name", "").strip() + inventory_scope = form.get("inventory_scope", "steward:all").strip() + + if not playbook_path: + return "playbook_path is required", 400 + + all_sources = _get_sources() + source = next((s for s in all_sources if s["name"] == source_name), None) + if source is None: + return "Source not found", 404 + + # Resolve inventory content for DB-backed scopes. + inventory_content: str | None = None + inventory_path: str | None = None + if inventory_scope.startswith("steward:"): + async with current_app.db_sessionmaker() as db: + targets = await fetch_scope_targets(db, inventory_scope) + inventory_content = _json.dumps(generate_inventory(targets)) + elif inventory_scope.startswith("repo:"): + # repo:: + parts = inventory_scope.split(":", 2) + inventory_path = parts[2] if len(parts) == 3 else "" + else: + return "Invalid inventory_scope", 400 + + # Optional run parameters. + extra_vars: list[str] = [] + for line in (form.get("extra_vars", "") or "").splitlines(): + line = line.strip() + if not line: + continue + if "=" not in line: + return f"Invalid extra var (expected key=value): {line!r}", 400 + extra_vars.append(line) + limit = (form.get("limit", "") or "").strip() + tags = (form.get("tags", "") or "").strip() + check = "check" in form + + params: dict = {} + if extra_vars: + params["extra_vars"] = extra_vars + if limit: + params["limit"] = limit + if tags: + params["tags"] = tags + if check: + params["check"] = True + params_or_none = params or None + + run_id = str(uuid.uuid4()) + now = datetime.now(timezone.utc) + + run = AnsibleRun( + id=run_id, + playbook_path=playbook_path, + inventory_path=inventory_path, + inventory_scope=inventory_scope, + source_name=source_name, + triggered_by=session["user_id"], + status=AnsibleRunStatus.running, + started_at=now, + params=params_or_none, + ) + + async with current_app.db_sessionmaker() as db: + async with db.begin(): + db.add(run) + + asyncio.ensure_future( + executor.start_run( + current_app._get_current_object(), # type: ignore[attr-defined] + run_id, + playbook_path=playbook_path, + inventory_path=inventory_path or "", + source_path=source["path"], + params=params_or_none, + inventory_content=inventory_content, + ) + ) + return await render_template("ansible/run_started.html", run_id=run_id) +``` + +- [ ] **Step 6.3: Update `browse()` route to pass scope context** + +The `browse()` route needs to pass targets and groups for the scope picker. Update it: + +```python +@ansible_bp.get("/browse") +@require_role(UserRole.viewer) +async def browse(): + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + + all_sources = _get_sources() + source_data = [] + for source in all_sources: + playbooks = src_module.discover_playbooks(source["path"]) + inventories = src_module.discover_inventories(source["path"]) + source_data.append({ + "source": source, + "playbooks": playbooks, + "inventories": inventories, + }) + + async with current_app.db_sessionmaker() as db: + targets = (await db.execute( + select(AnsibleTarget).order_by(AnsibleTarget.name) + )).scalars().all() + groups = (await db.execute( + select(AnsibleGroup).order_by(AnsibleGroup.name) + )).scalars().all() + + return await render_template( + "ansible/browse.html", + source_data=source_data, + targets=targets, + groups=groups, + ) +``` + +- [ ] **Step 6.4: Update `steward/templates/ansible/browse.html` run form** + +Replace the existing "Inventory" form-group block (lines 73–85) with the scope picker. The existing block is: + +```html +
+ + +
+
+ + +
+``` + +Replace with: + +```html +
+ + +
+``` + +Also update the "Execute" button's onclick — remove the `manual-inv` logic since there's no manual field anymore: + +```html + +``` + +(The form can submit directly now.) + +- [ ] **Step 6.5: Update `run_detail.html` to show scope label** + +In `steward/templates/ansible/run_detail.html`, find where `run.inventory_path` is displayed and add `inventory_scope` display. After the playbook path line, add: + +```html + + Scope + {{ run.inventory_scope }} + +``` + +(Check the existing table structure in `run_detail.html` first — look for `run.playbook_path` and follow the same `` pattern.) + +- [ ] **Step 6.6: Commit** + +```bash +git add steward/ansible/routes.py \ + steward/templates/ansible/browse.html \ + steward/templates/ansible/run_detail.html +git commit -m "feat(ansible): scope picker in run form + create_run() DB inventory generation" +``` + +--- + +## Task 7: Inventory Blueprint — Groups CRUD + +**Files:** +- Create: `steward/ansible/inventory_routes.py` +- Create: `steward/templates/ansible/inventory/groups.html` +- Create: `steward/templates/ansible/inventory/group_detail.html` +- Modify: `steward/app.py` + +- [ ] **Step 7.1: Create `steward/ansible/inventory_routes.py`** + +```python +"""CRUD routes for Ansible inventory: groups and targets.""" +from __future__ import annotations +import uuid +import yaml + +from quart import Blueprint, current_app, redirect, render_template, request, url_for +from sqlalchemy import select +from sqlalchemy.orm import selectinload + +from steward.auth.middleware import require_role +from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget +from steward.models.users import UserRole + +inventory_bp = Blueprint("ansible_inventory", __name__, url_prefix="/ansible/inventory") + + +def _parse_ansible_vars(raw: str) -> dict: + """Parse a YAML/JSON string into a dict. Raises ValueError on invalid input.""" + raw = raw.strip() + if not raw: + return {} + try: + parsed = yaml.safe_load(raw) + except yaml.YAMLError as exc: + raise ValueError(f"Invalid YAML: {exc}") from exc + if parsed is None: + return {} + if not isinstance(parsed, dict): + raise ValueError("ansible_vars must be a YAML mapping (key: value pairs)") + return parsed + + +@inventory_bp.get("/groups") +@require_role(UserRole.viewer) +async def groups_list(): + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleGroup) + .options(selectinload(AnsibleGroup.targets)) + .order_by(AnsibleGroup.name) + ) + groups = result.scalars().all() + return await render_template("ansible/inventory/groups.html", groups=groups) + + +@inventory_bp.post("/groups") +@require_role(UserRole.operator) +async def groups_create(): + form = await request.form + name = form.get("name", "").strip() + if not name: + return "name is required", 400 + try: + ansible_vars = _parse_ansible_vars(form.get("ansible_vars", "")) + except ValueError as exc: + return str(exc), 400 + + group = AnsibleGroup(id=str(uuid.uuid4()), name=name, ansible_vars=ansible_vars) + async with current_app.db_sessionmaker() as db: + async with db.begin(): + db.add(group) + return redirect(url_for("ansible_inventory.groups_list")) + + +@inventory_bp.get("/groups/") +@require_role(UserRole.viewer) +async def group_detail(group_id: str): + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleGroup) + .where(AnsibleGroup.id == group_id) + .options(selectinload(AnsibleGroup.targets)) + ) + group = result.scalar_one_or_none() + if group is None: + return "Group not found", 404 + all_targets = ( + await db.execute(select(AnsibleTarget).order_by(AnsibleTarget.name)) + ).scalars().all() + member_ids = {t.id for t in group.targets} + ansible_vars_yaml = yaml.dump(group.ansible_vars) if group.ansible_vars else "" + return await render_template( + "ansible/inventory/group_detail.html", + group=group, + all_targets=all_targets, + member_ids=member_ids, + ansible_vars_yaml=ansible_vars_yaml, + ) + + +@inventory_bp.post("/groups/") +@require_role(UserRole.operator) +async def group_update(group_id: str): + form = await request.form + try: + ansible_vars = _parse_ansible_vars(form.get("ansible_vars", "")) + except ValueError as exc: + return str(exc), 400 + + async with current_app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute( + select(AnsibleGroup) + .where(AnsibleGroup.id == group_id) + .options(selectinload(AnsibleGroup.targets)) + ) + group = result.scalar_one_or_none() + if group is None: + return "Group not found", 404 + + name = form.get("name", group.name).strip() + if name: + group.name = name + group.ansible_vars = ansible_vars + + # Update member list — form sends target_ids[] checkboxes. + new_ids = set(form.getlist("target_ids")) + all_targets = ( + await db.execute(select(AnsibleTarget)) + ).scalars().all() + target_map = {t.id: t for t in all_targets} + group.targets = [target_map[tid] for tid in new_ids if tid in target_map] + + return redirect(url_for("ansible_inventory.group_detail", group_id=group_id)) + + +@inventory_bp.post("/groups//delete") +@require_role(UserRole.operator) +async def group_delete(group_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute( + select(AnsibleGroup).where(AnsibleGroup.id == group_id) + ) + group = result.scalar_one_or_none() + if group is not None: + await db.delete(group) + return redirect(url_for("ansible_inventory.groups_list")) +``` + +- [ ] **Step 7.2: Create `steward/templates/ansible/inventory/groups.html`** + +```html +{% extends "base.html" %} +{% block title %}Inventory Groups — Steward{% endblock %} +{% block content %} +
+

Inventory Groups

+ Targets +
+ +
+

New Group

+
+
+ + +
+ +
+
+ +{% if not groups %} +
+

No groups yet. Create one above.

+
+{% else %} +
+ + + + + + {% for grp in groups %} + + + + + + {% endfor %} + +
NameMembers
{{ grp.name }}{{ grp.targets | length }} + Edit +
+ +
+
+
+{% endif %} +{% endblock %} +``` + +- [ ] **Step 7.3: Create `steward/templates/ansible/inventory/group_detail.html`** + +```html +{% extends "base.html" %} +{% block title %}Group: {{ group.name }} — Steward{% endblock %} +{% block content %} +
+

Group: {{ group.name }}

+ ← Groups +
+ +
+
+

Settings

+
+ + +
+
+ + +
+
+ +
+

Members

+ {% if not all_targets %} +

No targets exist yet. + Add targets first.

+ {% else %} +
+ {% for tgt in all_targets %} + + {% endfor %} +
+ {% endif %} +
+ + +
+{% endblock %} +``` + +- [ ] **Step 7.4: Register blueprint in `steward/app.py`** + +After the existing `from .ansible.routes import ansible_bp` and `app.register_blueprint(ansible_bp)` lines, add: + +```python +from .ansible.inventory_routes import inventory_bp +app.register_blueprint(inventory_bp) +``` + +(These are inside the `create_app()` function, alongside the other blueprint registrations.) + +- [ ] **Step 7.5: Run the app and verify `/ansible/inventory/groups` loads** + +```bash +cd /home/bvandeusen/Nextcloud/Projects/FabledSteward +docker compose up -d # or start local dev server +# Navigate to http://localhost:5000/ansible/inventory/groups in browser +``` + +Expected: Groups list page renders with "No groups yet" message. No 500 errors. + +- [ ] **Step 7.6: Commit** + +```bash +git add steward/ansible/inventory_routes.py \ + steward/app.py \ + steward/templates/ansible/inventory/groups.html \ + steward/templates/ansible/inventory/group_detail.html +git commit -m "feat(ansible): inventory groups CRUD routes + templates" +``` + +--- + +## Task 8: Inventory Blueprint — Targets CRUD + +**Files:** +- Modify: `steward/ansible/inventory_routes.py` +- Create: `steward/templates/ansible/inventory/targets.html` +- Create: `steward/templates/ansible/inventory/target_detail.html` + +- [ ] **Step 8.1: Add targets routes to `steward/ansible/inventory_routes.py`** + +Append to the existing `inventory_routes.py` file (after the group routes): + +```python +@inventory_bp.get("/targets") +@require_role(UserRole.viewer) +async def targets_list(): + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleTarget) + .options(selectinload(AnsibleTarget.groups)) + .order_by(AnsibleTarget.name) + ) + targets = result.scalars().all() + return await render_template("ansible/inventory/targets.html", targets=targets) + + +@inventory_bp.post("/targets") +@require_role(UserRole.operator) +async def targets_create(): + form = await request.form + name = form.get("name", "").strip() + address = form.get("address", "").strip() + if not name or not address: + return "name and address are required", 400 + try: + ansible_vars = _parse_ansible_vars(form.get("ansible_vars", "")) + except ValueError as exc: + return str(exc), 400 + + target = AnsibleTarget( + id=str(uuid.uuid4()), name=name, address=address, ansible_vars=ansible_vars + ) + async with current_app.db_sessionmaker() as db: + async with db.begin(): + db.add(target) + return redirect(url_for("ansible_inventory.targets_list")) + + +@inventory_bp.get("/targets/") +@require_role(UserRole.viewer) +async def target_detail(target_id: str): + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(AnsibleTarget) + .where(AnsibleTarget.id == target_id) + .options(selectinload(AnsibleTarget.groups)) + ) + target = result.scalar_one_or_none() + if target is None: + return "Target not found", 404 + all_groups = ( + await db.execute(select(AnsibleGroup).order_by(AnsibleGroup.name)) + ).scalars().all() + member_group_ids = {g.id for g in target.groups} + ansible_vars_yaml = yaml.dump(target.ansible_vars) if target.ansible_vars else "" + return await render_template( + "ansible/inventory/target_detail.html", + target=target, + all_groups=all_groups, + member_group_ids=member_group_ids, + ansible_vars_yaml=ansible_vars_yaml, + ) + + +@inventory_bp.post("/targets/") +@require_role(UserRole.operator) +async def target_update(target_id: str): + form = await request.form + try: + ansible_vars = _parse_ansible_vars(form.get("ansible_vars", "")) + except ValueError as exc: + return str(exc), 400 + + async with current_app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute( + select(AnsibleTarget) + .where(AnsibleTarget.id == target_id) + .options(selectinload(AnsibleTarget.groups)) + ) + target = result.scalar_one_or_none() + if target is None: + return "Target not found", 404 + + name = form.get("name", target.name).strip() + address = form.get("address", target.address).strip() + if name: + target.name = name + if address: + target.address = address + target.ansible_vars = ansible_vars + + new_group_ids = set(form.getlist("group_ids")) + all_groups_result = await db.execute(select(AnsibleGroup)) + group_map = {g.id: g for g in all_groups_result.scalars().all()} + target.groups = [group_map[gid] for gid in new_group_ids if gid in group_map] + + return redirect(url_for("ansible_inventory.target_detail", target_id=target_id)) + + +@inventory_bp.post("/targets//delete") +@require_role(UserRole.operator) +async def target_delete(target_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute( + select(AnsibleTarget).where(AnsibleTarget.id == target_id) + ) + target = result.scalar_one_or_none() + if target is not None: + await db.delete(target) + return redirect(url_for("ansible_inventory.targets_list")) +``` + +- [ ] **Step 8.2: Create `steward/templates/ansible/inventory/targets.html`** + +```html +{% extends "base.html" %} +{% block title %}Inventory Targets — Steward{% endblock %} +{% block content %} +
+

Inventory Targets

+ Groups +
+ +
+

New Target

+
+
+ + +
+
+ + +
+ +
+
+ +{% if not targets %} +
+

No targets yet. Add one above.

+
+{% else %} +
+ + + + + + {% for tgt in targets %} + + + + + + + {% endfor %} + +
NameAddressGroups
{{ tgt.name }}{{ tgt.address }} + {% for grp in tgt.groups %} + {{ grp.name }} + {% endfor %} + + Edit +
+ +
+
+
+{% endif %} +{% endblock %} +``` + +- [ ] **Step 8.3: Create `steward/templates/ansible/inventory/target_detail.html`** + +```html +{% extends "base.html" %} +{% block title %}Target: {{ target.name }} — Steward{% endblock %} +{% block content %} +
+

Target: {{ target.name }}

+ ← Targets +
+ +
+
+

Connection

+
+ + +
+
+ + +
+
+ + +
+
+ +
+

Groups

+ {% if not all_groups %} +

No groups exist yet. + Create a group first.

+ {% else %} +
+ {% for grp in all_groups %} + + {% endfor %} +
+ {% endif %} +
+ + +
+{% endblock %} +``` + +- [ ] **Step 8.4: Verify targets CRUD works** + +Navigate to `http://localhost:5000/ansible/inventory/targets` in browser. Create a target, edit it, assign it to a group, verify it appears in the groups list as a member. + +- [ ] **Step 8.5: Commit** + +```bash +git add steward/ansible/inventory_routes.py \ + steward/templates/ansible/inventory/targets.html \ + steward/templates/ansible/inventory/target_detail.html +git commit -m "feat(ansible): inventory targets CRUD routes + templates" +``` + +--- + +## Task 9: Host Edit Form — Ansible Target Link Section + +**Files:** +- Modify: `steward/templates/hosts/form.html` +- Modify: `steward/hosts/routes.py` + +- [ ] **Step 9.1: Update `edit_host()` route in `steward/hosts/routes.py`** + +The `edit_host()` function currently (around line ~149) renders `hosts/form.html` with just the host. Extend it to also pass the linked Ansible target and all available targets: + +```python +@hosts_bp.get("//edit") +@require_role(UserRole.operator) +async def edit_host(host_id: str): + from steward.models.ansible_inventory import AnsibleTarget + from sqlalchemy.orm import selectinload + + async with current_app.db_sessionmaker() as db: + result = await db.execute( + select(Host).where(Host.id == host_id) + ) + host = result.scalar_one_or_none() + if host is None: + return "Host not found", 404 + + linked_target = None + linked_result = await db.execute( + select(AnsibleTarget) + .where(AnsibleTarget.host_id == host_id) + .options(selectinload(AnsibleTarget.groups)) + ) + linked_target = linked_result.scalar_one_or_none() + + all_targets = ( + await db.execute( + select(AnsibleTarget) + .where(AnsibleTarget.host_id.is_(None)) + .order_by(AnsibleTarget.name) + ) + ).scalars().all() + + return await render_template( + "hosts/form.html", + host=host, + probe_types=list(ProbeType), + linked_target=linked_target, + linkable_targets=all_targets, + ) +``` + +Note: `all_targets` shows only unlinked targets (those without a `host_id`) plus the currently linked target if any — so the user can re-select it. The form template handles this. + +Also add a new route `POST //ansible-link` to handle the link action: + +```python +@hosts_bp.post("//ansible-link") +@require_role(UserRole.operator) +async def ansible_link(host_id: str): + from steward.models.ansible_inventory import AnsibleTarget + + form = await request.form + action = form.get("action", "") + + async with current_app.db_sessionmaker() as db: + async with db.begin(): + if action == "unlink": + # Clear host_id from the currently linked target. + linked_result = await db.execute( + select(AnsibleTarget).where(AnsibleTarget.host_id == host_id) + ) + linked = linked_result.scalar_one_or_none() + if linked: + linked.host_id = None + elif action == "link": + target_id = form.get("target_id", "").strip() + if target_id: + result = await db.execute( + select(AnsibleTarget).where(AnsibleTarget.id == target_id) + ) + target = result.scalar_one_or_none() + if target: + # Clear any previous link for this host. + old_result = await db.execute( + select(AnsibleTarget).where(AnsibleTarget.host_id == host_id) + ) + old = old_result.scalar_one_or_none() + if old and old.id != target_id: + old.host_id = None + target.host_id = host_id + elif action == "create": + # Create a new target pre-filled from host data. + host_result = await db.execute( + select(Host).where(Host.id == host_id) + ) + host = host_result.scalar_one_or_none() + if host: + new_target = AnsibleTarget( + id=str(uuid.uuid4()), + name=host.name, + address=host.address, + host_id=host_id, + ) + db.add(new_target) + + return redirect(f"/hosts/{host_id}/edit") +``` + +Add `import uuid` to the top of `hosts/routes.py` if not already present. Also add `from sqlalchemy import select` and `from steward.models.hosts import Host, ProbeType` imports (check what's already imported). + +- [ ] **Step 9.2: Add Ansible section to `steward/templates/hosts/form.html`** + +At the end of the file, before `{% endblock %}`, add: + +```html +{% if host %} +
+

Ansible Target

+ {% if linked_target %} +
+
+ {{ linked_target.name }} + {{ linked_target.address }} + {% if linked_target.groups %} +
+ {% for grp in linked_target.groups %} + {{ grp.name }} + {% endfor %} +
+ {% endif %} +
+ Edit Target +
+ + +
+
+ {% else %} +

+ No Ansible target linked. Link an existing target or create a new one from this host's data. +

+
+ {% if linkable_targets %} +
+ + + +
+ {% endif %} +
+ + +
+
+ {% endif %} +
+{% endif %} +``` + +- [ ] **Step 9.3: Verify the Ansible section appears on host edit** + +Navigate to `http://localhost:5000/hosts//edit`. Verify the Ansible Target section appears at the bottom. Create a target from a host, verify it links and the target appears. Unlink and verify it clears. + +- [ ] **Step 9.4: Commit** + +```bash +git add steward/hosts/routes.py steward/templates/hosts/form.html +git commit -m "feat(ansible): host edit page Ansible target link/create/unlink section" +``` + +--- + +## Task 10: Integration Test — Migrations + fetch_scope_targets + +**Files:** +- Create: `tests/integration/test_ansible_inventory.py` + +- [ ] **Step 10.1: Write integration tests** + +Create `tests/integration/test_ansible_inventory.py`: + +```python +"""Integration tests: ansible_targets / ansible_groups tables + fetch_scope_targets. + +Requires a live Postgres DB. Run with: + pytest tests/integration/test_ansible_inventory.py -m integration -v +""" +import pytest +import pytest_asyncio +from sqlalchemy import select, text + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_ansible_targets_table_exists(app): + """Migration 0015 created the ansible_targets table.""" + async with app.db_sessionmaker() as db: + result = await db.execute( + text("SELECT column_name FROM information_schema.columns WHERE table_name = 'ansible_targets'") + ) + columns = {row[0] for row in result.fetchall()} + assert "id" in columns + assert "name" in columns + assert "address" in columns + assert "ansible_vars" in columns + assert "host_id" in columns + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_ansible_groups_table_exists(app): + """Migration 0016 created ansible_groups and ansible_target_groups.""" + async with app.db_sessionmaker() as db: + result = await db.execute( + text("SELECT column_name FROM information_schema.columns WHERE table_name = 'ansible_groups'") + ) + columns = {row[0] for row in result.fetchall()} + assert "id" in columns + assert "name" in columns + assert "ansible_vars" in columns + + async with app.db_sessionmaker() as db: + result = await db.execute( + text("SELECT column_name FROM information_schema.columns WHERE table_name = 'ansible_target_groups'") + ) + join_columns = {row[0] for row in result.fetchall()} + assert "target_id" in join_columns + assert "group_id" in join_columns + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_ansible_run_scope_column_exists(app): + """Migration 0017 added inventory_scope to ansible_runs.""" + async with app.db_sessionmaker() as db: + result = await db.execute( + text("SELECT column_name FROM information_schema.columns WHERE table_name = 'ansible_runs'") + ) + columns = {row[0] for row in result.fetchall()} + assert "inventory_scope" in columns + assert "inventory_path" in columns # still present, now nullable + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_create_target_and_group_and_fetch(app): + """Can create a target+group, assign membership, and fetch via fetch_scope_targets.""" + import uuid + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory + + target_id = str(uuid.uuid4()) + group_id = str(uuid.uuid4()) + + async with app.db_sessionmaker() as db: + async with db.begin(): + grp = AnsibleGroup( + id=group_id, + name=f"test-group-{group_id[:8]}", + ansible_vars={"env": "test"}, + ) + tgt = AnsibleTarget( + id=target_id, + name=f"test-target-{target_id[:8]}", + address="10.0.99.1", + ansible_vars={"ansible_user": "ubuntu"}, + ) + tgt.groups.append(grp) + db.add(grp) + db.add(tgt) + + # Fetch all + async with app.db_sessionmaker() as db: + all_targets = await fetch_scope_targets(db, "steward:all") + names = [t.name for t in all_targets] + assert f"test-target-{target_id[:8]}" in names + + # Fetch by group + async with app.db_sessionmaker() as db: + group_targets = await fetch_scope_targets(db, f"steward:group:{group_id}") + assert any(t.id == target_id for t in group_targets) + + # Fetch single target + async with app.db_sessionmaker() as db: + single = await fetch_scope_targets(db, f"steward:target:{target_id}") + assert len(single) == 1 + assert single[0].id == target_id + + # Generate inventory + async with app.db_sessionmaker() as db: + targets = await fetch_scope_targets(db, f"steward:group:{group_id}") + inv = generate_inventory(targets) + target_name = f"test-target-{target_id[:8]}" + group_name = f"test-group-{group_id[:8]}" + assert target_name in inv["all"]["hosts"] + assert target_name in inv[group_name]["hosts"] + assert inv["_meta"]["hostvars"][target_name]["ansible_host"] == "10.0.99.1" + assert inv["_meta"]["hostvars"][target_name]["ansible_user"] == "ubuntu" + assert inv["_meta"]["hostvars"][target_name]["env"] == "test" + + # Cleanup + async with app.db_sessionmaker() as db: + async with db.begin(): + result = await db.execute(select(AnsibleTarget).where(AnsibleTarget.id == target_id)) + t = result.scalar_one_or_none() + if t: + await db.delete(t) + result = await db.execute(select(AnsibleGroup).where(AnsibleGroup.id == group_id)) + g = result.scalar_one_or_none() + if g: + await db.delete(g) +``` + +- [ ] **Step 10.2: Run integration tests** + +```bash +pytest tests/integration/test_ansible_inventory.py -m integration -v +``` + +Expected: 4 PASSED (requires running Postgres with migrations applied). + +- [ ] **Step 10.3: Commit** + +```bash +git add tests/integration/test_ansible_inventory.py +git commit -m "test(ansible): integration tests for inventory tables + fetch_scope_targets" +``` + +--- + +## Self-Review Checklist + +- [x] **Spec coverage**: All 8 spec sections are covered: + - §1 Data Model → Tasks 1–3 + - §2 PAT/HTTP Token → Task 5 + - §3 Inventory Generation → Task 4 + - §4 Run Form → Task 6 + - §5 UI Surfaces → Tasks 7–9 + - §6 Repo integration → Task 6 (scope fallback preserved) + - §7 Migrations → Tasks 1–3 + - §8 Out of scope → not implemented ✓ +- [x] **No placeholders**: All code blocks are complete. +- [x] **Type consistency**: `AnsibleTarget`, `AnsibleGroup`, `ansible_target_groups` defined in Task 1 and referenced consistently throughout. `fetch_scope_targets(db, scope)` / `generate_inventory(targets)` defined in Task 4 and called in Task 6. +- [x] **Migration chain**: `0015 → 0016 → 0017`, each `down_revision` points to the previous. +- [x] **Session pattern**: All routes use `async with current_app.db_sessionmaker() as db:` matching existing code. +- [x] **Import pattern**: New models added to `steward/models/__init__.py` for Alembic discovery. +- [x] **`get_sources()` http_token**: Added for both git and local sources (empty string for local). From c11b3f01ed5f3257736e9a6d9acd5246e52b6131 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 18:45:00 -0400 Subject: [PATCH 022/126] feat(ansible): AnsibleTarget model + 0015_ansible_inventory_targets migration --- .../0015_ansible_inventory_targets.py | 41 +++++++++++ steward/models/__init__.py | 2 + steward/models/ansible_inventory.py | 69 +++++++++++++++++++ 3 files changed, 112 insertions(+) create mode 100644 steward/migrations/versions/0015_ansible_inventory_targets.py create mode 100644 steward/models/ansible_inventory.py diff --git a/steward/migrations/versions/0015_ansible_inventory_targets.py b/steward/migrations/versions/0015_ansible_inventory_targets.py new file mode 100644 index 0000000..2957770 --- /dev/null +++ b/steward/migrations/versions/0015_ansible_inventory_targets.py @@ -0,0 +1,41 @@ +"""Add ansible_targets table + +Revision ID: 0015_ansible_inventory_targets +Revises: 0014_alert_ansible_action +Create Date: 2026-06-05 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "0015_ansible_inventory_targets" +down_revision: Union[str, None] = "0014_alert_ansible_action" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "ansible_targets", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("name", sa.String(128), nullable=False), + sa.Column("address", sa.String(255), nullable=False), + sa.Column("ansible_vars", sa.JSON(), nullable=False, server_default="{}"), + sa.Column( + "host_id", + sa.String(36), + sa.ForeignKey("hosts.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.UniqueConstraint("name", name="uq_ansible_targets_name"), + ) + + +def downgrade() -> None: + op.drop_table("ansible_targets") diff --git a/steward/models/__init__.py b/steward/models/__init__.py index cdc8272..d8502b5 100644 --- a/steward/models/__init__.py +++ b/steward/models/__init__.py @@ -5,6 +5,7 @@ from .monitors import PingResult, DnsResult, PingStatus, DnsStatus from .metrics import PluginMetric from .alerts import AlertRule, AlertState, AlertEvent, AlertOperator, AlertStateEnum from .ansible import AnsibleRun, AnsibleRunStatus +from .ansible_inventory import AnsibleTarget, AnsibleGroup, ansible_target_groups from .settings import AppSetting from .dashboard import Dashboard, DashboardWidget, DashboardShareToken @@ -15,6 +16,7 @@ __all__ = [ "PluginMetric", "AlertRule", "AlertState", "AlertEvent", "AlertOperator", "AlertStateEnum", "AnsibleRun", "AnsibleRunStatus", + "AnsibleTarget", "AnsibleGroup", "ansible_target_groups", "AppSetting", "Dashboard", "DashboardWidget", "DashboardShareToken", ] diff --git a/steward/models/ansible_inventory.py b/steward/models/ansible_inventory.py new file mode 100644 index 0000000..9c7bff8 --- /dev/null +++ b/steward/models/ansible_inventory.py @@ -0,0 +1,69 @@ +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import Column, DateTime, ForeignKey, JSON, String, Table +from sqlalchemy.orm import Mapped, mapped_column, relationship +from .base import Base + +ansible_target_groups = Table( + "ansible_target_groups", + Base.metadata, + Column( + "target_id", + String(36), + ForeignKey("ansible_targets.id", ondelete="CASCADE"), + primary_key=True, + ), + Column( + "group_id", + String(36), + ForeignKey("ansible_groups.id", ondelete="CASCADE"), + primary_key=True, + ), +) + + +class AnsibleTarget(Base): + __tablename__ = "ansible_targets" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + name: Mapped[str] = mapped_column(String(128), nullable=False, unique=True) + address: Mapped[str] = mapped_column(String(255), nullable=False) + ansible_vars: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + host_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="SET NULL"), nullable=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + groups: Mapped[list["AnsibleGroup"]] = relationship( + "AnsibleGroup", + secondary=ansible_target_groups, + back_populates="targets", + lazy="select", + ) + + +class AnsibleGroup(Base): + __tablename__ = "ansible_groups" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + name: Mapped[str] = mapped_column(String(128), nullable=False, unique=True) + ansible_vars: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + targets: Mapped[list["AnsibleTarget"]] = relationship( + "AnsibleTarget", + secondary=ansible_target_groups, + back_populates="groups", + lazy="select", + ) From fa32488fdf565166eaa5b8ec0c156a710d9c052e Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 18:45:16 -0400 Subject: [PATCH 023/126] =?UTF-8?q?feat(ansible):=20AnsibleGroup=20+=20joi?= =?UTF-8?q?n=20table=20=E2=80=94=200016=5Fansible=5Finventory=5Fgroups=20m?= =?UTF-8?q?igration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../versions/0016_ansible_inventory_groups.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 steward/migrations/versions/0016_ansible_inventory_groups.py diff --git a/steward/migrations/versions/0016_ansible_inventory_groups.py b/steward/migrations/versions/0016_ansible_inventory_groups.py new file mode 100644 index 0000000..76c5b03 --- /dev/null +++ b/steward/migrations/versions/0016_ansible_inventory_groups.py @@ -0,0 +1,51 @@ +"""Add ansible_groups and ansible_target_groups tables + +Revision ID: 0016_ansible_inventory_groups +Revises: 0015_ansible_inventory_targets +Create Date: 2026-06-05 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "0016_ansible_inventory_groups" +down_revision: Union[str, None] = "0015_ansible_inventory_targets" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "ansible_groups", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("name", sa.String(128), nullable=False), + sa.Column("ansible_vars", sa.JSON(), nullable=False, server_default="{}"), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.UniqueConstraint("name", name="uq_ansible_groups_name"), + ) + op.create_table( + "ansible_target_groups", + sa.Column( + "target_id", + sa.String(36), + sa.ForeignKey("ansible_targets.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "group_id", + sa.String(36), + sa.ForeignKey("ansible_groups.id", ondelete="CASCADE"), + nullable=False, + ), + sa.PrimaryKeyConstraint("target_id", "group_id"), + ) + + +def downgrade() -> None: + op.drop_table("ansible_target_groups") + op.drop_table("ansible_groups") From 10df05c13ee567c8d262740a0a8093bbd6c0a8d6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 18:45:38 -0400 Subject: [PATCH 024/126] =?UTF-8?q?feat(ansible):=20add=20inventory=5Fscop?= =?UTF-8?q?e=20to=20AnsibleRun=20=E2=80=94=200017=5Fansible=5Frun=5Fscope?= =?UTF-8?q?=20migration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../versions/0017_ansible_run_scope.py | 41 +++++++++++++++++++ steward/models/ansible.py | 5 ++- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 steward/migrations/versions/0017_ansible_run_scope.py diff --git a/steward/migrations/versions/0017_ansible_run_scope.py b/steward/migrations/versions/0017_ansible_run_scope.py new file mode 100644 index 0000000..765f71b --- /dev/null +++ b/steward/migrations/versions/0017_ansible_run_scope.py @@ -0,0 +1,41 @@ +"""Add inventory_scope to ansible_runs; make inventory_path nullable; backfill scope + +Revision ID: 0017_ansible_run_scope +Revises: 0016_ansible_inventory_groups +Create Date: 2026-06-05 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "0017_ansible_run_scope" +down_revision: Union[str, None] = "0016_ansible_inventory_groups" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "ansible_runs", + sa.Column("inventory_scope", sa.String(255), nullable=True), + ) + # Backfill: existing rows used repo inventory files — reconstruct the legacy scope string. + op.execute( + """ + UPDATE ansible_runs + SET inventory_scope = 'repo:' || source_name || ':' || inventory_path + WHERE inventory_path IS NOT NULL AND inventory_path != '' + """ + ) + # Rows with empty inventory_path get 'steward:all' + op.execute( + "UPDATE ansible_runs SET inventory_scope = 'steward:all' WHERE inventory_scope IS NULL" + ) + op.alter_column("ansible_runs", "inventory_scope", nullable=False) + # Make inventory_path nullable — new steward:* runs won't have a file path. + op.alter_column("ansible_runs", "inventory_path", nullable=True) + + +def downgrade() -> None: + op.drop_column("ansible_runs", "inventory_scope") + op.alter_column("ansible_runs", "inventory_path", nullable=False) diff --git a/steward/models/ansible.py b/steward/models/ansible.py index 2adf06e..36731dc 100644 --- a/steward/models/ansible.py +++ b/steward/models/ansible.py @@ -19,7 +19,10 @@ class AnsibleRun(Base): id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) playbook_path: Mapped[str] = mapped_column(String(512), nullable=False) - inventory_path: Mapped[str] = mapped_column(String(512), nullable=False) + inventory_path: Mapped[str | None] = mapped_column(String(512), nullable=True) + inventory_scope: Mapped[str] = mapped_column( + String(255), nullable=False, default="steward:all" + ) source_name: Mapped[str] = mapped_column(String(128), nullable=False) # Nullable: automated runs (alert actions, schedules) have no human actor — # NULL renders as "system". Manual runs still record the triggering user. From ea47169049129df7635d2f023e892a9d6af480fe Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 18:46:22 -0400 Subject: [PATCH 025/126] feat(ansible): generate_inventory() + fetch_scope_targets() + unit tests --- steward/ansible/inventory_gen.py | 85 +++++++++++++++++++++++++++++ tests/core/test_inventory_gen.py | 94 ++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 steward/ansible/inventory_gen.py create mode 100644 tests/core/test_inventory_gen.py diff --git a/steward/ansible/inventory_gen.py b/steward/ansible/inventory_gen.py new file mode 100644 index 0000000..09b230c --- /dev/null +++ b/steward/ansible/inventory_gen.py @@ -0,0 +1,85 @@ +"""Ansible --list inventory generation from DB records.""" +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from steward.models.ansible_inventory import AnsibleTarget + + +def generate_inventory(targets: list) -> dict: + """Build an Ansible --list JSON inventory from a list of AnsibleTarget objects. + + Accepts any objects with .name, .address, .ansible_vars (dict), .groups (list + of objects with .name and .ansible_vars). Uses duck typing so unit tests can + pass SimpleNamespace objects. + + Var precedence (lowest → highest, matching Ansible default): + group vars (sorted alphabetically by group name) → target ansible_vars → ansible_host + """ + inv: dict = {"all": {"hosts": []}, "_meta": {"hostvars": {}}} + + for target in targets: + inv["all"]["hosts"].append(target.name) + + # Merge vars: group vars alphabetically, then host vars, then force ansible_host. + merged: dict = {} + for group in sorted(target.groups, key=lambda g: g.name): + merged.update(group.ansible_vars or {}) + merged.update(target.ansible_vars or {}) + merged["ansible_host"] = target.address + + inv["_meta"]["hostvars"][target.name] = merged + + for group in target.groups: + if group.name not in inv: + inv[group.name] = { + "hosts": [], + "vars": dict(group.ansible_vars or {}), + } + inv[group.name]["hosts"].append(target.name) + + return inv + + +async def fetch_scope_targets(db, scope: str) -> list: + """Query AnsibleTarget objects for a given inventory scope string. + + Scopes: + steward:all — every target + steward:group: — targets in that group + steward:target: — single target by ID + + Returns a list with groups pre-loaded (selectinload). Returns [] for + repo:* scopes (caller handles those via legacy path). + """ + from sqlalchemy import select + from sqlalchemy.orm import selectinload + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + + if scope == "steward:all": + stmt = ( + select(AnsibleTarget) + .options(selectinload(AnsibleTarget.groups)) + .order_by(AnsibleTarget.name) + ) + elif scope.startswith("steward:group:"): + group_id = scope[len("steward:group:"):] + stmt = ( + select(AnsibleTarget) + .join(AnsibleTarget.groups) + .where(AnsibleGroup.id == group_id) + .options(selectinload(AnsibleTarget.groups)) + .order_by(AnsibleTarget.name) + ) + elif scope.startswith("steward:target:"): + target_id = scope[len("steward:target:"):] + stmt = ( + select(AnsibleTarget) + .where(AnsibleTarget.id == target_id) + .options(selectinload(AnsibleTarget.groups)) + ) + else: + return [] + + result = await db.execute(stmt) + return list(result.scalars().all()) diff --git a/tests/core/test_inventory_gen.py b/tests/core/test_inventory_gen.py new file mode 100644 index 0000000..f67f1cb --- /dev/null +++ b/tests/core/test_inventory_gen.py @@ -0,0 +1,94 @@ +"""Unit tests for generate_inventory(). + +Uses SimpleNamespace duck-typed objects so no DB or SQLAlchemy is required. +""" +from types import SimpleNamespace +import pytest +from steward.ansible.inventory_gen import generate_inventory + + +def _group(name, ansible_vars=None): + g = SimpleNamespace() + g.name = name + g.ansible_vars = ansible_vars or {} + return g + + +def _target(name, address, ansible_vars=None, groups=None): + t = SimpleNamespace() + t.name = name + t.address = address + t.ansible_vars = ansible_vars or {} + t.groups = groups or [] + return t + + +def test_empty_targets(): + result = generate_inventory([]) + assert result == {"all": {"hosts": []}, "_meta": {"hostvars": {}}} + + +def test_single_target_no_groups(): + target = _target("webserver", "192.168.1.10", {"ansible_user": "ubuntu"}) + result = generate_inventory([target]) + assert result["all"]["hosts"] == ["webserver"] + assert result["_meta"]["hostvars"]["webserver"]["ansible_host"] == "192.168.1.10" + assert result["_meta"]["hostvars"]["webserver"]["ansible_user"] == "ubuntu" + assert set(result.keys()) == {"all", "_meta"} + + +def test_ansible_host_overrides_group_var(): + """ansible_host is always target.address, even if a group sets it to something else.""" + grp = _group("servers", {"ansible_host": "10.0.0.1"}) + target = _target("host1", "192.168.1.50", groups=[grp]) + result = generate_inventory([target]) + assert result["_meta"]["hostvars"]["host1"]["ansible_host"] == "192.168.1.50" + + +def test_group_vars_applied(): + grp = _group("webservers", {"http_port": 80, "ansible_user": "deploy"}) + target = _target("web1", "192.168.1.10", groups=[grp]) + result = generate_inventory([target]) + hostvars = result["_meta"]["hostvars"]["web1"] + assert hostvars["http_port"] == 80 + assert hostvars["ansible_user"] == "deploy" + assert result["webservers"]["hosts"] == ["web1"] + assert result["webservers"]["vars"]["http_port"] == 80 + + +def test_host_vars_override_group_vars(): + """Per-target ansible_vars beat group vars for the same key.""" + grp = _group("g1", {"ansible_user": "group_user"}) + target = _target("t1", "1.2.3.4", {"ansible_user": "host_user"}, groups=[grp]) + result = generate_inventory([target]) + assert result["_meta"]["hostvars"]["t1"]["ansible_user"] == "host_user" + + +def test_group_var_precedence_is_alphabetical(): + """When multiple groups set the same key, alphabetically-last group wins (Ansible default).""" + g_a = _group("aaa", {"timeout": 10}) + g_b = _group("bbb", {"timeout": 20}) + target = _target("t1", "1.1.1.1", groups=[g_b, g_a]) # deliberately reversed order + result = generate_inventory([target]) + # Groups sorted alphabetically: aaa then bbb → bbb wins + assert result["_meta"]["hostvars"]["t1"]["timeout"] == 20 + + +def test_multiple_targets_in_group(): + grp = _group("db", {"db_port": 5432}) + t1 = _target("db1", "10.0.0.1", groups=[grp]) + t2 = _target("db2", "10.0.0.2", groups=[grp]) + result = generate_inventory([t1, t2]) + assert set(result["db"]["hosts"]) == {"db1", "db2"} + assert result["all"]["hosts"] == ["db1", "db2"] + + +def test_target_in_multiple_groups(): + g1 = _group("web", {"role": "web"}) + g2 = _group("prod", {"env": "production"}) + target = _target("web-prod-01", "192.168.1.1", groups=[g1, g2]) + result = generate_inventory([target]) + assert "web-prod-01" in result["web"]["hosts"] + assert "web-prod-01" in result["prod"]["hosts"] + assert result["_meta"]["hostvars"]["web-prod-01"]["role"] == "web" + assert result["_meta"]["hostvars"]["web-prod-01"]["env"] == "production" From d906232eb22dc9dd2b55b513a6fe9097b6d9503d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 18:48:00 -0400 Subject: [PATCH 026/126] feat(ansible): GIT_ASKPASS per-source HTTP token auth + settings UI --- steward/ansible/sources.py | 88 ++++++++--- steward/settings/routes.py | 9 ++ .../templates/settings/_ansible_sources.html | 15 ++ tests/core/test_git_askpass.py | 143 ++++++++++++++++++ 4 files changed, 232 insertions(+), 23 deletions(-) create mode 100644 tests/core/test_git_askpass.py diff --git a/steward/ansible/sources.py b/steward/ansible/sources.py index a457d4d..e1b6855 100644 --- a/steward/ansible/sources.py +++ b/steward/ansible/sources.py @@ -1,6 +1,9 @@ from __future__ import annotations import asyncio import logging +import os +import stat +import tempfile from pathlib import Path logger = logging.getLogger(__name__) @@ -48,6 +51,7 @@ def get_sources(ansible_cfg: dict) -> list[dict]: "url": src.get("url"), "branch": src.get("branch", "main"), "pull_interval_seconds": int(src.get("pull_interval_seconds", 3600)), + "http_token": src.get("http_token", "") if src_type == "git" else "", }) return result @@ -99,27 +103,65 @@ def read_playbook(source_path: str, relative_path: str) -> str | None: async def git_pull(source: dict) -> None: - """Clone the git repo if absent; pull if already present.""" + """Clone the git repo if absent; pull if already present. + + When source['http_token'] is set, injects credentials via a temporary + GIT_ASKPASS script so they never touch .git/config or process args. + """ path = Path(source["path"]) - if not (path / ".git").exists(): - path.mkdir(parents=True, exist_ok=True) - proc = await asyncio.create_subprocess_exec( - "git", "clone", - "--branch", source["branch"], - "--single-branch", - source["url"], str(path), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - logger.error("git clone failed for %r: %s", source["name"], stderr.decode(errors="replace")) - else: - proc = await asyncio.create_subprocess_exec( - "git", "-C", str(path), "pull", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - logger.error("git pull failed for %r: %s", source["name"], stderr.decode(errors="replace")) + token = source.get("http_token", "") + + env = None + askpass_path = None + if token: + fd, askpass_path = tempfile.mkstemp(prefix="steward-askpass-", suffix=".sh") + try: + script = ( + "#!/bin/sh\n" + 'case "$1" in\n' + ' Username*) echo "oauth2" ;;\n' + f' Password*) echo "{token}" ;;\n' + "esac\n" + ) + os.write(fd, script.encode()) + finally: + os.close(fd) + os.chmod(askpass_path, stat.S_IRWXU) + env = {**os.environ, "GIT_ASKPASS": askpass_path} + + try: + if not (path / ".git").exists(): + path.mkdir(parents=True, exist_ok=True) + proc = await asyncio.create_subprocess_exec( + "git", "clone", + "--branch", source["branch"], + "--single-branch", + source["url"], str(path), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + _, stderr = await proc.communicate() + if proc.returncode != 0: + logger.error( + "git clone failed for %r: %s", + source["name"], + stderr.decode(errors="replace"), + ) + else: + proc = await asyncio.create_subprocess_exec( + "git", "-C", str(path), "pull", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + _, stderr = await proc.communicate() + if proc.returncode != 0: + logger.error( + "git pull failed for %r: %s", + source["name"], + stderr.decode(errors="replace"), + ) + finally: + if askpass_path and os.path.exists(askpass_path): + os.unlink(askpass_path) diff --git a/steward/settings/routes.py b/steward/settings/routes.py index 4414031..805b5a1 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -226,6 +226,9 @@ async def ansible_add_source(): entry["pull_interval_seconds"] = int(form.get("pull_interval_seconds", 3600)) except ValueError: entry["pull_interval_seconds"] = 3600 + token = form.get("http_token", "").strip() + if token: + entry["http_token"] = token else: entry["path"] = form.get("path", "").strip() sources = await _get_ansible_sources() @@ -263,6 +266,12 @@ async def ansible_save_source(idx: int): entry["pull_interval_seconds"] = int(form.get("pull_interval_seconds", 3600)) except ValueError: entry["pull_interval_seconds"] = 3600 + token = form.get("http_token", "").strip() + if token: + entry["http_token"] = token + elif "http_token" in sources[idx]: + # Preserve existing token — password fields don't round-trip via browser + entry["http_token"] = sources[idx]["http_token"] else: entry["path"] = form.get("path", "").strip() sources[idx] = entry diff --git a/steward/templates/settings/_ansible_sources.html b/steward/templates/settings/_ansible_sources.html index ba47f76..610102c 100644 --- a/steward/templates/settings/_ansible_sources.html +++ b/steward/templates/settings/_ansible_sources.html @@ -94,6 +94,15 @@
+
+ + +
Pull interval (seconds)
+
+ + +
Date: Fri, 5 Jun 2026 18:49:02 -0400 Subject: [PATCH 027/126] feat(ansible): scope picker in run form + create_run() DB inventory generation --- steward/ansible/routes.py | 44 ++++++++++++++++++++--- steward/templates/ansible/browse.html | 44 +++++++++++++++-------- steward/templates/ansible/run_detail.html | 2 +- 3 files changed, 70 insertions(+), 20 deletions(-) diff --git a/steward/ansible/routes.py b/steward/ansible/routes.py index b304bd5..02918a6 100644 --- a/steward/ansible/routes.py +++ b/steward/ansible/routes.py @@ -37,6 +37,8 @@ async def index(): @ansible_bp.get("/browse") @require_role(UserRole.viewer) async def browse(): + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + all_sources = _get_sources() source_data = [] for source in all_sources: @@ -47,7 +49,21 @@ async def browse(): "playbooks": playbooks, "inventories": inventories, }) - return await render_template("ansible/browse.html", source_data=source_data) + + async with current_app.db_sessionmaker() as db: + targets = (await db.execute( + select(AnsibleTarget).order_by(AnsibleTarget.name) + )).scalars().all() + groups = (await db.execute( + select(AnsibleGroup).order_by(AnsibleGroup.name) + )).scalars().all() + + return await render_template( + "ansible/browse.html", + source_data=source_data, + targets=targets, + groups=groups, + ) @ansible_bp.get("/browse//") @@ -72,19 +88,35 @@ async def view_playbook(source_name: str, playbook_path: str): @ansible_bp.post("/runs") @require_role(UserRole.operator) async def create_run(): + import json as _json + from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory + form = await request.form playbook_path = form.get("playbook_path", "").strip() source_name = form.get("source_name", "").strip() - inventory_path = form.get("inventory_path", "").strip() + inventory_scope = form.get("inventory_scope", "steward:all").strip() - if not playbook_path or not inventory_path: - return "playbook_path and inventory_path are required", 400 + if not playbook_path: + return "playbook_path is required", 400 all_sources = _get_sources() source = next((s for s in all_sources if s["name"] == source_name), None) if source is None: return "Source not found", 404 + # Resolve inventory for this scope. + inventory_content: str | None = None + inventory_path: str | None = None + if inventory_scope.startswith("steward:"): + async with current_app.db_sessionmaker() as db: + targets = await fetch_scope_targets(db, inventory_scope) + inventory_content = _json.dumps(generate_inventory(targets)) + elif inventory_scope.startswith("repo:"): + parts = inventory_scope.split(":", 2) + inventory_path = parts[2] if len(parts) == 3 else "" + else: + return "Invalid inventory_scope", 400 + # Optional run parameters (all passed as argv by the executor — no shell). extra_vars: list[str] = [] for line in (form.get("extra_vars", "") or "").splitlines(): @@ -116,6 +148,7 @@ async def create_run(): id=run_id, playbook_path=playbook_path, inventory_path=inventory_path, + inventory_scope=inventory_scope, source_name=source_name, triggered_by=session["user_id"], status=AnsibleRunStatus.running, @@ -131,9 +164,10 @@ async def create_run(): current_app._get_current_object(), # type: ignore[attr-defined] run_id, playbook_path, - inventory_path, + inventory_path or "", source["path"], params_or_none, + inventory_content, ) ) task.add_done_callback( diff --git a/steward/templates/ansible/browse.html b/steward/templates/ansible/browse.html index e0b95b8..6591d79 100644 --- a/steward/templates/ansible/browse.html +++ b/steward/templates/ansible/browse.html @@ -71,18 +71,37 @@ readonly style="color:var(--text-muted);" value="">
- - + {% if targets %} + + + + {% if groups %} + + {% for grp in groups %} + + {% endfor %} + + {% endif %} + + {% for tgt in targets %} + + {% endfor %} + + {% endif %} + {% if sd.inventories %} + + {% for inv in sd.inventories %} + + {% endfor %} + + {% endif %} + {% if not targets and not sd.inventories %} + + {% endif %}
-
- - -
+
+ + +
+

Members

+ {% if not all_targets %} +

No targets exist yet. + Add targets first.

+ {% else %} +
+ {% for tgt in all_targets %} + + {% endfor %} +
+ {% endif %} +
+ + + +{% endblock %} diff --git a/steward/templates/ansible/inventory/groups.html b/steward/templates/ansible/inventory/groups.html new file mode 100644 index 0000000..bdb0ec3 --- /dev/null +++ b/steward/templates/ansible/inventory/groups.html @@ -0,0 +1,48 @@ +{% extends "base.html" %} +{% block title %}Inventory Groups — Steward{% endblock %} +{% block content %} +
+

Inventory Groups

+ Targets → +
+ +
+

New Group

+
+
+ + +
+ +
+
+ +{% if not groups %} +
+

No groups yet. Create one above.

+
+{% else %} +
+ + + + + + {% for grp in groups %} + + + + + + {% endfor %} + +
NameMembers
{{ grp.name }}{{ grp.targets | length }} + Edit +
+ +
+
+
+{% endif %} +{% endblock %} From 37d9ca8a8a8b57b17ce43928e2763fe046525acd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 18:50:40 -0400 Subject: [PATCH 029/126] feat(ansible): inventory targets CRUD templates --- .../ansible/inventory/target_detail.html | 53 +++++++++++++++++ .../templates/ansible/inventory/targets.html | 57 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 steward/templates/ansible/inventory/target_detail.html create mode 100644 steward/templates/ansible/inventory/targets.html diff --git a/steward/templates/ansible/inventory/target_detail.html b/steward/templates/ansible/inventory/target_detail.html new file mode 100644 index 0000000..588c756 --- /dev/null +++ b/steward/templates/ansible/inventory/target_detail.html @@ -0,0 +1,53 @@ +{% extends "base.html" %} +{% block title %}Target: {{ target.name }} — Steward{% endblock %} +{% block content %} +
+

Target: {{ target.name }}

+ ← Targets +
+ +
+
+

Connection

+
+ + +
+
+ + +
+
+ + +
+
+ +
+

Groups

+ {% if not all_groups %} +

No groups exist yet. + Create a group first.

+ {% else %} +
+ {% for grp in all_groups %} + + {% endfor %} +
+ {% endif %} +
+ + +
+{% endblock %} diff --git a/steward/templates/ansible/inventory/targets.html b/steward/templates/ansible/inventory/targets.html new file mode 100644 index 0000000..17c00a5 --- /dev/null +++ b/steward/templates/ansible/inventory/targets.html @@ -0,0 +1,57 @@ +{% extends "base.html" %} +{% block title %}Inventory Targets — Steward{% endblock %} +{% block content %} +
+

Inventory Targets

+ ← Groups +
+ +
+

New Target

+
+
+ + +
+
+ + +
+ +
+
+ +{% if not targets %} +
+

No targets yet. Add one above.

+
+{% else %} +
+ + + + + + {% for tgt in targets %} + + + + + + + {% endfor %} + +
NameAddressGroups
{{ tgt.name }}{{ tgt.address }} + {% for grp in tgt.groups %} + {{ grp.name }} + {% endfor %} + + Edit +
+ +
+
+
+{% endif %} +{% endblock %} From dd0acaf623728667b5bdfca68f4ea6c573d5b190 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 18:51:33 -0400 Subject: [PATCH 030/126] feat(ansible): host edit page Ansible target link/create/unlink section --- steward/hosts/routes.py | 83 +++++++++++++++++++++++++++++-- steward/templates/hosts/form.html | 47 +++++++++++++++++ 2 files changed, 126 insertions(+), 4 deletions(-) diff --git a/steward/hosts/routes.py b/steward/hosts/routes.py index c48e795..56b66d1 100644 --- a/steward/hosts/routes.py +++ b/steward/hosts/routes.py @@ -147,14 +147,38 @@ async def create_host(): @hosts_bp.get("//edit") @require_role(UserRole.operator) async def edit_host(host_id: str): + from steward.models.ansible_inventory import AnsibleTarget + from sqlalchemy.orm import selectinload + async with current_app.db_sessionmaker() as db: result = await db.execute(select(Host).where(Host.id == host_id)) host = result.scalar_one_or_none() - if host is None: - return "Not found", 404 + if host is None: + return "Not found", 404 + + linked_result = await db.execute( + select(AnsibleTarget) + .where(AnsibleTarget.host_id == host_id) + .options(selectinload(AnsibleTarget.groups)) + ) + linked_target = linked_result.scalar_one_or_none() + + # Only show targets not yet linked to any host (plus the currently linked one) + linkable_result = await db.execute( + select(AnsibleTarget) + .where(AnsibleTarget.host_id.is_(None)) + .order_by(AnsibleTarget.name) + ) + linkable_targets = linkable_result.scalars().all() + return await render_template( - "hosts/form.html", host=host, probe_types=list(ProbeType), - ansible_sources=_ansible_source_names()) + "hosts/form.html", + host=host, + probe_types=list(ProbeType), + ansible_sources=_ansible_source_names(), + linked_target=linked_target, + linkable_targets=linkable_targets, + ) @hosts_bp.post("/") @@ -178,6 +202,57 @@ async def update_host(host_id: str): return redirect(url_for("hosts.list_hosts")) +@hosts_bp.post("//ansible-link") +@require_role(UserRole.operator) +async def ansible_link(host_id: str): + """Link, unlink, or create an AnsibleTarget from this host.""" + from steward.models.ansible_inventory import AnsibleTarget + + form = await request.form + action = form.get("action", "") + + async with current_app.db_sessionmaker() as db: + async with db.begin(): + if action == "unlink": + linked_result = await db.execute( + select(AnsibleTarget).where(AnsibleTarget.host_id == host_id) + ) + linked = linked_result.scalar_one_or_none() + if linked: + linked.host_id = None + + elif action == "link": + target_id = form.get("target_id", "").strip() + if target_id: + result = await db.execute( + select(AnsibleTarget).where(AnsibleTarget.id == target_id) + ) + target = result.scalar_one_or_none() + if target: + # Clear any previous link for this host first + old_result = await db.execute( + select(AnsibleTarget).where(AnsibleTarget.host_id == host_id) + ) + old = old_result.scalar_one_or_none() + if old and old.id != target_id: + old.host_id = None + target.host_id = host_id + + elif action == "create": + host_result = await db.execute(select(Host).where(Host.id == host_id)) + host = host_result.scalar_one_or_none() + if host: + new_target = AnsibleTarget( + id=str(uuid.uuid4()), + name=host.name, + address=host.address, + host_id=host_id, + ) + db.add(new_target) + + return redirect(f"/hosts/{host_id}/edit") + + @hosts_bp.post("//run-playbook") @require_role(UserRole.operator) async def run_playbook(host_id: str): diff --git a/steward/templates/hosts/form.html b/steward/templates/hosts/form.html index 3963be2..1ff251f 100644 --- a/steward/templates/hosts/form.html +++ b/steward/templates/hosts/form.html @@ -88,4 +88,51 @@ {% endif %} + +{% if host %} +
+

Ansible Target

+ {% if linked_target %} +
+
+ {{ linked_target.name }} + {{ linked_target.address }} + {% if linked_target.groups %} +
+ {% for grp in linked_target.groups %} + {{ grp.name }} + {% endfor %} +
+ {% endif %} +
+ Edit Target +
+ + +
+
+ {% else %} +

+ No Ansible target linked. Link an existing target or create one from this host. +

+
+ {% if linkable_targets %} +
+ + + +
+ {% endif %} +
+ + +
+
+ {% endif %} +
+{% endif %} {% endblock %} From 923905e72df914c36039f56e940d5a75f84bbd65 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 18:52:10 -0400 Subject: [PATCH 031/126] test(ansible): integration tests for inventory tables + fetch_scope_targets --- tests/integration/test_ansible_inventory.py | 178 ++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 tests/integration/test_ansible_inventory.py diff --git a/tests/integration/test_ansible_inventory.py b/tests/integration/test_ansible_inventory.py new file mode 100644 index 0000000..57e65e1 --- /dev/null +++ b/tests/integration/test_ansible_inventory.py @@ -0,0 +1,178 @@ +"""Integration tests: ansible_targets / ansible_groups tables + fetch_scope_targets. + +Requires a live Postgres DB (STEWARD_DATABASE_URL). +Run with: + pytest tests/integration/test_ansible_inventory.py -m integration -v +""" +from __future__ import annotations +import os +import uuid + +import pytest +from sqlalchemy import select, text + +pytestmark = pytest.mark.integration + +_NEEDS_DB = pytest.mark.skipif( + not os.environ.get("STEWARD_DATABASE_URL"), + reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)", +) + + +def _make_app(): + from steward.app import create_app + import asyncio + return asyncio.run(create_app(testing=False)) + + +@_NEEDS_DB +def test_ansible_targets_table_exists(): + """Migration 0015 created the ansible_targets table.""" + app = _make_app() + + async def _check(): + async with app.db_sessionmaker() as db: + result = await db.execute( + text( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = 'ansible_targets'" + ) + ) + return {row[0] for row in result.fetchall()} + + import asyncio + columns = asyncio.run(_check()) + assert "id" in columns + assert "name" in columns + assert "address" in columns + assert "ansible_vars" in columns + assert "host_id" in columns + + +@_NEEDS_DB +def test_ansible_groups_table_exists(): + """Migration 0016 created ansible_groups and ansible_target_groups.""" + app = _make_app() + + async def _check(): + async with app.db_sessionmaker() as db: + groups_result = await db.execute( + text( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = 'ansible_groups'" + ) + ) + groups_cols = {row[0] for row in groups_result.fetchall()} + + join_result = await db.execute( + text( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = 'ansible_target_groups'" + ) + ) + join_cols = {row[0] for row in join_result.fetchall()} + return groups_cols, join_cols + + import asyncio + groups_cols, join_cols = asyncio.run(_check()) + assert "id" in groups_cols + assert "name" in groups_cols + assert "ansible_vars" in groups_cols + assert "target_id" in join_cols + assert "group_id" in join_cols + + +@_NEEDS_DB +def test_ansible_run_scope_column_exists(): + """Migration 0017 added inventory_scope to ansible_runs.""" + app = _make_app() + + async def _check(): + async with app.db_sessionmaker() as db: + result = await db.execute( + text( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = 'ansible_runs'" + ) + ) + return {row[0] for row in result.fetchall()} + + import asyncio + columns = asyncio.run(_check()) + assert "inventory_scope" in columns + assert "inventory_path" in columns + + +@_NEEDS_DB +def test_create_target_and_group_and_fetch(): + """Can create a target+group, assign membership, and fetch via fetch_scope_targets.""" + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory + + app = _make_app() + target_id = str(uuid.uuid4()) + group_id = str(uuid.uuid4()) + target_name = f"test-target-{target_id[:8]}" + group_name = f"test-group-{group_id[:8]}" + + async def _run(): + # Create + async with app.db_sessionmaker() as db: + async with db.begin(): + grp = AnsibleGroup( + id=group_id, + name=group_name, + ansible_vars={"env": "test"}, + ) + tgt = AnsibleTarget( + id=target_id, + name=target_name, + address="10.0.99.1", + ansible_vars={"ansible_user": "ubuntu"}, + ) + tgt.groups.append(grp) + db.add(grp) + db.add(tgt) + + # Fetch all — target must be in the list + async with app.db_sessionmaker() as db: + all_targets = await fetch_scope_targets(db, "steward:all") + assert any(t.id == target_id for t in all_targets) + + # Fetch by group + async with app.db_sessionmaker() as db: + group_targets = await fetch_scope_targets(db, f"steward:group:{group_id}") + assert any(t.id == target_id for t in group_targets) + + # Fetch single target + async with app.db_sessionmaker() as db: + single = await fetch_scope_targets(db, f"steward:target:{target_id}") + assert len(single) == 1 + assert single[0].id == target_id + + # Generate inventory and verify var merging + async with app.db_sessionmaker() as db: + targets = await fetch_scope_targets(db, f"steward:group:{group_id}") + inv = generate_inventory(targets) + assert target_name in inv["all"]["hosts"] + assert target_name in inv[group_name]["hosts"] + assert inv["_meta"]["hostvars"][target_name]["ansible_host"] == "10.0.99.1" + assert inv["_meta"]["hostvars"][target_name]["ansible_user"] == "ubuntu" + assert inv["_meta"]["hostvars"][target_name]["env"] == "test" + + # Cleanup + async with app.db_sessionmaker() as db: + async with db.begin(): + t = (await db.execute( + select(AnsibleTarget).where(AnsibleTarget.id == target_id) + )).scalar_one_or_none() + if t: + await db.delete(t) + g = (await db.execute( + select(AnsibleGroup).where(AnsibleGroup.id == group_id) + )).scalar_one_or_none() + if g: + await db.delete(g) + + import asyncio + asyncio.run(_run()) From 80613d6310c1fa0b5908bfc51aa5c620fc4ff6e4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 18:55:21 -0400 Subject: [PATCH 032/126] chore(lint): remove unused imports flagged by ruff Co-Authored-By: Claude Sonnet 4.6 --- steward/ansible/inventory_gen.py | 4 ---- tests/core/test_git_askpass.py | 1 - tests/core/test_inventory_gen.py | 1 - 3 files changed, 6 deletions(-) diff --git a/steward/ansible/inventory_gen.py b/steward/ansible/inventory_gen.py index 09b230c..d6ab7ff 100644 --- a/steward/ansible/inventory_gen.py +++ b/steward/ansible/inventory_gen.py @@ -1,9 +1,5 @@ """Ansible --list inventory generation from DB records.""" from __future__ import annotations -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from steward.models.ansible_inventory import AnsibleTarget def generate_inventory(targets: list) -> dict: diff --git a/tests/core/test_git_askpass.py b/tests/core/test_git_askpass.py index 76a1e5a..a2e9a8e 100644 --- a/tests/core/test_git_askpass.py +++ b/tests/core/test_git_askpass.py @@ -1,5 +1,4 @@ """Unit tests for GIT_ASKPASS injection in git_pull().""" -import asyncio import os import stat from pathlib import Path diff --git a/tests/core/test_inventory_gen.py b/tests/core/test_inventory_gen.py index f67f1cb..5952e3a 100644 --- a/tests/core/test_inventory_gen.py +++ b/tests/core/test_inventory_gen.py @@ -3,7 +3,6 @@ Uses SimpleNamespace duck-typed objects so no DB or SQLAlchemy is required. """ from types import SimpleNamespace -import pytest from steward.ansible.inventory_gen import generate_inventory From b49496b57b23b9befd6f54ccb5982f36d8e55812 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 15 Jun 2026 21:24:21 -0400 Subject: [PATCH 033/126] feat(plugins): default-enable generic bundled plugins on fresh install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fresh install enabled zero plugins — settings.py DEFAULTS had no plugin.* keys, so to_plugins_cfg returned {} and every plugin had to be flipped on by hand. Seed docker, host_agent, http, snmp as default-on (generic, non- vendor-specific); traefik and unifi stay opt-in. Stored choices override the default, so disabling persists. Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/core/settings.py | 11 ++++++ tests/core/test_settings_default_plugins.py | 41 +++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 tests/core/test_settings_default_plugins.py diff --git a/steward/core/settings.py b/steward/core/settings.py index a1e055a..f8cf0d1 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -60,6 +60,17 @@ DEFAULTS: dict[str, Any] = { "ping.threshold.good_ms": 50, "ping.threshold.warn_ms": 200, "plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml", + # Default-enabled plugins. These are the generic, non-vendor-specific + # bundled plugins (protocols/standards, not a single product) — useful on + # almost any install, so a fresh deployment comes up monitoring rather than + # blank. Vendor-specific plugins (traefik, unifi) stay opt-in. An operator + # who disables one writes plugin.={"enabled": False}, which overrides + # these defaults (stored value wins in get_all_settings/load_settings_sync). + # Per-plugin yaml config defaults are merged on top at load time. + "plugin.docker": {"enabled": True}, + "plugin.host_agent": {"enabled": True}, + "plugin.http": {"enabled": True}, + "plugin.snmp": {"enabled": True}, # OIDC single-sign-on "oidc.enabled": False, "oidc.discovery_url": "", diff --git a/tests/core/test_settings_default_plugins.py b/tests/core/test_settings_default_plugins.py new file mode 100644 index 0000000..32a45c9 --- /dev/null +++ b/tests/core/test_settings_default_plugins.py @@ -0,0 +1,41 @@ +"""Pure-function tests for the default-enabled plugin set. + +A fresh install (no stored plugin.* rows) should come up with the generic, +non-vendor-specific bundled plugins already enabled, while vendor-specific +plugins stay opt-in. An operator's stored choice must override the default. +No DB / Quart fixtures — we exercise to_plugins_cfg over the DEFAULTS dict +and a DEFAULTS-merged-with-stored dict, mirroring get_all_settings' merge. +""" +from steward.core import settings as settings_module +from steward.core.settings import DEFAULTS, to_plugins_cfg + +DEFAULT_ON = {"docker", "host_agent", "http", "snmp"} +VENDOR_OPT_IN = {"traefik", "unifi"} + + +def test_generic_plugins_enabled_by_default(): + cfg = to_plugins_cfg(DEFAULTS) + for name in DEFAULT_ON: + assert cfg.get(name, {}).get("enabled") is True, name + + +def test_vendor_plugins_not_enabled_by_default(): + cfg = to_plugins_cfg(DEFAULTS) + for name in VENDOR_OPT_IN: + assert name not in cfg, name + + +def test_stored_choice_overrides_default(): + # get_all_settings / load_settings_sync overlay stored values onto DEFAULTS; + # a stored disable must win over the built-in default-on. + merged = {**DEFAULTS, "plugin.docker": {"enabled": False}} + cfg = to_plugins_cfg(merged) + assert cfg["docker"]["enabled"] is False + # untouched defaults remain enabled + assert cfg["http"]["enabled"] is True + + +def test_defaults_use_plugin_dot_namespace(): + # The keys must live under the plugin. namespace to_plugins_cfg reads. + for name in DEFAULT_ON: + assert f"plugin.{name}" in settings_module.DEFAULTS From 67a1bc740d2e8860ee3aaffc4a97dc594d028d87 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 15 Jun 2026 21:39:41 -0400 Subject: [PATCH 034/126] fix(test): call sync create_app directly in ansible inventory integration test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create_app is synchronous (returns Quart) and runs its own internal asyncio.run for settings/migrations. _make_app wrapped it in asyncio.run, which nested event loops and raised "a coroutine is required" — the other integration tests already call create_app() directly. This test was added in an unpushed commit, so CI never exercised it until now. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/integration/test_ansible_inventory.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_ansible_inventory.py b/tests/integration/test_ansible_inventory.py index 57e65e1..711f4e0 100644 --- a/tests/integration/test_ansible_inventory.py +++ b/tests/integration/test_ansible_inventory.py @@ -20,9 +20,11 @@ _NEEDS_DB = pytest.mark.skipif( def _make_app(): + # create_app is synchronous and runs its own internal asyncio.run for + # settings/migrations — call it directly (matching the other integration + # tests); wrapping it in asyncio.run nests event loops and fails. from steward.app import create_app - import asyncio - return asyncio.run(create_app(testing=False)) + return create_app(testing=False) @_NEEDS_DB From 3b6e005ed84830ab3535fab1a44be43b6e091408 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 15 Jun 2026 22:18:29 -0400 Subject: [PATCH 035/126] feat(status): unified Status page + widget across ping/DNS/HTTP monitors Adds a Kuma-style "is everything up?" surface that aggregates heterogeneous monitor types via a status-source registry (steward/core/status.py): each type registers an async source(db) -> [StatusEntry]. Core registers ping/DNS; the http plugin registers its own from setup() so core never imports plugin tables. Per entry: current up/down, last-30 heartbeat bar, uptime % (24h/7d/30d), latest latency + response sparkline, and TLS expiry countdown (HTTP). New /status page (live htmx refresh) + a status_overview dashboard widget + nav link. Pure-function unit tests for registry + sparkline. First deliverable of milestone #68 (task #866). Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/http/__init__.py | 4 + plugins/http/routes.py | 87 +++++++++- steward/app.py | 8 + steward/core/status.py | 233 +++++++++++++++++++++++++++ steward/core/widgets.py | 10 ++ steward/status/__init__.py | 1 + steward/status/routes.py | 51 ++++++ steward/templates/base.html | 1 + steward/templates/status/index.html | 12 ++ steward/templates/status/rows.html | 96 +++++++++++ steward/templates/status/widget.html | 27 ++++ tests/core/test_status.py | 71 ++++++++ 12 files changed, 599 insertions(+), 2 deletions(-) create mode 100644 steward/core/status.py create mode 100644 steward/status/__init__.py create mode 100644 steward/status/routes.py create mode 100644 steward/templates/status/index.html create mode 100644 steward/templates/status/rows.html create mode 100644 steward/templates/status/widget.html create mode 100644 tests/core/test_status.py diff --git a/plugins/http/__init__.py b/plugins/http/__init__.py index cb79685..ef17129 100644 --- a/plugins/http/__init__.py +++ b/plugins/http/__init__.py @@ -12,6 +12,10 @@ def setup(app: "Quart") -> None: global _app _app = app from .models import HttpMonitor, HttpResult # noqa: F401 + # Contribute HTTP monitors to the core unified Status page. + from steward.core.status import register_status_source + from .routes import http_status_source + register_status_source(http_status_source) def get_scheduled_tasks() -> list: diff --git a/plugins/http/routes.py b/plugins/http/routes.py index 24bede7..6f69d85 100644 --- a/plugins/http/routes.py +++ b/plugins/http/routes.py @@ -1,12 +1,13 @@ # plugins/http/routes.py from __future__ import annotations import uuid -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from quart import Blueprint, current_app, render_template, request, redirect, url_for -from sqlalchemy import Integer, cast, func, select +from sqlalchemy import Integer, and_, case, cast, func, select from steward.auth.middleware import require_role from steward.models.users import UserRole +from steward.core.status import StatusEntry, HEARTBEAT_COUNT from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds from .models import HttpMonitor, HttpResult @@ -94,6 +95,88 @@ async def _render_rows(range_key: str = DEFAULT_RANGE): return monitor_data, up, down, range_key +async def http_status_source(db) -> list[StatusEntry]: + """Contribute enabled HTTP monitors to the unified Status page. + + Registered with steward.core.status from this plugin's setup() so the core + Status surface can include HTTP without importing plugin tables directly. + """ + monitors = list((await db.execute( + select(HttpMonitor).where(HttpMonitor.enabled == True) # noqa: E712 + .order_by(HttpMonitor.created_at) + )).scalars()) + ids = [m.id for m in monitors] + if not ids: + return [] + + now = datetime.now(timezone.utc) + c30, c7, c24 = now - timedelta(days=30), now - timedelta(days=7), now - timedelta(hours=24) + up_res = await db.execute( + select( + HttpResult.monitor_id, + func.count().label("total_30d"), + func.sum(case((HttpResult.is_up == True, 1), else_=0)).label("up_30d"), # noqa: E712 + func.sum(case((HttpResult.checked_at >= c7, 1), else_=0)).label("total_7d"), + func.sum(case((and_(HttpResult.checked_at >= c7, HttpResult.is_up == True), 1), else_=0)).label("up_7d"), # noqa: E712 + func.sum(case((HttpResult.checked_at >= c24, 1), else_=0)).label("total_24h"), + func.sum(case((and_(HttpResult.checked_at >= c24, HttpResult.is_up == True), 1), else_=0)).label("up_24h"), # noqa: E712 + ) + .where(HttpResult.monitor_id.in_(ids)) + .where(HttpResult.checked_at >= c30) + .group_by(HttpResult.monitor_id) + ) + + def _pct(u, t): + return round(float(u) / float(t) * 100, 2) if t else None + + uptime = {r.monitor_id: { + "24h": _pct(r.up_24h, r.total_24h), + "7d": _pct(r.up_7d, r.total_7d), + "30d": _pct(r.up_30d, r.total_30d), + } for r in up_res} + + # Last N results per monitor (one query) for the heartbeat bar + sparkline. + rn = func.row_number().over( + partition_by=HttpResult.monitor_id, order_by=HttpResult.checked_at.desc() + ).label("rn") + subq = select(HttpResult.id, rn).where(HttpResult.monitor_id.in_(ids)).subquery() + recent_res = await db.execute( + select(HttpResult) + .join(subq, HttpResult.id == subq.c.id) + .where(subq.c.rn <= HEARTBEAT_COUNT) + .order_by(HttpResult.monitor_id, HttpResult.checked_at.asc()) + ) + recent: dict[str, list] = {mid: [] for mid in ids} + for row in recent_res.scalars(): + recent[row.monitor_id].append(row) + + entries: list[StatusEntry] = [] + for m in monitors: + rows = recent.get(m.id, []) + latest = rows[-1] if rows else None + heartbeat = [{ + "state": "up" if r.is_up else "down", + "title": ( + f"{r.status_code or '—'} · {r.response_ms:.0f} ms — {r.checked_at:%H:%M:%S} UTC" + if r.is_up and r.response_ms is not None + else f"{'Up' if r.is_up else 'Down'} — {r.checked_at:%H:%M:%S} UTC" + ), + } for r in rows] + status = "pending" if latest is None else ("up" if latest.is_up else "down") + tls_days = None + if latest and latest.tls_expires_at: + tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0 + entries.append(StatusEntry( + kind="http", key=m.id, name=m.name, target=m.url, status=status, + last_checked=latest.checked_at if latest else None, + uptime=uptime.get(m.id, {}), heartbeat=heartbeat, + latency_ms=latest.response_ms if latest else None, + spark=[r.response_ms for r in rows if r.response_ms is not None], + tls_days=tls_days, detail_url="/plugins/http/", + )) + return entries + + @http_bp.get("/") @require_role(UserRole.viewer) async def index(): diff --git a/steward/app.py b/steward/app.py index d1d3683..f488b87 100644 --- a/steward/app.py +++ b/steward/app.py @@ -98,6 +98,7 @@ def create_app( from .hosts.routes import hosts_bp from .ping.routes import ping_bp from .dns.routes import dns_bp + from .status.routes import status_bp from .alerts.routes import alerts_bp from .ansible.routes import ansible_bp from .ansible.inventory_routes import inventory_bp @@ -109,12 +110,19 @@ def create_app( app.register_blueprint(hosts_bp) app.register_blueprint(ping_bp) app.register_blueprint(dns_bp) + app.register_blueprint(status_bp) app.register_blueprint(alerts_bp) app.register_blueprint(ansible_bp) app.register_blueprint(inventory_bp) app.register_blueprint(settings_bp) app.register_blueprint(audit_bp) + # Register the core (ping/DNS) status sources for the unified Status page. + # Plugins register their own sources from setup() (e.g. http). Idempotent. + from .core.status import register_status_source, ping_status_source, dns_status_source + register_status_source(ping_status_source) + register_status_source(dns_status_source) + # ── 8. Build task registry ───────────────────────────────────────────────── app._task_registry = [] diff --git a/steward/core/status.py b/steward/core/status.py new file mode 100644 index 0000000..7c903be --- /dev/null +++ b/steward/core/status.py @@ -0,0 +1,233 @@ +# steward/core/status.py +"""Unified monitor-status aggregation. + +A single readable "is everything up?" surface (the Status page + dashboard +widget) is assembled from heterogeneous monitor types — core ping/DNS, plus +any plugin that contributes (e.g. the http plugin). Rather than have the core +status page import plugin tables (plugins are optional and loaded late), each +monitor type registers a *status source*: an async callable(db) that returns a +list of normalised StatusEntry objects. Core registers its ping/DNS sources in +create_app; a plugin registers its own from setup(). +""" +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Awaitable, Callable + +from sqlalchemy import and_, case, func, select + +logger = logging.getLogger(__name__) + +HEARTBEAT_COUNT = 30 # number of recent checks shown in the heartbeat bar + + +@dataclass +class StatusEntry: + """One monitored thing, normalised across monitor types for display.""" + kind: str # "ping" | "dns" | "http" | ... + key: str # unique within kind (host_id / monitor_id) + name: str + target: str = "" # address / URL shown as subtitle + status: str = "pending" # "up" | "down" | "pending" + last_checked: datetime | None = None + uptime: dict[str, float | None] = field(default_factory=dict) # 24h/7d/30d + heartbeat: list[dict] = field(default_factory=list) # oldest-first {state,title} + latency_ms: float | None = None + spark: list[float] = field(default_factory=list) # response series + spark_svg: str = "" # filled by the route via sparkline_svg() + tls_days: float | None = None # days until TLS expiry (http only) + detail_url: str | None = None + + +StatusSource = Callable[[object], Awaitable[list[StatusEntry]]] +_SOURCES: list[StatusSource] = [] + + +def register_status_source(fn: StatusSource) -> None: + """Register a status source. Idempotent on the same callable.""" + if fn not in _SOURCES: + _SOURCES.append(fn) + + +def clear_status_sources() -> None: + """Reset the registry (used by tests).""" + _SOURCES.clear() + + +async def collect_status(db) -> list[StatusEntry]: + """Gather entries from every registered source, down-first then by name. + + A failing source is logged and skipped so one broken plugin can't blank + the whole Status page. + """ + entries: list[StatusEntry] = [] + for src in list(_SOURCES): + try: + entries.extend(await src(db)) + except Exception: + logger.exception("status source %r failed", getattr(src, "__name__", src)) + # Down first (most urgent), then pending, then up; alphabetical within. + order = {"down": 0, "pending": 1, "up": 2} + entries.sort(key=lambda e: (order.get(e.status, 3), e.kind, e.name.lower())) + return entries + + +def sparkline_svg(values: list[float], width: int = 80, height: int = 20, + stroke: str = "#6060c0") -> str: + """Tiny inline SVG polyline for a response-time series.""" + vals = [v for v in values if v is not None] + if len(vals) < 2: + return f'' + mn, mx = min(vals), max(vals) + if mx == mn: + mx = mn + 1.0 + step = width / (len(vals) - 1) + pts = [] + for i, v in enumerate(vals): + x = i * step + y = height - (v - mn) / (mx - mn) * (height - 2) - 1 + pts.append(f"{x:.1f},{y:.1f}") + poly = " ".join(pts) + return ( + f'' + f'' + f'' + ) + + +# ── Shared query helpers (host-keyed result tables: ping, dns) ──────────────── + +async def _last_n_by_host(db, model, ts_col, host_ids: list[str], + n: int = HEARTBEAT_COUNT) -> dict[str, list]: + """Return {host_id: [rows]} of the last n results per host, oldest-first.""" + if not host_ids: + return {} + rn = func.row_number().over( + partition_by=model.host_id, order_by=ts_col.desc() + ).label("rn") + subq = select(model.id, rn).where(model.host_id.in_(host_ids)).subquery() + res = await db.execute( + select(model) + .join(subq, model.id == subq.c.id) + .where(subq.c.rn <= n) + .order_by(model.host_id, ts_col.asc()) + ) + out: dict[str, list] = {hid: [] for hid in host_ids} + for row in res.scalars(): + out[row.host_id].append(row) + return out + + +async def _uptime_by_host(db, model, ts_col, status_col, up_value, + host_ids: list[str]) -> dict[str, dict]: + """Per-host uptime % over 24h/7d/30d in one grouped query.""" + if not host_ids: + return {} + now = datetime.now(timezone.utc) + c30, c7, c24 = now - timedelta(days=30), now - timedelta(days=7), now - timedelta(hours=24) + res = await db.execute( + select( + model.host_id, + func.count().label("total_30d"), + func.sum(case((status_col == up_value, 1), else_=0)).label("up_30d"), + func.sum(case((ts_col >= c7, 1), else_=0)).label("total_7d"), + func.sum(case((and_(ts_col >= c7, status_col == up_value), 1), else_=0)).label("up_7d"), + func.sum(case((ts_col >= c24, 1), else_=0)).label("total_24h"), + func.sum(case((and_(ts_col >= c24, status_col == up_value), 1), else_=0)).label("up_24h"), + ) + .where(model.host_id.in_(host_ids)) + .where(ts_col >= c30) + .group_by(model.host_id) + ) + + def _pct(up, total): + return round(float(up) / float(total) * 100, 2) if total else None + + out: dict[str, dict] = {} + for r in res: + out[r.host_id] = { + "24h": _pct(r.up_24h, r.total_24h), + "7d": _pct(r.up_7d, r.total_7d), + "30d": _pct(r.up_30d, r.total_30d), + } + return out + + +# ── Core status sources: ping + DNS ─────────────────────────────────────────── + +async def ping_status_source(db) -> list[StatusEntry]: + from steward.models.hosts import Host + from steward.models.monitors import PingResult, PingStatus + + hosts = (await db.execute( + select(Host).where(Host.ping_enabled.is_(True)).order_by(Host.name) + )).scalars().all() + ids = [h.id for h in hosts] + recent = await _last_n_by_host(db, PingResult, PingResult.probed_at, ids) + uptime = await _uptime_by_host( + db, PingResult, PingResult.probed_at, PingResult.status, PingStatus.up, ids + ) + + entries: list[StatusEntry] = [] + for h in hosts: + rows = recent.get(h.id, []) + latest = rows[-1] if rows else None + heartbeat = [{ + "state": "down" if r.status == PingStatus.down else "up", + "title": ( + f"Down — {r.probed_at:%H:%M:%S} UTC" if r.status == PingStatus.down + else f"{r.response_time_ms:.0f} ms — {r.probed_at:%H:%M:%S} UTC" + if r.response_time_ms is not None else f"Up — {r.probed_at:%H:%M:%S} UTC" + ), + } for r in rows] + status = "pending" if latest is None else ( + "down" if latest.status == PingStatus.down else "up" + ) + entries.append(StatusEntry( + kind="ping", key=h.id, name=h.name, target=h.address, status=status, + last_checked=latest.probed_at if latest else None, + uptime=uptime.get(h.id, {}), heartbeat=heartbeat, + latency_ms=latest.response_time_ms if latest else None, + spark=[r.response_time_ms for r in rows if r.response_time_ms is not None], + detail_url="/ping/", + )) + return entries + + +async def dns_status_source(db) -> list[StatusEntry]: + from steward.models.hosts import Host + from steward.models.monitors import DnsResult, DnsStatus + + hosts = (await db.execute( + select(Host).where(Host.dns_enabled.is_(True)).order_by(Host.name) + )).scalars().all() + ids = [h.id for h in hosts] + recent = await _last_n_by_host(db, DnsResult, DnsResult.resolved_at, ids) + uptime = await _uptime_by_host( + db, DnsResult, DnsResult.resolved_at, DnsResult.status, DnsStatus.resolved, ids + ) + + entries: list[StatusEntry] = [] + for h in hosts: + rows = recent.get(h.id, []) + latest = rows[-1] if rows else None + heartbeat = [{ + "state": "up" if r.status == DnsStatus.resolved else "down", + "title": ( + f"{r.resolved_ip} — {r.resolved_at:%H:%M:%S} UTC" if r.status == DnsStatus.resolved + else f"Failed — {r.resolved_at:%H:%M:%S} UTC" + ), + } for r in rows] + status = "pending" if latest is None else ( + "up" if latest.status == DnsStatus.resolved else "down" + ) + entries.append(StatusEntry( + kind="dns", key=h.id, name=h.name, target=h.address, status=status, + last_checked=latest.resolved_at if latest else None, + uptime=uptime.get(h.id, {}), heartbeat=heartbeat, + detail_url="/dns/", + )) + return entries diff --git a/steward/core/widgets.py b/steward/core/widgets.py index e956a4b..ba01d7f 100644 --- a/steward/core/widgets.py +++ b/steward/core/widgets.py @@ -22,6 +22,16 @@ WIDGET_REGISTRY: dict[str, dict] = { "poll": True, "params": [], }, + "status_overview": { + "key": "status_overview", + "label": "Status — Overview", + "description": "Unified up/down/pending summary across ping, DNS, and HTTP monitors", + "hx_url": "/status/widget", + "detail_url": "/status", + "plugin": None, + "poll": True, + "params": [], + }, "uptime_summary": { "key": "uptime_summary", "label": "Uptime / SLA", diff --git a/steward/status/__init__.py b/steward/status/__init__.py new file mode 100644 index 0000000..d0efc1c --- /dev/null +++ b/steward/status/__init__.py @@ -0,0 +1 @@ +# steward/status/ — unified monitor-status surface (page + dashboard widget) diff --git a/steward/status/routes.py b/steward/status/routes.py new file mode 100644 index 0000000..d4edcc9 --- /dev/null +++ b/steward/status/routes.py @@ -0,0 +1,51 @@ +from __future__ import annotations +from quart import Blueprint, current_app, render_template, request + +from steward.auth.middleware import require_role +from steward.core.status import collect_status, sparkline_svg +from steward.models.users import UserRole + +status_bp = Blueprint("status", __name__, url_prefix="/status") + + +def _summarize(entries) -> tuple[int, int, int]: + up = sum(1 for e in entries if e.status == "up") + down = sum(1 for e in entries if e.status == "down") + pending = sum(1 for e in entries if e.status == "pending") + return up, down, pending + + +@status_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + return await render_template("status/index.html", poll_interval=poll_interval) + + +@status_bp.get("/rows") +@require_role(UserRole.viewer) +async def rows(): + """HTMX fragment — the full status list + summary, refreshed on poll.""" + async with current_app.db_sessionmaker() as db: + entries = await collect_status(db) + for e in entries: + if e.spark: + e.spark_svg = sparkline_svg(e.spark) + up, down, pending = _summarize(entries) + return await render_template( + "status/rows.html", entries=entries, up=up, down=down, pending=pending + ) + + +@status_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """HTMX dashboard widget — compact up/down/pending summary + problem list.""" + async with current_app.db_sessionmaker() as db: + entries = await collect_status(db) + up, down, pending = _summarize(entries) + return await render_template( + "status/widget.html", + entries=entries, up=up, down=down, pending=pending, + widget_id=request.args.get("wid", "0"), + ) diff --git a/steward/templates/base.html b/steward/templates/base.html index 3f6837f..a23329d 100644 --- a/steward/templates/base.html +++ b/steward/templates/base.html @@ -198,6 +198,7 @@ textarea { resize: vertical; } Steward Dashboard + Status Hosts Uptime Ping diff --git a/steward/templates/status/index.html b/steward/templates/status/index.html new file mode 100644 index 0000000..8de5b21 --- /dev/null +++ b/steward/templates/status/index.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} +{% block title %}Status — Steward{% endblock %} +{% block content %} +
+

Status

+ Live — refreshes every {{ poll_interval }}s +
+ +
+
Loading status…
+
+{% endblock %} diff --git a/steward/templates/status/rows.html b/steward/templates/status/rows.html new file mode 100644 index 0000000..152e44e --- /dev/null +++ b/steward/templates/status/rows.html @@ -0,0 +1,96 @@ +{# status/rows.html — unified monitor list + summary, refreshed on poll #} +{% macro uptime_cell(pct) %} +{%- if pct is none -%}— +{%- elif pct >= 99.9 -%}{{ "%.2f"|format(pct) }}% +{%- elif pct >= 99 -%}{{ "%.2f"|format(pct) }}% +{%- elif pct >= 95 -%}{{ "%.2f"|format(pct) }}% +{%- else -%}{{ "%.2f"|format(pct) }}% +{%- endif -%} +{% endmacro %} + +{# ── Summary ─────────────────────────────────────────────────────────────── #} +
+
+
Up
+ {{ up }} +
+
+
Down
+ {{ down }} +
+
+
Pending
+ {{ pending }} +
+
+ +{% if not entries %} +
+ No monitors configured yet. Add hosts (ping / DNS) or HTTP monitors to populate the board. +
+{% else %} +
+ {# header row #} +
+ Monitor + Recent checks + Latest + 24h + 7d + 30d + TLS +
+ + {% for e in entries %} +
+ {# name + kind #} +
+
+ + {% if e.detail_url %}{{ e.name }} + {% else %}{{ e.name }}{% endif %} +
+
+ {{ e.kind }} · {{ e.target }} +
+
+ + {# heartbeat bar #} +
+ {% set pad = 30 - e.heartbeat|length %} + {% if pad > 0 %}{% for _ in range(pad) %}{% endfor %}{% endif %} + {% for hb in e.heartbeat %} + + {% endfor %} +
+ + {# latest latency + sparkline #} +
+ {% if e.status == 'down' %}DOWN + {% elif e.latency_ms is not none %} + {% if e.spark_svg %}{{ e.spark_svg|safe }}
{% endif %} + {{ "%.0f"|format(e.latency_ms) }} ms + {% elif e.status == 'up' %}UP + {% else %}—{% endif %} +
+ + {# uptime windows #} + {{ uptime_cell(e.uptime.get('24h')) }} + {{ uptime_cell(e.uptime.get('7d')) }} + {{ uptime_cell(e.uptime.get('30d')) }} + + {# TLS expiry countdown #} + + {% if e.tls_days is none %}— + {% elif e.tls_days < 0 %}expired + {% elif e.tls_days < 14 %}{{ e.tls_days|round|int }}d + {% elif e.tls_days < 30 %}{{ e.tls_days|round|int }}d + {% else %}{{ e.tls_days|round|int }}d{% endif %} + +
+ {% endfor %} +
+

+ Heartbeat shows the last 30 checks (newest on the right). Uptime % is over the rolling window. TLS countdown applies to HTTPS monitors. +

+{% endif %} diff --git a/steward/templates/status/widget.html b/steward/templates/status/widget.html new file mode 100644 index 0000000..eef40b8 --- /dev/null +++ b/steward/templates/status/widget.html @@ -0,0 +1,27 @@ +{# status/widget.html — compact dashboard summary + problem list #} +
+ {{ up }}up + {{ down }}down + {{ pending }}pending +
+ +{% set problems = entries | rejectattr("status", "equalto", "up") | list %} +{% if not entries %} +
No monitors configured.
+{% elif not problems %} +
✓ All {{ up }} monitors are up.
+{% else %} +
+ {% for e in problems[:8] %} +
+ + {{ e.name }} + {{ e.kind }} + {{ e.status }} +
+ {% endfor %} + {% if problems|length > 8 %} +
+{{ problems|length - 8 }} more…
+ {% endif %} +
+{% endif %} diff --git a/tests/core/test_status.py b/tests/core/test_status.py new file mode 100644 index 0000000..5a51a5d --- /dev/null +++ b/tests/core/test_status.py @@ -0,0 +1,71 @@ +"""Unit tests for the status-source registry + sparkline (no DB, no Quart). + +The DB-backed sources (ping/dns/http) are exercised in the integration lane; +here we cover the registry contract, ordering, failure isolation, and the +pure sparkline helper. +""" +import asyncio + +from steward.core import status as status_mod +from steward.core.status import ( + StatusEntry, + clear_status_sources, + collect_status, + register_status_source, + sparkline_svg, +) + + +def _entry(name, st, kind="ping"): + return StatusEntry(kind=kind, key=name, name=name, status=st) + + +def test_collect_orders_down_then_pending_then_up_alphabetical(): + clear_status_sources() + + async def src(db): + return [_entry("zeta", "up"), _entry("alpha", "down"), + _entry("beta", "pending"), _entry("gamma", "up")] + + register_status_source(src) + entries = asyncio.run(collect_status(None)) + assert [e.status for e in entries] == ["down", "pending", "up", "up"] + # within the trailing "up" group, alphabetical by name + assert entries[2].name == "gamma" + assert entries[3].name == "zeta" + clear_status_sources() + + +def test_failing_source_is_skipped_not_fatal(): + clear_status_sources() + + async def good(db): + return [_entry("a", "up")] + + async def bad(db): + raise RuntimeError("boom") + + register_status_source(good) + register_status_source(bad) + entries = asyncio.run(collect_status(None)) + assert [e.name for e in entries] == ["a"] + clear_status_sources() + + +def test_register_is_idempotent_on_same_callable(): + clear_status_sources() + + async def src(db): + return [] + + register_status_source(src) + register_status_source(src) + assert status_mod._SOURCES.count(src) == 1 + clear_status_sources() + + +def test_sparkline_svg_handles_edge_cases(): + assert "polyline" not in sparkline_svg([5]) # <2 points → empty svg + assert "polyline" in sparkline_svg([1, 2, 3]) + assert "polyline" in sparkline_svg([4, 4, 4]) # flat series, no div-by-zero + assert "polyline" in sparkline_svg([1, None, 3]) # None values filtered From a9e7baee6a5256953c8f0f69a88f326c1af1a58d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 10:11:30 -0400 Subject: [PATCH 036/126] feat(host_agent): collect network, disk I/O, per-core CPU, temps, memory PSI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the push agent (v1.2.0) with the Netdata-style signals the operator wants: per-core CPU, per-interface network throughput and per-disk I/O (rates derived in-agent from monotonic /proc counter deltas, with counter-reset clamping), hardware temperatures (/sys/class/hwmon), memory-pressure PSI (/proc/pressure), and cached/buffers memory breakdown. All stdlib-only. Server side needs no migration — these land as additional rows in the shared PluginMetric table via _expand_sample_to_metrics. Per-resource series use a ':' in resource_name (host:net:eth0, host:core0, host:temp:Package) so the existing fleet widget's ':' filter ignores them; host-level totals/max are emitted at the bare host resource. New sample keys are optional, so older agents keep ingesting unchanged. Unit tests for the new parsers, rate/reset logic, and the expander contract. Data foundation for the Netdata-style host view (task #867). Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/agent.py | 257 ++++++++++++++++-- plugins/host_agent/plugin.yaml | 4 +- plugins/host_agent/routes.py | 54 ++++ tests/plugins/host_agent/test_collectors.py | 96 +++++++ .../plugins/host_agent/test_expand_metrics.py | 60 ++++ 5 files changed, 447 insertions(+), 24 deletions(-) create mode 100644 tests/plugins/host_agent/test_collectors.py create mode 100644 tests/plugins/host_agent/test_expand_metrics.py diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index 2051c60..f78cef0 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -8,6 +8,7 @@ from __future__ import annotations import json import os +import re import shutil import signal import socket @@ -18,7 +19,7 @@ import urllib.request from collections import deque from datetime import datetime, timezone -AGENT_VERSION = "1.1.0" +AGENT_VERSION = "1.2.0" class ConfigError(Exception): @@ -71,6 +72,15 @@ MEMINFO_PATH = "/proc/meminfo" LOADAVG_PATH = "/proc/loadavg" UPTIME_PATH = "/proc/uptime" OS_RELEASE_PATH = "/etc/os-release" +NETDEV_PATH = "/proc/net/dev" +DISKSTATS_PATH = "/proc/diskstats" +HWMON_DIR = "/sys/class/hwmon" +PSI_DIR = "/proc/pressure" + +# Partition names to drop from disk I/O (we report whole-disk throughput only). +_PARTITION_RE = re.compile( + r"^(?:sd[a-z]+\d+|vd[a-z]+\d+|hd[a-z]+\d+|nvme\d+n\d+p\d+|mmcblk\d+p\d+)$" +) def _read_file(path: str) -> str: @@ -78,28 +88,49 @@ def _read_file(path: str) -> str: return f.read() -def _parse_cpu_line(stat_text: str) -> tuple[int, int]: - """Return (total_jiffies, idle_jiffies) for the aggregate cpu line.""" +def _parse_cpu_lines(stat_text: str) -> dict: + """Return {name: (total_jiffies, idle_jiffies)} for every cpu line. + + 'cpu' is the aggregate; 'cpu0', 'cpu1', … are per-core. + """ + out: dict = {} for line in stat_text.splitlines(): - if line.startswith("cpu "): - parts = line.split() + if not line.startswith("cpu"): + continue + parts = line.split() + try: fields = [int(x) for x in parts[1:]] - idle = fields[3] + (fields[4] if len(fields) > 4 else 0) # idle + iowait - total = sum(fields) - return total, idle - raise RuntimeError("no aggregate cpu line in /proc/stat") + except ValueError: + continue + if len(fields) < 4: + continue + idle = fields[3] + (fields[4] if len(fields) > 4 else 0) # idle + iowait + out[parts[0]] = (sum(fields), idle) + return out -def collect_cpu(sample_window: float = 0.2) -> float: - """Return CPU utilization % over sample_window seconds.""" - t1, i1 = _parse_cpu_line(_read_file(STAT_PATH)) +def collect_cpu(sample_window: float = 0.2) -> tuple[float, list]: + """Return (aggregate_pct, [per_core_pct, …]) over sample_window seconds.""" + a = _parse_cpu_lines(_read_file(STAT_PATH)) time.sleep(sample_window) - t2, i2 = _parse_cpu_line(_read_file(STAT_PATH)) - dt = t2 - t1 - di = i2 - i1 - if dt <= 0: - return 0.0 - return round(100.0 * (dt - di) / dt, 2) + b = _parse_cpu_lines(_read_file(STAT_PATH)) + + def _pct(name: str): + if name not in a or name not in b: + return None + dt = b[name][0] - a[name][0] + di = b[name][1] - a[name][1] + if dt <= 0: + return 0.0 + return round(100.0 * (dt - di) / dt, 2) + + agg = _pct("cpu") or 0.0 + cores: list = [] + i = 0 + while f"cpu{i}" in b: + cores.append(_pct(f"cpu{i}")) + i += 1 + return agg, cores def collect_memory() -> dict: @@ -125,6 +156,8 @@ def collect_memory() -> dict: "used_bytes": max(total - available, 0), "available_bytes": available, "swap_used_bytes": max(swap_total - swap_free, 0), + "cached_bytes": info.get("Cached", 0), + "buffers_bytes": info.get("Buffers", 0), } @@ -210,6 +243,149 @@ def default_mounts() -> list[str]: return mounts or ["/"] +def collect_net_raw() -> dict: + """Return {iface: (rx_bytes, tx_bytes)} cumulative counters from /proc/net/dev. + + Skips loopback and veth* (container) interfaces to cut noise. Counters are + monotonic; per-second rates are derived from deltas in build_sample(). + """ + out: dict = {} + try: + text = _read_file(NETDEV_PATH) + except OSError: + return out + for line in text.splitlines(): + if ":" not in line: + continue + name, _, rest = line.partition(":") + name = name.strip() + if name == "lo" or name.startswith("veth"): + continue + f = rest.split() + if len(f) < 16: + continue + try: + out[name] = (int(f[0]), int(f[8])) # rx_bytes, tx_bytes + except ValueError: + continue + return out + + +def collect_diskio_raw() -> dict: + """Return {device: (read_bytes, write_bytes)} cumulative from /proc/diskstats. + + Whole disks only — loop/ram/sr/fd and partitions are skipped. Sectors are + 512 bytes. Counters are monotonic; rates derived from deltas. + """ + out: dict = {} + try: + text = _read_file(DISKSTATS_PATH) + except OSError: + return out + for line in text.splitlines(): + f = line.split() + if len(f) < 10: + continue + name = f[2] + if name.startswith(("loop", "ram", "sr", "fd")) or _PARTITION_RE.match(name): + continue + try: + out[name] = (int(f[5]) * 512, int(f[9]) * 512) # sectors_read, sectors_written + except ValueError: + continue + return out + + +def collect_temps() -> list: + """Return [{label, celsius}] from /sys/class/hwmon (best-effort, may be empty).""" + out: list = [] + try: + chips = os.listdir(HWMON_DIR) + except OSError: + return out + for hw in chips: + d = os.path.join(HWMON_DIR, hw) + chip = "" + try: + chip = _read_file(os.path.join(d, "name")).strip() + except OSError: + pass + try: + files = os.listdir(d) + except OSError: + continue + for fn in files: + if not (fn.startswith("temp") and fn.endswith("_input")): + continue + idx = fn[: -len("_input")] # e.g. "temp1" + try: + milli = int(_read_file(os.path.join(d, fn)).strip()) + except (OSError, ValueError): + continue + label = "" + try: + label = _read_file(os.path.join(d, idx + "_label")).strip() + except OSError: + pass + name = label or (f"{chip}_{idx}" if chip else idx) + out.append({"label": name, "celsius": round(milli / 1000.0, 1)}) + return out + + +def _parse_psi(text: str) -> dict: + """Parse a /proc/pressure/* file into {some_avg10, some_avg60, full_avg10, …}.""" + res: dict = {} + for line in text.splitlines(): + parts = line.split() + if not parts: + continue + kind = parts[0] # "some" | "full" + for p in parts[1:]: + for prefix, suffix in (("avg10=", "avg10"), ("avg60=", "avg60")): + if p.startswith(prefix): + try: + res[f"{kind}_{suffix}"] = float(p[len(prefix):]) + except ValueError: + pass + return res + + +def collect_psi() -> dict: + """Return pressure-stall (PSI) gauges, prefixed by resource (mem/cpu/io). + + Absent on kernels without CONFIG_PSI — returns only what exists. + """ + out: dict = {} + for res_name, fname in (("mem", "memory"), ("cpu", "cpu"), ("io", "io")): + try: + text = _read_file(os.path.join(PSI_DIR, fname)) + except OSError: + continue + for k, v in _parse_psi(text).items(): + out[f"{res_name}_{k}"] = v + return out + + +def _rates(cur: dict, prev: dict, dt: float) -> dict: + """Per-second rates for counters present in both snapshots over dt seconds. + + Skips names missing from prev and negative deltas (counter reset / iface or + disk hot-plug), so a reboot never emits a bogus spike. + """ + out: dict = {} + if dt <= 0: + return out + for name, cur_vals in cur.items(): + if name not in prev: + continue + prev_vals = prev[name] + deltas = [c - p for c, p in zip(cur_vals, prev_vals)] + if any(d < 0 for d in deltas): + continue + out[name] = tuple(d / dt for d in deltas) + return out + + # ─── ring buffer ───────────────────────────────────────────────────────────── @@ -232,11 +408,17 @@ class RingBuffer: # ─── payload ───────────────────────────────────────────────────────────────── -def build_sample(mounts: list[str]) -> dict: - """Collect one full sample. Partial samples allowed if a collector fails.""" +def build_sample(mounts: list[str], state: dict) -> dict: + """Collect one full sample. Partial samples allowed if a collector fails. + + `state` carries the previous network/disk counters + monotonic timestamp so + throughput rates can be derived from deltas; it is mutated in place. + """ sample: dict = {"ts": datetime.now(timezone.utc).isoformat()} try: - sample["cpu_pct"] = collect_cpu() + agg, cores = collect_cpu() + sample["cpu_pct"] = agg + sample["cpu_cores"] = cores except Exception: sample["cpu_pct"] = None try: @@ -255,6 +437,35 @@ def build_sample(mounts: list[str]) -> dict: sample["storage"] = collect_storage(mounts) except Exception: sample["storage"] = [] + try: + sample["temps"] = collect_temps() + except Exception: + sample["temps"] = [] + try: + sample["psi"] = collect_psi() + except Exception: + sample["psi"] = {} + + # Throughput rates: diff cumulative counters against the previous sample. + now_mono = time.monotonic() + try: + net_raw = collect_net_raw() + except Exception: + net_raw = {} + try: + disk_raw = collect_diskio_raw() + except Exception: + disk_raw = {} + dt = now_mono - state["mono"] if state.get("mono") else 0.0 + sample["net"] = [ + {"iface": n, "rx_bps": round(rx, 1), "tx_bps": round(tx, 1)} + for n, (rx, tx) in _rates(net_raw, state.get("net", {}), dt).items() + ] + sample["diskio"] = [ + {"device": n, "read_bps": round(rd, 1), "write_bps": round(wr, 1)} + for n, (rd, wr) in _rates(disk_raw, state.get("disk", {}), dt).items() + ] + state["net"], state["disk"], state["mono"] = net_raw, disk_raw, now_mono return sample @@ -338,6 +549,8 @@ def main_loop(conf_path: str) -> int: mounts = cfg.get("mounts") or default_mounts() buffer = RingBuffer(maxlen=20) backoff = 0 + # Carries previous net/disk counters + monotonic ts for rate computation. + rate_state: dict = {} _log("INFO", f"steward-host-agent {AGENT_VERSION} starting " f"(url={cfg['url']}, interval={cfg['interval_seconds']}s)") @@ -353,7 +566,7 @@ def main_loop(conf_path: str) -> int: _log("ERROR", f"reload failed: {e}") _reload_requested = False - sample = build_sample(mounts) + sample = build_sample(mounts, rate_state) buffered = buffer.drain() payload = build_payload( samples=buffered + [sample], diff --git a/plugins/host_agent/plugin.yaml b/plugins/host_agent/plugin.yaml index d6716e8..ab23525 100644 --- a/plugins/host_agent/plugin.yaml +++ b/plugins/host_agent/plugin.yaml @@ -1,7 +1,7 @@ # plugins/host_agent/plugin.yaml name: host_agent -version: "1.1.0" -description: "Remote Linux host resource monitoring via a lightweight Python push agent (CPU, memory, storage, load, uptime)" +version: "1.2.0" +description: "Remote Linux host resource monitoring via a lightweight Python push agent (CPU incl. per-core, memory + PSI, storage, disk I/O, network throughput, load, temperatures, uptime)" author: "Steward" license: "MIT" min_app_version: "0.1.0" diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 584db94..8b7cf96 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -95,6 +95,60 @@ def _expand_sample_to_metrics( if sample.get("storage"): row("disk_used_pct_worst", host_name, worst_pct) + # Per-core CPU. Sub-resources carry a ':' so the fleet widget filters them out. + for i, core_pct in enumerate(sample.get("cpu_cores") or []): + if core_pct is not None: + row("cpu_pct", f"{host_name}:core{i}", core_pct) + + # Richer memory breakdown. + if mem.get("cached_bytes") is not None: + row("mem_cached_bytes", host_name, mem["cached_bytes"]) + if mem.get("buffers_bytes") is not None: + row("mem_buffers_bytes", host_name, mem["buffers_bytes"]) + + # Network throughput — per interface plus a host-level total for the fleet view. + net_rx_total = net_tx_total = 0.0 + for iface in sample.get("net") or []: + rx, tx = iface.get("rx_bps", 0.0), iface.get("tx_bps", 0.0) + res = f"{host_name}:net:{iface['iface']}" + row("net_rx_bps", res, rx) + row("net_tx_bps", res, tx) + net_rx_total += rx + net_tx_total += tx + if sample.get("net"): + row("net_rx_bps", host_name, net_rx_total) + row("net_tx_bps", host_name, net_tx_total) + + # Disk I/O — per device plus host-level total. + dr_total = dw_total = 0.0 + for dev in sample.get("diskio") or []: + rd, wr = dev.get("read_bps", 0.0), dev.get("write_bps", 0.0) + res = f"{host_name}:diskio:{dev['device']}" + row("disk_read_bps", res, rd) + row("disk_write_bps", res, wr) + dr_total += rd + dw_total += wr + if sample.get("diskio"): + row("disk_read_bps", host_name, dr_total) + row("disk_write_bps", host_name, dw_total) + + # Temperatures — per sensor plus the host max for at-a-glance. + max_temp: float | None = None + for t in sample.get("temps") or []: + c = t.get("celsius") + if c is None: + continue + row("temp_c", f"{host_name}:temp:{t['label']}", c) + if max_temp is None or c > max_temp: + max_temp = c + if max_temp is not None: + row("temp_c_max", host_name, max_temp) + + # Pressure stall information (PSI) — host-level gauges (mem/cpu/io some|full). + for k, v in (sample.get("psi") or {}).items(): + if isinstance(v, (int, float)): + row(f"psi_{k}", host_name, v) + return rows diff --git a/tests/plugins/host_agent/test_collectors.py b/tests/plugins/host_agent/test_collectors.py new file mode 100644 index 0000000..33370d3 --- /dev/null +++ b/tests/plugins/host_agent/test_collectors.py @@ -0,0 +1,96 @@ +"""Unit tests for the host agent's new metric collectors / parsers. + +Pure parsers and the rate helper are tested directly; the file-reading +collectors are tested by monkeypatching agent._read_file with fixtures. +""" +import plugins.host_agent.agent as agent + + +def test_parse_cpu_lines_aggregate_and_per_core(): + text = ( + "cpu 100 0 100 800 0 0 0\n" + "cpu0 50 0 50 400 0\n" + "cpu1 50 0 50 400 0\n" + "intr 12345\n" + ) + parsed = agent._parse_cpu_lines(text) + assert set(parsed) == {"cpu", "cpu0", "cpu1"} + assert parsed["cpu"] == (1000, 800) # total, idle(=idle+iowait) + + +def test_collect_cpu_returns_aggregate_and_cores(monkeypatch): + snapshots = iter([ + "cpu 100 0 100 800\ncpu0 50 0 50 400\ncpu1 50 0 50 400\n", + "cpu 200 0 200 1200\ncpu0 100 0 100 600\ncpu1 100 0 100 600\n", + ]) + monkeypatch.setattr(agent, "_read_file", lambda p: next(snapshots)) + monkeypatch.setattr(agent.time, "sleep", lambda s: None) + agg, cores = agent.collect_cpu() + assert isinstance(agg, float) + assert len(cores) == 2 + + +def test_parse_psi(): + text = ( + "some avg10=1.50 avg60=2.00 avg300=3.00 total=12345\n" + "full avg10=0.50 avg60=0.10 avg300=0.00 total=678\n" + ) + r = agent._parse_psi(text) + assert r["some_avg10"] == 1.5 + assert r["some_avg60"] == 2.0 + assert r["full_avg10"] == 0.5 + + +def test_rates_basic_and_counter_reset(): + cur = {"eth0": (2000, 1000), "sda": (5000, 0)} + prev = {"eth0": (1000, 500), "sda": (10000, 0)} # sda counter went backwards + out = agent._rates(cur, prev, dt=10.0) + assert out["eth0"] == (100.0, 50.0) + assert "sda" not in out # negative delta dropped + assert agent._rates(cur, prev, dt=0.0) == {} # no elapsed time → no rates + + +def test_rates_skips_names_missing_from_prev(): + assert agent._rates({"new": (5, 5)}, {}, dt=1.0) == {} + + +def test_collect_net_raw_skips_lo_and_veth(monkeypatch): + fixture = ( + "Inter-| Receive\n" + " face |bytes\n" + " lo: 100 1 0 0 0 0 0 0 200 2 0 0 0 0 0 0\n" + " eth0: 1000 5 0 0 0 0 0 0 500 4 0 0 0 0 0 0\n" + " veth9: 9 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0\n" + ) + monkeypatch.setattr(agent, "_read_file", lambda p: fixture) + assert agent.collect_net_raw() == {"eth0": (1000, 500)} + + +def test_collect_diskio_raw_whole_disks_only(monkeypatch): + # fields: major minor name reads rd_merged sectors_read ms ... sectors_written ... + fixture = ( + " 8 0 sda 1 0 10 0 1 0 20 0 0 0 0\n" + " 8 1 sda1 1 0 5 0 1 0 5 0 0 0 0\n" + " 259 0 nvme0n1 1 0 100 0 1 0 200 0 0 0 0\n" + " 7 0 loop0 1 0 1 0 1 0 1 0 0 0 0\n" + ) + monkeypatch.setattr(agent, "_read_file", lambda p: fixture) + out = agent.collect_diskio_raw() + assert out == {"sda": (10 * 512, 20 * 512), "nvme0n1": (100 * 512, 200 * 512)} + + +def test_build_sample_first_call_has_no_rates_but_keeps_state(monkeypatch): + monkeypatch.setattr(agent, "collect_cpu", lambda: (5.0, [5.0])) + monkeypatch.setattr(agent, "collect_memory", lambda: {"total_bytes": 10, "available_bytes": 5}) + monkeypatch.setattr(agent, "collect_load", lambda: {"1m": 0.0, "5m": 0.0, "15m": 0.0}) + monkeypatch.setattr(agent, "collect_uptime", lambda: 1) + monkeypatch.setattr(agent, "collect_storage", lambda m: []) + monkeypatch.setattr(agent, "collect_temps", lambda: []) + monkeypatch.setattr(agent, "collect_psi", lambda: {}) + monkeypatch.setattr(agent, "collect_net_raw", lambda: {"eth0": (1000, 500)}) + monkeypatch.setattr(agent, "collect_diskio_raw", lambda: {"sda": (10, 20)}) + state: dict = {} + s1 = agent.build_sample(["/"], state) + assert s1["net"] == [] and s1["diskio"] == [] # no prior counters yet + assert state["mono"] is not None + assert state["net"] == {"eth0": (1000, 500)} diff --git a/tests/plugins/host_agent/test_expand_metrics.py b/tests/plugins/host_agent/test_expand_metrics.py new file mode 100644 index 0000000..112dd97 --- /dev/null +++ b/tests/plugins/host_agent/test_expand_metrics.py @@ -0,0 +1,60 @@ +"""Unit tests for _expand_sample_to_metrics — sample dict → PluginMetric rows. + +No DB: PluginMetric objects are constructed in-memory, so we assert the +(metric_name, resource_name) → value contract directly. +""" +from datetime import datetime, timezone + +from plugins.host_agent.routes import _expand_sample_to_metrics + + +def _index(rows): + return {(r.metric_name, r.resource_name): r.value for r in rows} + + +def test_expand_emits_all_new_metric_families(): + sample = { + "ts": "2026-06-16T00:00:00+00:00", + "cpu_pct": 12.5, + "cpu_cores": [10.0, 15.0], + "mem": {"total_bytes": 100, "available_bytes": 40, "swap_used_bytes": 5, + "cached_bytes": 20, "buffers_bytes": 3}, + "load": {"1m": 0.1, "5m": 0.2, "15m": 0.3}, + "uptime_secs": 1000, + "storage": [{"mount": "/", "total_bytes": 100, "used_bytes": 50}], + "net": [{"iface": "eth0", "rx_bps": 1000.0, "tx_bps": 500.0}], + "diskio": [{"device": "sda", "read_bps": 2000.0, "write_bps": 1000.0}], + "temps": [{"label": "Package", "celsius": 45.0}, {"label": "Core0", "celsius": 50.0}], + "psi": {"mem_some_avg10": 1.5, "cpu_some_avg10": 0.0}, + } + idx = _index(_expand_sample_to_metrics(sample, "alpha", datetime.now(timezone.utc))) + + # per-core CPU as sub-resources + assert idx[("cpu_pct", "alpha:core0")] == 10.0 + assert idx[("cpu_pct", "alpha:core1")] == 15.0 + # richer memory + assert idx[("mem_cached_bytes", "alpha")] == 20 + assert idx[("mem_buffers_bytes", "alpha")] == 3 + # network: per-iface + host total + assert idx[("net_rx_bps", "alpha:net:eth0")] == 1000.0 + assert idx[("net_rx_bps", "alpha")] == 1000.0 + # disk I/O: per-device + host total + assert idx[("disk_read_bps", "alpha:diskio:sda")] == 2000.0 + assert idx[("disk_write_bps", "alpha")] == 1000.0 + # temps: per-sensor + host max + assert idx[("temp_c", "alpha:temp:Package")] == 45.0 + assert idx[("temp_c_max", "alpha")] == 50.0 + # PSI + assert idx[("psi_mem_some_avg10", "alpha")] == 1.5 + + +def test_expand_is_backcompat_with_old_agent_sample(): + """An old agent (no new keys) must still expand cleanly to legacy metrics.""" + sample = {"ts": "x", "cpu_pct": 5.0, "mem": {"total_bytes": 10, "available_bytes": 5}} + rows = _expand_sample_to_metrics(sample, "h", datetime.now(timezone.utc)) + names = {r.metric_name for r in rows} + assert "cpu_pct" in names + assert not any( + n.startswith(("net_", "disk_read", "disk_write", "psi_", "temp_", "mem_cached")) + for n in names + ) From f037b69c585c4bfbe9b60d2800bc714bafc68a35 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 10:15:43 -0400 Subject: [PATCH 037/126] test(host_agent): update existing collector tests for new agent API collect_cpu now returns (aggregate, [per_core]) and build_sample takes a rate-state dict, so the pre-existing tests that asserted the old float return / no-state signature needed updating. Mocks the new collectors (temps/psi/net/diskio) for determinism. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../host_agent/test_agent_build_payload.py | 19 ++++++++++++++----- .../host_agent/test_agent_collectors.py | 5 +++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/plugins/host_agent/test_agent_build_payload.py b/tests/plugins/host_agent/test_agent_build_payload.py index 2d7f26a..6e03452 100644 --- a/tests/plugins/host_agent/test_agent_build_payload.py +++ b/tests/plugins/host_agent/test_agent_build_payload.py @@ -9,14 +9,19 @@ def test_build_sample_shape(): fake_load = {"1m": 0.1, "5m": 0.2, "15m": 0.3} fake_storage = [{"mount": "/", "total_bytes": 1000, "used_bytes": 400}] - with patch.object(a, "collect_cpu", return_value=12.5), \ + with patch.object(a, "collect_cpu", return_value=(12.5, [12.0, 13.0])), \ patch.object(a, "collect_memory", return_value=fake_mem), \ patch.object(a, "collect_load", return_value=fake_load), \ patch.object(a, "collect_uptime", return_value=1234), \ - patch.object(a, "collect_storage", return_value=fake_storage): - sample = a.build_sample(mounts=["/"]) + patch.object(a, "collect_storage", return_value=fake_storage), \ + patch.object(a, "collect_temps", return_value=[]), \ + patch.object(a, "collect_psi", return_value={}), \ + patch.object(a, "collect_net_raw", return_value={}), \ + patch.object(a, "collect_diskio_raw", return_value={}): + sample = a.build_sample(["/"], {}) assert sample["cpu_pct"] == 12.5 + assert sample["cpu_cores"] == [12.0, 13.0] assert sample["mem"]["used_bytes"] == 40 assert sample["load"]["1m"] == 0.1 assert sample["uptime_secs"] == 1234 @@ -36,8 +41,12 @@ def test_build_sample_tolerates_collector_failure(): "available_bytes": 1, "swap_used_bytes": 0}), \ patch.object(a, "collect_load", return_value={"1m": 0.0, "5m": 0.0, "15m": 0.0}), \ patch.object(a, "collect_uptime", return_value=0), \ - patch.object(a, "collect_storage", return_value=[]): - sample = a.build_sample(mounts=["/"]) + patch.object(a, "collect_storage", return_value=[]), \ + patch.object(a, "collect_temps", return_value=[]), \ + patch.object(a, "collect_psi", return_value={}), \ + patch.object(a, "collect_net_raw", return_value={}), \ + patch.object(a, "collect_diskio_raw", return_value={}): + sample = a.build_sample(["/"], {}) assert sample["cpu_pct"] is None assert sample["mem"]["total_bytes"] == 1 diff --git a/tests/plugins/host_agent/test_agent_collectors.py b/tests/plugins/host_agent/test_agent_collectors.py index 9f3d1fa..44aa775 100644 --- a/tests/plugins/host_agent/test_agent_collectors.py +++ b/tests/plugins/host_agent/test_agent_collectors.py @@ -56,10 +56,11 @@ def test_collect_cpu_reads_twice(): with patch.object(a, "_read_file", _read), \ patch.object(a.time, "sleep", lambda _s: None): - pct = a.collect_cpu(sample_window=0.0) + pct, cores = a.collect_cpu(sample_window=0.0) # totals: 1000 → 2000 (delta 1000). idle+iowait: 850 → 1700 (delta 850). - # busy = 150. pct = 15.0. + # busy = 150. pct = 15.0. No cpuN lines in the fixture → no per-core values. assert pct == pytest.approx(15.0, abs=0.1) + assert cores == [] def test_collect_storage(): From bca1b92cc62aaac756a27fab1bf4418cb8fd7ec7 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 10:27:57 -0400 Subject: [PATCH 038/126] feat(host_agent): Netdata-style fleet overview + per-host detail view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surfaces the metrics the agent now collects (task #868). Adds a viewer-facing fleet page (/plugins/host_agent/) with per-host cards (CPU/mem/disk/load/temp, stale flag) and a per-host detail page (/plugins/host_agent//) — the link the fleet widget already pointed at but had no route for. Detail page shows current gauges (CPU, memory incl. swap/cache, load, network rx/tx, disk I/O, temp max, memory/cpu/io PSI), per-core CPU bars, per-mount filesystem bars, and per-interface/disk/sensor breakdowns, plus history charts (utilization %, throughput B/s, load & pressure) over a selectable range. Charts use a linear epoch-ms x-axis so no Chart.js date adapter is needed. Stale state uses the plugin's stale_after_seconds threshold. Repoints the host_resources widget detail link from admin settings to the new fleet page. Completes milestone #68 (task #867). Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 202 ++++++++++++-- plugins/host_agent/templates/host_detail.html | 251 ++++++++++++++++++ plugins/host_agent/templates/host_list.html | 43 +++ .../host_agent/templates/widget_table.html | 2 +- steward/core/widgets.py | 2 +- 5 files changed, 470 insertions(+), 30 deletions(-) create mode 100644 plugins/host_agent/templates/host_detail.html create mode 100644 plugins/host_agent/templates/host_list.html diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 8b7cf96..ed8ef20 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -11,10 +11,11 @@ import secrets from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for from steward.auth.middleware import require_role from steward.models.users import UserRole -from sqlalchemy import select, func +from sqlalchemy import select, func, or_ from datetime import timedelta from steward.core.settings import public_base_url +from steward.core.time_range import parse_range, RANGE_OPTIONS from steward.models.hosts import Host from steward.models.metrics import PluginMetric from .models import HostAgentRegistration @@ -299,35 +300,45 @@ async def agent_source(): return Response(body, mimetype="text/x-python") -@host_agent_bp.get("/widget") -async def widget_table(): - """Fleet-glance table: one row per monitored host, latest metrics.""" - async with current_app.db_sessionmaker() as session: - regs = (await session.execute(select(HostAgentRegistration))).scalars().all() - host_ids = [r.host_id for r in regs] - hosts = { - h.id: h for h in (await session.execute( - select(Host).where(Host.id.in_(host_ids)) - )).scalars().all() - } if host_ids else {} +def _stale_after_seconds() -> int: + return int( + current_app.config.get("PLUGINS", {}) + .get(SOURCE_MODULE, {}) + .get("stale_after_seconds", 180) + ) - subq = ( - select( - PluginMetric.resource_name, - PluginMetric.metric_name, - func.max(PluginMetric.recorded_at).label("max_ts"), - ) - .where(PluginMetric.source_module == SOURCE_MODULE) - .group_by(PluginMetric.resource_name, PluginMetric.metric_name) - ).subquery() - latest_rows = (await session.execute( - select(PluginMetric).join( - subq, - (PluginMetric.resource_name == subq.c.resource_name) & - (PluginMetric.metric_name == subq.c.metric_name) & - (PluginMetric.recorded_at == subq.c.max_ts), - ).where(PluginMetric.source_module == SOURCE_MODULE) + +async def _fleet_rows(session) -> list[dict]: + """One row per registered host with its latest host-level metrics + stale flag. + + Only bare host-level resources are used here (sub-resources carry a ':' and + are skipped) so the fleet glance stays one line per host. + """ + regs = (await session.execute(select(HostAgentRegistration))).scalars().all() + host_ids = [r.host_id for r in regs] + hosts = { + h.id: h for h in (await session.execute( + select(Host).where(Host.id.in_(host_ids)) )).scalars().all() + } if host_ids else {} + + subq = ( + select( + PluginMetric.resource_name, + PluginMetric.metric_name, + func.max(PluginMetric.recorded_at).label("max_ts"), + ) + .where(PluginMetric.source_module == SOURCE_MODULE) + .group_by(PluginMetric.resource_name, PluginMetric.metric_name) + ).subquery() + latest_rows = (await session.execute( + select(PluginMetric).join( + subq, + (PluginMetric.resource_name == subq.c.resource_name) & + (PluginMetric.metric_name == subq.c.metric_name) & + (PluginMetric.recorded_at == subq.c.max_ts), + ).where(PluginMetric.source_module == SOURCE_MODULE) + )).scalars().all() latest: dict[str, dict[str, float]] = {} for row in latest_rows: @@ -335,21 +346,46 @@ async def widget_table(): continue latest.setdefault(row.resource_name, {})[row.metric_name] = row.value + stale_after = _stale_after_seconds() + now = datetime.now(timezone.utc) rows = [] for reg in regs: host = hosts.get(reg.host_id) if host is None: continue m = latest.get(host.name, {}) + ls = reg.last_seen_at + stale = ls is None or (now - ls).total_seconds() > stale_after rows.append({ "host": host, "reg": reg, + "stale": stale, "cpu_pct": m.get("cpu_pct"), "mem_used_pct": m.get("mem_used_pct"), "disk_worst": m.get("disk_used_pct_worst"), "load_1m": m.get("load_1m"), + "temp_max": m.get("temp_c_max"), + "net_rx_bps": m.get("net_rx_bps"), + "net_tx_bps": m.get("net_tx_bps"), }) + rows.sort(key=lambda r: r["host"].name.lower()) + return rows + +@host_agent_bp.get("/") +@require_role(UserRole.viewer) +async def index(): + """Fleet overview page — cards per host linking to the detail view.""" + async with current_app.db_sessionmaker() as session: + rows = await _fleet_rows(session) + return await render_template("host_list.html", rows=rows) + + +@host_agent_bp.get("/widget") +async def widget_table(): + """Fleet-glance dashboard widget: one row per monitored host, latest metrics.""" + async with current_app.db_sessionmaker() as session: + rows = await _fleet_rows(session) return await render_template("widget_table.html", rows=rows) @@ -381,6 +417,116 @@ async def widget_history(): return await render_template("widget_history.html", host=host, series=series, hours=hours) +# Host-level metrics charted on the detail page (sub-resources are shown as +# current-value lists, not time series, to keep the page readable). +HISTORY_METRICS = ( + "cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m", + "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps", + "temp_c_max", "psi_mem_some_avg10", +) + + +async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]: + """{resource_name: {metric: value}} of the latest sample for a host + sub-resources.""" + subq = ( + select( + PluginMetric.resource_name, + PluginMetric.metric_name, + func.max(PluginMetric.recorded_at).label("max_ts"), + ) + .where(PluginMetric.source_module == SOURCE_MODULE) + .where(or_( + PluginMetric.resource_name == host_name, + PluginMetric.resource_name.like(host_name + ":%"), + )) + .group_by(PluginMetric.resource_name, PluginMetric.metric_name) + ).subquery() + rows = (await session.execute( + select(PluginMetric).join( + subq, + (PluginMetric.resource_name == subq.c.resource_name) & + (PluginMetric.metric_name == subq.c.metric_name) & + (PluginMetric.recorded_at == subq.c.max_ts), + ).where(PluginMetric.source_module == SOURCE_MODULE) + )).scalars().all() + out: dict[str, dict[str, float]] = {} + for r in rows: + out.setdefault(r.resource_name, {})[r.metric_name] = r.value + return out + + +async def _history_for_host(session, host_name: str, since) -> dict[str, list]: + """{metric: [[epoch_ms, value], …]} host-level series since `since`. + + Epoch-ms x values let the charts use a linear axis (no Chart.js date adapter). + """ + rows = (await session.execute( + select(PluginMetric).where( + PluginMetric.source_module == SOURCE_MODULE, + PluginMetric.resource_name == host_name, + PluginMetric.metric_name.in_(HISTORY_METRICS), + PluginMetric.recorded_at >= since, + ).order_by(PluginMetric.recorded_at) + )).scalars().all() + series: dict[str, list] = {m: [] for m in HISTORY_METRICS} + for p in rows: + series[p.metric_name].append([int(p.recorded_at.timestamp() * 1000), round(p.value, 2)]) + return series + + +@host_agent_bp.get("//") +@require_role(UserRole.viewer) +async def host_detail(host_id: str): + """Netdata-style per-host detail: current gauges + history charts.""" + since, range_key = parse_range(request.args.get("range")) + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return _error(404, "not_found") + reg = (await session.execute(select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + latest = await _latest_metrics_for_host(session, host.name) + series = await _history_for_host(session, host.name, since) + + hostlvl = latest.get(host.name, {}) + cores: list = [] + nets: dict = {} + disks_io: dict = {} + temps: dict = {} + mounts: dict = {} + plen = len(host.name) + 1 + for res, metrics in latest.items(): + if res == host.name: + continue + suffix = res[plen:] + if suffix.startswith("core"): + try: + idx = int(suffix[4:]) + except ValueError: + idx = 0 + cores.append((idx, metrics.get("cpu_pct"))) + elif suffix.startswith("net:"): + nets[suffix[len("net:"):]] = metrics + elif suffix.startswith("diskio:"): + disks_io[suffix[len("diskio:"):]] = metrics + elif suffix.startswith("temp:"): + temps[suffix[len("temp:"):]] = metrics.get("temp_c") + else: + mounts[suffix] = metrics + cores.sort(key=lambda c: c[0]) + + ls = reg.last_seen_at if reg else None + stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds() + + return await render_template( + "host_detail.html", + host=host, reg=reg, stale=stale, hostlvl=hostlvl, + cores=cores, nets=nets, disks_io=disks_io, temps=temps, mounts=mounts, + series=series, range_key=range_key, range_options=RANGE_OPTIONS, + ) + + def _new_token_pair() -> tuple[str, str]: raw = secrets.token_urlsafe(32) return raw, _hash_token(raw) diff --git a/plugins/host_agent/templates/host_detail.html b/plugins/host_agent/templates/host_detail.html new file mode 100644 index 0000000..1129759 --- /dev/null +++ b/plugins/host_agent/templates/host_detail.html @@ -0,0 +1,251 @@ +{% extends "base.html" %} +{% block title %}{{ host.name }} — Host Agent — Steward{% endblock %} + +{% macro fmt_bps(v) %} +{%- if v is none -%}— +{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s +{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s +{%- else -%}{{ "%.0f"|format(v) }} B/s +{%- endif -%} +{% endmacro %} +{% macro fmt_bytes(v) %} +{%- if v is none -%}— +{%- elif v >= 1073741824 -%}{{ "%.1f"|format(v / 1073741824) }} GB +{%- elif v >= 1048576 -%}{{ "%.0f"|format(v / 1048576) }} MB +{%- else -%}{{ "%.0f"|format(v / 1024) }} KB +{%- endif -%} +{% endmacro %} +{% macro bar(pct) %} +{%- set p = pct if pct is not none else 0 -%} +
+
+
+{% endmacro %} + +{% block content %} +
+
+ ← Fleet +

{{ host.name }}

+ {% if stale %}stale{% else %}live{% endif %} +
+
+ {% for r in range_options %} + {{ r }} + {% endfor %} +
+
+ +{# ── Identity / metadata ─────────────────────────────────────────────────── #} +
+ Address {{ host.address or "—" }} + {% if reg %} + {% if reg.distro %}OS {{ reg.distro }}{% endif %} + {% if reg.kernel %}Kernel {{ reg.kernel }}{% endif %} + {% if reg.arch %}Arch {{ reg.arch }}{% endif %} + {% if reg.agent_version %}Agent v{{ reg.agent_version }}{% endif %} + {% set up = hostlvl.get('uptime_secs') %} + {% if up %}Uptime {{ (up // 86400)|int }}d {{ ((up % 86400) // 3600)|int }}h{% endif %} + Last seen {{ reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") ~ " UTC" if reg.last_seen_at else "never" }} + {% else %} + No agent registration found for this host. + {% endif %} +
+ +{% if not hostlvl %} +
No metrics reported yet. Once the agent checks in, CPU, memory, network, disk I/O, temperatures, and pressure will appear here.
+{% else %} + +{# ── Current headline gauges ─────────────────────────────────────────────── #} +
+
+
CPU
+ {% set c = hostlvl.get('cpu_pct') %} + {% if c is not none %}{{ "%.0f"|format(c) }}%{% else %}—{% endif %} +
{{ bar(c) }}
+
+
+
Memory
+ {% set mp = hostlvl.get('mem_used_pct') %} + {% if mp is not none %}{{ "%.0f"|format(mp) }}%{% else %}—{% endif %} +
{{ bar(mp) }}
+
+ avail {{ fmt_bytes(hostlvl.get('mem_available_bytes')) }} · cache {{ fmt_bytes(hostlvl.get('mem_cached_bytes')) }} · swap {{ fmt_bytes(hostlvl.get('swap_used_bytes')) }} +
+
+
+
Load
+ {% if hostlvl.get('load_1m') is not none %}{{ "%.2f"|format(hostlvl.get('load_1m')) }}{% else %}—{% endif %} +
+ 5m {% if hostlvl.get('load_5m') is not none %}{{ "%.2f"|format(hostlvl.get('load_5m')) }}{% else %}—{% endif %} · + 15m {% if hostlvl.get('load_15m') is not none %}{{ "%.2f"|format(hostlvl.get('load_15m')) }}{% else %}—{% endif %} +
+
+
+
Network
+
+
↓ {{ fmt_bps(hostlvl.get('net_rx_bps')) }}
+
↑ {{ fmt_bps(hostlvl.get('net_tx_bps')) }}
+
+
+
+
Disk I/O
+
+
rd {{ fmt_bps(hostlvl.get('disk_read_bps')) }}
+
wr {{ fmt_bps(hostlvl.get('disk_write_bps')) }}
+
+
+ {% if hostlvl.get('temp_c_max') is not none %} +
+
Temp (max)
+ {% set t = hostlvl.get('temp_c_max') %} + {{ "%.0f"|format(t) }}°C +
+ {% endif %} + {% if hostlvl.get('psi_mem_some_avg10') is not none %} +
+
Pressure (10s)
+
+
mem {{ "%.1f"|format(hostlvl.get('psi_mem_some_avg10')) }}%
+ {% if hostlvl.get('psi_cpu_some_avg10') is not none %}
cpu {{ "%.1f"|format(hostlvl.get('psi_cpu_some_avg10')) }}%
{% endif %} + {% if hostlvl.get('psi_io_some_avg10') is not none %}
io {{ "%.1f"|format(hostlvl.get('psi_io_some_avg10')) }}%
{% endif %} +
+
+ {% endif %} +
+ +{# ── Per-core CPU ─────────────────────────────────────────────────────────── #} +{% if cores %} +
+
Per-core CPU
+
+ {% for idx, pct in cores %} +
+
+ core {{ idx }}{% if pct is not none %}{{ "%.0f"|format(pct) }}%{% else %}—{% endif %} +
+ {{ bar(pct) }} +
+ {% endfor %} +
+
+{% endif %} + +{# ── Filesystems ──────────────────────────────────────────────────────────── #} +{% if mounts %} +
+
Filesystems
+ {% for mount, m in mounts.items() %} +
+
+ {{ mount }} + {{ fmt_bytes(m.get('disk_used_bytes')) }} / {{ fmt_bytes(m.get('disk_total_bytes')) }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %} +
+ {{ bar(m.get('disk_used_pct')) }} +
+ {% endfor %} +
+{% endif %} + +{# ── Interfaces / disks / sensors current detail ──────────────────────────── #} +{% if nets or disks_io or temps %} +
+ {% if nets %} +
+
Interfaces
+ {% for iface, m in nets.items() %} +
+ {{ iface }} + ↓ {{ fmt_bps(m.get('net_rx_bps')) }} · ↑ {{ fmt_bps(m.get('net_tx_bps')) }} +
+ {% endfor %} +
+ {% endif %} + {% if disks_io %} +
+
Disks
+ {% for dev, m in disks_io.items() %} +
+ {{ dev }} + rd {{ fmt_bps(m.get('disk_read_bps')) }} · wr {{ fmt_bps(m.get('disk_write_bps')) }} +
+ {% endfor %} +
+ {% endif %} + {% if temps %} +
+
Temperatures
+ {% for label, c in temps.items() %} +
+ {{ label }} + {% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %} +
+ {% endfor %} +
+ {% endif %} +
+{% endif %} + +{# ── History charts ───────────────────────────────────────────────────────── #} +
+
+
Utilization % — last {{ range_key }}
+
+
+
+
Throughput (B/s) — last {{ range_key }}
+
+
+
+
Load & pressure — last {{ range_key }}
+
+
+
+ + +{% endif %} +{% endblock %} diff --git a/plugins/host_agent/templates/host_list.html b/plugins/host_agent/templates/host_list.html new file mode 100644 index 0000000..69aad8e --- /dev/null +++ b/plugins/host_agent/templates/host_list.html @@ -0,0 +1,43 @@ +{% extends "base.html" %} +{% block title %}Host Agents — Steward{% endblock %} +{% block content %} +
+

Host Agents

+ {% if session.user_role == 'admin' %} + Manage agents → + {% endif %} +
+ +{% if not rows %} +
+ No hosts are reporting agent metrics yet. + {% if session.user_role == 'admin' %} + Add one from Host Agent settings and run the install command on the target. + {% else %} + Ask an admin to install the agent on a host. + {% endif %} +
+{% else %} + +{% endif %} +{% endblock %} diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html index a646fad..c391b0c 100644 --- a/plugins/host_agent/templates/widget_table.html +++ b/plugins/host_agent/templates/widget_table.html @@ -2,7 +2,7 @@ {% if rows %}
{% for r in rows %} - {% set stale = not r.reg.last_seen_at %} + {% set stale = r.stale %}
Date: Tue, 16 Jun 2026 11:04:02 -0400 Subject: [PATCH 039/126] feat(ansible): scheduled recurring playbook runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds cron-like recurring runs (the engine the maintenance-automation work needs). New AnsibleSchedule model + migration 0018; a core ScheduledTask (ansible_scheduled_runs, 60s) fires due schedules, each creating a system-triggered AnsibleRun (triggered_by=None). Centralises the resolve-inventory → create-run → launch flow in ansible/runner.trigger_run, shared by the manual route (refactored to use it) and the scheduler. Schedules UI under /ansible/schedules: create/edit/pause/delete/run-now, with interval presets, scope targeting (all / group / target), extra-vars / limit / tags / dry-run, and last-run status (resolved via last_run_id) + next-run. Unit test for the due-check. Task #549 (milestone #37). Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/ansible/routes.py | 279 +++++++++++++----- steward/ansible/runner.py | 78 +++++ steward/ansible/scheduler.py | 66 +++++ steward/app.py | 15 + .../versions/0018_ansible_schedules.py | 36 +++ steward/models/__init__.py | 2 + steward/models/ansible_schedule.py | 34 +++ steward/templates/ansible/index.html | 5 +- steward/templates/ansible/schedules.html | 141 +++++++++ tests/test_ansible_schedule_due.py | 28 ++ 10 files changed, 604 insertions(+), 80 deletions(-) create mode 100644 steward/ansible/runner.py create mode 100644 steward/ansible/scheduler.py create mode 100644 steward/migrations/versions/0018_ansible_schedules.py create mode 100644 steward/models/ansible_schedule.py create mode 100644 steward/templates/ansible/schedules.html create mode 100644 tests/test_ansible_schedule_due.py diff --git a/steward/ansible/routes.py b/steward/ansible/routes.py index 02918a6..61ae3aa 100644 --- a/steward/ansible/routes.py +++ b/steward/ansible/routes.py @@ -1,20 +1,28 @@ # steward/ansible/routes.py from __future__ import annotations -import asyncio -import uuid -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from quart import ( - Blueprint, Response, current_app, render_template, - request, session, + Blueprint, Response, current_app, redirect, render_template, + request, session, url_for, ) from sqlalchemy import select from steward.ansible import executor, sources as src_module from steward.auth.middleware import require_role from steward.models.ansible import AnsibleRun, AnsibleRunStatus +from steward.models.ansible_schedule import AnsibleSchedule from steward.models.users import User, UserRole +INTERVAL_PRESETS = [ + (300, "Every 5 minutes"), + (3600, "Hourly"), + (21600, "Every 6 hours"), + (43200, "Every 12 hours"), + (86400, "Daily"), + (604800, "Weekly"), +] + ansible_bp = Blueprint("ansible", __name__, url_prefix="/ansible") @@ -85,11 +93,34 @@ async def view_playbook(source_name: str, playbook_path: str): ) +def _parse_run_params(form) -> tuple[dict | None, str | None]: + """Build the executor params dict from form fields. Returns (params, error).""" + extra_vars: list[str] = [] + for line in (form.get("extra_vars", "") or "").splitlines(): + line = line.strip() + if not line: + continue + if "=" not in line: + return None, f"Invalid extra var (expected key=value): {line!r}" + extra_vars.append(line) + params: dict = {} + if extra_vars: + params["extra_vars"] = extra_vars + limit = (form.get("limit", "") or "").strip() + if limit: + params["limit"] = limit + tags = (form.get("tags", "") or "").strip() + if tags: + params["tags"] = tags + if "check" in form: + params["check"] = True + return (params or None), None + + @ansible_bp.post("/runs") @require_role(UserRole.operator) async def create_run(): - import json as _json - from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory + from steward.ansible.runner import trigger_run form = await request.form playbook_path = form.get("playbook_path", "").strip() @@ -99,82 +130,22 @@ async def create_run(): if not playbook_path: return "playbook_path is required", 400 - all_sources = _get_sources() - source = next((s for s in all_sources if s["name"] == source_name), None) - if source is None: - return "Source not found", 404 + params_or_none, err = _parse_run_params(form) + if err: + return err, 400 - # Resolve inventory for this scope. - inventory_content: str | None = None - inventory_path: str | None = None - if inventory_scope.startswith("steward:"): - async with current_app.db_sessionmaker() as db: - targets = await fetch_scope_targets(db, inventory_scope) - inventory_content = _json.dumps(generate_inventory(targets)) - elif inventory_scope.startswith("repo:"): - parts = inventory_scope.split(":", 2) - inventory_path = parts[2] if len(parts) == 3 else "" - else: - return "Invalid inventory_scope", 400 - - # Optional run parameters (all passed as argv by the executor — no shell). - extra_vars: list[str] = [] - for line in (form.get("extra_vars", "") or "").splitlines(): - line = line.strip() - if not line: - continue - if "=" not in line: - return f"Invalid extra var (expected key=value): {line!r}", 400 - extra_vars.append(line) - limit = (form.get("limit", "") or "").strip() - tags = (form.get("tags", "") or "").strip() - check = "check" in form - - params: dict = {} - if extra_vars: - params["extra_vars"] = extra_vars - if limit: - params["limit"] = limit - if tags: - params["tags"] = tags - if check: - params["check"] = True - params_or_none = params or None - - run_id = str(uuid.uuid4()) - now = datetime.now(timezone.utc) - - run = AnsibleRun( - id=run_id, - playbook_path=playbook_path, - inventory_path=inventory_path, - inventory_scope=inventory_scope, + run, source, err = await trigger_run( + current_app._get_current_object(), # type: ignore[attr-defined] source_name=source_name, - triggered_by=session["user_id"], - status=AnsibleRunStatus.running, - started_at=now, + playbook_path=playbook_path, + inventory_scope=inventory_scope, params=params_or_none, + triggered_by=session["user_id"], ) - async with current_app.db_sessionmaker() as db: - async with db.begin(): - db.add(run) - - task = asyncio.create_task( - executor.start_run( - current_app._get_current_object(), # type: ignore[attr-defined] - run_id, - playbook_path, - inventory_path or "", - source["path"], - params_or_none, - inventory_content, - ) - ) - task.add_done_callback( - lambda t: t.exception() and current_app.logger.error( - "Ansible run %s raised: %s", run_id, t.exception() - ) - ) + if err == "Source not found": + return err, 404 + if err: + return err, 400 return await render_template( "ansible/run_started.html", @@ -233,3 +204,153 @@ async def run_detail(run_id: str): triggered_label = res.scalar_one_or_none() or "(deleted user)" return await render_template( "ansible/run_detail.html", run=run, triggered_label=triggered_label) + + +# ── Scheduled recurring runs ────────────────────────────────────────────────── + +@ansible_bp.get("/schedules") +@require_role(UserRole.viewer) +async def schedules(): + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + + editing_id = request.args.get("edit") + async with current_app.db_sessionmaker() as db: + scheds = (await db.execute( + select(AnsibleSchedule).order_by(AnsibleSchedule.name))).scalars().all() + groups = (await db.execute( + select(AnsibleGroup).order_by(AnsibleGroup.name))).scalars().all() + targets = (await db.execute( + select(AnsibleTarget).order_by(AnsibleTarget.name))).scalars().all() + run_ids = [s.last_run_id for s in scheds if s.last_run_id] + runs = { + r.id: r for r in (await db.execute( + select(AnsibleRun).where(AnsibleRun.id.in_(run_ids)))).scalars().all() + } if run_ids else {} + + source_data = [ + {"name": s["name"], "playbooks": src_module.discover_playbooks(s["path"])} + for s in _get_sources() + ] + all_playbooks = sorted({p for sd in source_data for p in sd["playbooks"]}) + + rows = [] + editing = None + for s in scheds: + rows.append({ + "s": s, + "last_run": runs.get(s.last_run_id) if s.last_run_id else None, + "next_run": (s.last_run_at + timedelta(seconds=s.interval_seconds)) if s.last_run_at else None, + }) + if editing_id and s.id == editing_id: + editing = s + + return await render_template( + "ansible/schedules.html", + rows=rows, groups=groups, targets=targets, + source_data=source_data, all_playbooks=all_playbooks, + editing=editing, interval_presets=INTERVAL_PRESETS, + ) + + +def _schedule_form_fields(form) -> tuple[dict | None, str | None]: + """Validate + extract schedule fields from a form. Returns (fields, error).""" + name = (form.get("name", "") or "").strip() + source_name = (form.get("source_name", "") or "").strip() + playbook_path = (form.get("playbook_path", "") or "").strip() + inventory_scope = (form.get("inventory_scope", "steward:all") or "steward:all").strip() + try: + interval = int(form.get("interval_seconds", "0")) + except (TypeError, ValueError): + interval = 0 + if not name or not source_name or not playbook_path or interval <= 0: + return None, "name, source, playbook, and a positive interval are required" + params, err = _parse_run_params(form) + if err: + return None, err + return { + "name": name, "source_name": source_name, "playbook_path": playbook_path, + "inventory_scope": inventory_scope, "interval_seconds": interval, "params": params, + }, None + + +@ansible_bp.post("/schedules") +@require_role(UserRole.operator) +async def create_schedule(): + form = await request.form + fields, err = _schedule_form_fields(form) + if err: + return err, 400 + async with current_app.db_sessionmaker() as db: + async with db.begin(): + db.add(AnsibleSchedule(enabled=True, **fields)) + return redirect(url_for("ansible.schedules")) + + +@ansible_bp.post("/schedules/") +@require_role(UserRole.operator) +async def update_schedule(schedule_id: str): + form = await request.form + fields, err = _schedule_form_fields(form) + if err: + return err, 400 + async with current_app.db_sessionmaker() as db: + async with db.begin(): + sched = await db.get(AnsibleSchedule, schedule_id) + if sched is None: + return "Not found", 404 + for k, v in fields.items(): + setattr(sched, k, v) + return redirect(url_for("ansible.schedules")) + + +@ansible_bp.post("/schedules//toggle") +@require_role(UserRole.operator) +async def toggle_schedule(schedule_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + sched = await db.get(AnsibleSchedule, schedule_id) + if sched is None: + return "Not found", 404 + sched.enabled = not sched.enabled + return redirect(url_for("ansible.schedules")) + + +@ansible_bp.post("/schedules//delete") +@require_role(UserRole.operator) +async def delete_schedule(schedule_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + sched = await db.get(AnsibleSchedule, schedule_id) + if sched is not None: + await db.delete(sched) + return redirect(url_for("ansible.schedules")) + + +@ansible_bp.post("/schedules//run-now") +@require_role(UserRole.operator) +async def run_schedule_now(schedule_id: str): + from steward.ansible.runner import trigger_run + + async with current_app.db_sessionmaker() as db: + sched = await db.get(AnsibleSchedule, schedule_id) + if sched is None: + return "Not found", 404 + source_name, playbook_path = sched.source_name, sched.playbook_path + inventory_scope, params = sched.inventory_scope, sched.params + + run, _source, err = await trigger_run( + current_app._get_current_object(), # type: ignore[attr-defined] + source_name=source_name, playbook_path=playbook_path, + inventory_scope=inventory_scope, params=params, + triggered_by=session["user_id"], + ) + async with current_app.db_sessionmaker() as db: + async with db.begin(): + fresh = await db.get(AnsibleSchedule, schedule_id) + if fresh: + fresh.last_run_at = datetime.now(timezone.utc) + fresh.last_run_id = run.id if run else None + fresh.last_error = err + if run: + return redirect(url_for("ansible.run_detail", run_id=run.id)) + return redirect(url_for("ansible.schedules")) diff --git a/steward/ansible/runner.py b/steward/ansible/runner.py new file mode 100644 index 0000000..1aada6b --- /dev/null +++ b/steward/ansible/runner.py @@ -0,0 +1,78 @@ +# steward/ansible/runner.py +"""Shared playbook-run launcher used by the manual route, alerts, and schedules. + +Centralises the resolve-inventory → create AnsibleRun → launch executor flow so +manual, alert-triggered, and scheduled runs all take the exact same path. +""" +from __future__ import annotations + +import asyncio +import json +import uuid +from datetime import datetime, timezone + +from steward.ansible import executor, sources as src_module +from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory +from steward.models.ansible import AnsibleRun, AnsibleRunStatus + + +async def trigger_run( + app, + *, + source_name: str, + playbook_path: str, + inventory_scope: str = "steward:all", + params: dict | None = None, + triggered_by: str | None = None, +): + """Resolve inventory for the scope, persist an AnsibleRun, and launch it. + + triggered_by=None marks a system/automated run (alerts, schedules). + Returns (run, source, error): on success error is None; on failure run is + None and error is a short human-readable reason. + """ + sources = src_module.get_sources(app.config.get("ANSIBLE", {})) + source = next((s for s in sources if s["name"] == source_name), None) + if source is None: + return None, None, "Source not found" + + inventory_content: str | None = None + inventory_path: str | None = None + if inventory_scope.startswith("steward:"): + async with app.db_sessionmaker() as db: + targets = await fetch_scope_targets(db, inventory_scope) + inventory_content = json.dumps(generate_inventory(targets)) + elif inventory_scope.startswith("repo:"): + parts = inventory_scope.split(":", 2) + inventory_path = parts[2] if len(parts) == 3 else "" + else: + return None, source, "Invalid inventory_scope" + + run_id = str(uuid.uuid4()) + run = AnsibleRun( + id=run_id, + playbook_path=playbook_path, + inventory_path=inventory_path, + inventory_scope=inventory_scope, + source_name=source_name, + triggered_by=triggered_by, + status=AnsibleRunStatus.running, + started_at=datetime.now(timezone.utc), + params=params or None, + ) + async with app.db_sessionmaker() as db: + async with db.begin(): + db.add(run) + + task = asyncio.create_task( + executor.start_run( + app, run_id, playbook_path, inventory_path or "", + source["path"], params or None, inventory_content, + ) + ) + task.add_done_callback( + lambda t: t.exception() and app.logger.error( + "Ansible run %s raised: %s", run_id, t.exception() + ) + ) + return run, source, None diff --git a/steward/ansible/scheduler.py b/steward/ansible/scheduler.py new file mode 100644 index 0000000..f4efaff --- /dev/null +++ b/steward/ansible/scheduler.py @@ -0,0 +1,66 @@ +# steward/ansible/scheduler.py +"""Fire due Ansible schedules. Driven by the core ScheduledTask loop (~60s).""" +from __future__ import annotations + +import logging +from datetime import datetime, timezone + +from sqlalchemy import select + +from steward.ansible.runner import trigger_run +from steward.models.ansible_schedule import AnsibleSchedule + +logger = logging.getLogger(__name__) + + +def is_due(schedule: AnsibleSchedule, now: datetime) -> bool: + """True when an enabled schedule has never run or its interval has elapsed. + + Cadence is measured from the previous *fire* time (last_run_at), not run + completion, so a long-running playbook doesn't cause catch-up storms. + """ + if not schedule.enabled: + return False + if schedule.last_run_at is None: + return True + return (now - schedule.last_run_at).total_seconds() >= schedule.interval_seconds + + +async def run_due_schedules(app) -> int: + """Trigger every due schedule once. Returns the number launched.""" + now = datetime.now(timezone.utc) + async with app.db_sessionmaker() as db: + schedules = (await db.execute( + select(AnsibleSchedule).where(AnsibleSchedule.enabled.is_(True)) + )).scalars().all() + + fired = 0 + for s in schedules: + if not is_due(s, now): + continue + run = None + err: str | None = None + try: + run, _source, err = await trigger_run( + app, + source_name=s.source_name, + playbook_path=s.playbook_path, + inventory_scope=s.inventory_scope, + params=s.params, + triggered_by=None, + ) + except Exception: + logger.exception("Schedule %s (%s) failed to launch", s.id, s.name) + err = "launch error" + + async with app.db_sessionmaker() as db: + async with db.begin(): + fresh = await db.get(AnsibleSchedule, s.id) + if fresh: + fresh.last_run_at = now + fresh.last_run_id = run.id if run else None + fresh.last_error = err + if run: + fired += 1 + logger.info("Schedule %s (%s) fired run %s", s.id, s.name, run.id) + return fired diff --git a/steward/app.py b/steward/app.py index f488b87..d5bcf06 100644 --- a/steward/app.py +++ b/steward/app.py @@ -284,6 +284,21 @@ def _register_core_tasks(app: Quart) -> None: ) ) + # Fire due Ansible schedules (recurring playbook runs). Checks every minute; + # each schedule's own interval gates whether it actually runs. + async def run_ansible_schedules(): + from .ansible.scheduler import run_due_schedules + await run_due_schedules(app) + + app._task_registry.append( + ScheduledTask( + name="ansible_scheduled_runs", + coro_factory=run_ansible_schedules, + interval_seconds=60, + run_on_startup=False, + ) + ) + async def _mark_interrupted_runs(app: Quart) -> None: from sqlalchemy import update diff --git a/steward/migrations/versions/0018_ansible_schedules.py b/steward/migrations/versions/0018_ansible_schedules.py new file mode 100644 index 0000000..74e49d5 --- /dev/null +++ b/steward/migrations/versions/0018_ansible_schedules.py @@ -0,0 +1,36 @@ +"""Ansible schedules table — recurring playbook runs + +Revision ID: 0018_ansible_schedules +Revises: 0017_ansible_run_scope +Create Date: 2026-06-16 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "0018_ansible_schedules" +down_revision: Union[str, None] = "0017_ansible_run_scope" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "ansible_schedules", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("name", sa.String(128), nullable=False), + sa.Column("source_name", sa.String(128), nullable=False), + sa.Column("playbook_path", sa.String(512), nullable=False), + sa.Column("inventory_scope", sa.String(255), nullable=False, server_default="steward:all"), + sa.Column("params", sa.JSON(), nullable=True), + sa.Column("interval_seconds", sa.Integer(), nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("last_run_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_run_id", sa.String(36), nullable=True), + sa.Column("last_error", sa.String(255), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("ansible_schedules") diff --git a/steward/models/__init__.py b/steward/models/__init__.py index d8502b5..5f84866 100644 --- a/steward/models/__init__.py +++ b/steward/models/__init__.py @@ -6,6 +6,7 @@ from .metrics import PluginMetric from .alerts import AlertRule, AlertState, AlertEvent, AlertOperator, AlertStateEnum from .ansible import AnsibleRun, AnsibleRunStatus from .ansible_inventory import AnsibleTarget, AnsibleGroup, ansible_target_groups +from .ansible_schedule import AnsibleSchedule from .settings import AppSetting from .dashboard import Dashboard, DashboardWidget, DashboardShareToken @@ -17,6 +18,7 @@ __all__ = [ "AlertRule", "AlertState", "AlertEvent", "AlertOperator", "AlertStateEnum", "AnsibleRun", "AnsibleRunStatus", "AnsibleTarget", "AnsibleGroup", "ansible_target_groups", + "AnsibleSchedule", "AppSetting", "Dashboard", "DashboardWidget", "DashboardShareToken", ] diff --git a/steward/models/ansible_schedule.py b/steward/models/ansible_schedule.py new file mode 100644 index 0000000..9804c68 --- /dev/null +++ b/steward/models/ansible_schedule.py @@ -0,0 +1,34 @@ +from __future__ import annotations +import uuid +from datetime import datetime, timezone +from sqlalchemy import Boolean, DateTime, Integer, JSON, String +from sqlalchemy.orm import Mapped, mapped_column +from .base import Base + + +class AnsibleSchedule(Base): + """A recurring Ansible playbook run. + + Fired by the core scheduler (ansible_scheduled_runs task) on an interval; + each firing creates a system-triggered AnsibleRun (triggered_by=None). The + actual run outcome is read back via last_run_id → AnsibleRun.status, so we + only store provenance here, not a duplicated status. + """ + __tablename__ = "ansible_schedules" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + name: Mapped[str] = mapped_column(String(128), nullable=False) + source_name: Mapped[str] = mapped_column(String(128), nullable=False) + playbook_path: Mapped[str] = mapped_column(String(512), nullable=False) + inventory_scope: Mapped[str] = mapped_column(String(255), nullable=False, default="steward:all") + # {extra_vars: [...], limit, tags, check} — same shape as AnsibleRun.params. + params: Mapped[dict | None] = mapped_column(JSON, nullable=True) + interval_seconds: Mapped[int] = mapped_column(Integer, nullable=False) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + last_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_run_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + # Set only when a firing failed to launch (e.g. source vanished); else NULL. + last_error: Mapped[str | None] = mapped_column(String(255), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) + ) diff --git a/steward/templates/ansible/index.html b/steward/templates/ansible/index.html index 9807f46..c77be9a 100644 --- a/steward/templates/ansible/index.html +++ b/steward/templates/ansible/index.html @@ -3,7 +3,10 @@ {% block content %} {% if not runs %}
diff --git a/steward/templates/ansible/schedules.html b/steward/templates/ansible/schedules.html new file mode 100644 index 0000000..470fd6f --- /dev/null +++ b/steward/templates/ansible/schedules.html @@ -0,0 +1,141 @@ +{% extends "base.html" %} +{% block title %}Ansible Schedules — Steward{% endblock %} + +{% macro fmt_interval(secs) %} +{%- set m = {300: "every 5 min", 3600: "hourly", 21600: "every 6h", 43200: "every 12h", 86400: "daily", 604800: "weekly"} -%} +{{ m.get(secs, secs ~ "s") }} +{% endmacro %} + +{% block content %} +
+

Ansible Schedules

+ ← Runs +
+ +{# ── Existing schedules ───────────────────────────────────────────────────── #} +{% if not rows %} +
No schedules yet. Create one below to run a playbook on a recurring interval.
+{% else %} +
+ + + + + + + + + {% for row in rows %} + {% set s = row.s %} + + + + + + + + + + {% endfor %} + +
NamePlaybookTargetIntervalLast runNext run
{{ s.name }}{{ s.source_name }} / {{ s.playbook_path }}{{ s.inventory_scope }}{{ fmt_interval(s.interval_seconds) }} + {% if s.last_error %} + launch error + {% elif row.last_run %} + {% set sc = {"running":"var(--yellow)","success":"var(--green)","failed":"var(--red)","interrupted":"var(--orange)"}.get(row.last_run.status.value, "var(--text-muted)") %} + {{ row.last_run.status.value }} + · {{ s.last_run_at.strftime("%m-%d %H:%M") }} + {% else %} + never + {% endif %} + + {% if not s.enabled %}paused + {% elif row.next_run %}{{ row.next_run.strftime("%m-%d %H:%M") }} + {% else %}due now{% endif %} + +
+ +
+
+ +
+ Edit +
+ +
+
+
+{% endif %} + +{# ── Create / edit form ───────────────────────────────────────────────────── #} +{% set p = (editing.params or {}) if editing else {} %} +
+

{{ "Edit schedule" if editing else "New schedule" }}

+
+
+
+ + +
+
+ + +
+
+ + + + {% for pb in all_playbooks %}{% endfor %} + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+ + +
+
+ + {% if editing %}Cancel{% endif %} +
+
+
+{% endblock %} diff --git a/tests/test_ansible_schedule_due.py b/tests/test_ansible_schedule_due.py new file mode 100644 index 0000000..28d6658 --- /dev/null +++ b/tests/test_ansible_schedule_due.py @@ -0,0 +1,28 @@ +"""Unit tests for the schedule due-check (pure, no DB).""" +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +from steward.ansible.scheduler import is_due + +NOW = datetime(2026, 6, 16, 12, 0, 0, tzinfo=timezone.utc) + + +def _sched(enabled=True, last=None, interval=3600): + return SimpleNamespace(enabled=enabled, last_run_at=last, interval_seconds=interval) + + +def test_never_run_is_due(): + assert is_due(_sched(last=None), NOW) is True + + +def test_disabled_is_never_due(): + assert is_due(_sched(enabled=False, last=None), NOW) is False + + +def test_due_when_interval_elapsed(): + assert is_due(_sched(last=NOW - timedelta(seconds=3600), interval=3600), NOW) is True + assert is_due(_sched(last=NOW - timedelta(seconds=3700), interval=3600), NOW) is True + + +def test_not_due_before_interval(): + assert is_due(_sched(last=NOW - timedelta(seconds=1800), interval=3600), NOW) is False From 656bda2e3d9eb1a1d2aee7404a3c11a1341de294 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 11:07:34 -0400 Subject: [PATCH 040/126] feat(ansible): bundled first-party playbooks + built-in source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ships a read-only "steward-builtin" source (steward/ansible/bundled/) that always appears alongside operator-configured sources, with two playbooks: - maintenance/docker_prune.yml — docker system prune for swarm/standalone nodes (safe by default; prune_all_images / prune_volumes extra-vars to widen). Schedule it against a swarm-node group for recurring cleanup (#869). - host_agent/install.yml — installs/updates the host agent (mirrors install.sh: user, agent.py, config, hardened systemd unit), parameterised with steward_url + steward_token extra-vars. get_sources() now prepends the builtin source. Tests updated to find the configured git source by name; added coverage that the bundled playbooks are discoverable. Tasks #869 + agent-install (milestone #37). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ansible/bundled/host_agent/install.yml | 93 +++++++++++++++++++ .../bundled/maintenance/docker_prune.yml | 26 ++++++ steward/ansible/sources.py | 19 +++- tests/core/test_builtin_source.py | 19 ++++ tests/core/test_git_askpass.py | 6 +- 5 files changed, 160 insertions(+), 3 deletions(-) create mode 100644 steward/ansible/bundled/host_agent/install.yml create mode 100644 steward/ansible/bundled/maintenance/docker_prune.yml create mode 100644 tests/core/test_builtin_source.py diff --git a/steward/ansible/bundled/host_agent/install.yml b/steward/ansible/bundled/host_agent/install.yml new file mode 100644 index 0000000..bb39dd4 --- /dev/null +++ b/steward/ansible/bundled/host_agent/install.yml @@ -0,0 +1,93 @@ +--- +# Install (or update) the Steward host agent on a target. Mirrors the +# curl-based install.sh. Pass the per-host registration token as extra-vars: +# steward_url=https://steward.example steward_token= +# Re-running updates agent.py in place and restarts the service. +- name: Install Steward host agent + hosts: all + become: true + gather_facts: false + vars: + steward_url: "" + steward_token: "" + agent_interval: 30 + tasks: + - name: Require steward_url and steward_token + ansible.builtin.assert: + that: + - steward_url | length > 0 + - steward_token | length > 0 + fail_msg: "Pass steward_url and steward_token as extra-vars." + + - name: Create steward-agent system user + ansible.builtin.user: + name: steward-agent + system: true + shell: /usr/sbin/nologin + create_home: false + + - name: Create agent directory + ansible.builtin.file: + path: /usr/local/lib/steward-agent + state: directory + mode: "0755" + + - name: Download agent.py + ansible.builtin.get_url: + url: "{{ steward_url }}/plugins/host_agent/agent.py" + dest: /usr/local/lib/steward-agent/agent.py + mode: "0755" + force: true + notify: restart steward-agent + + - name: Write agent config + ansible.builtin.copy: + dest: /etc/steward-agent.conf + owner: root + group: steward-agent + mode: "0640" + content: | + url = {{ steward_url }} + token = {{ steward_token }} + interval_seconds = {{ agent_interval }} + notify: restart steward-agent + + - name: Install systemd unit + ansible.builtin.copy: + dest: /etc/systemd/system/steward-agent.service + mode: "0644" + content: | + [Unit] + Description=Steward host agent + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + User=steward-agent + Environment=STEWARD_AGENT_CONFIG=/etc/steward-agent.conf + ExecStart=/usr/bin/env python3 /usr/local/lib/steward-agent/agent.py + Restart=on-failure + RestartSec=5 + NoNewPrivileges=true + ProtectSystem=strict + ProtectHome=true + PrivateTmp=true + + [Install] + WantedBy=multi-user.target + notify: restart steward-agent + + - name: Enable and start the agent + ansible.builtin.systemd: + name: steward-agent + enabled: true + state: started + daemon_reload: true + + handlers: + - name: restart steward-agent + ansible.builtin.systemd: + name: steward-agent + state: restarted + daemon_reload: true diff --git a/steward/ansible/bundled/maintenance/docker_prune.yml b/steward/ansible/bundled/maintenance/docker_prune.yml new file mode 100644 index 0000000..1fcb8ec --- /dev/null +++ b/steward/ansible/bundled/maintenance/docker_prune.yml @@ -0,0 +1,26 @@ +--- +# Reclaim disk on Docker / Docker Swarm nodes by removing unused data. +# Safe by default: prunes dangling images, stopped containers, unused networks +# and build cache. Set extra-vars to widen scope: +# prune_all_images=true also remove ALL unused images (not just dangling) +# prune_volumes=true also remove unused named volumes (data loss risk) +- name: Docker system prune + hosts: all + gather_facts: false + become: true + vars: + prune_all_images: false + prune_volumes: false + tasks: + - name: Run docker system prune + ansible.builtin.command: + argv: >- + {{ ['docker', 'system', 'prune', '-f'] + + (['-a'] if prune_all_images | bool else []) + + (['--volumes'] if prune_volumes | bool else []) }} + register: prune_result + changed_when: "'Total reclaimed space: 0B' not in prune_result.stdout" + + - name: Report reclaimed space + ansible.builtin.debug: + msg: "{{ prune_result.stdout_lines | select | list }}" diff --git a/steward/ansible/sources.py b/steward/ansible/sources.py index e1b6855..04c0c7a 100644 --- a/steward/ansible/sources.py +++ b/steward/ansible/sources.py @@ -10,6 +10,22 @@ logger = logging.getLogger(__name__) INVENTORY_NAMES = {"hosts", "inventory", "inventory.yml", "inventory.ini"} +# Name of the always-present, read-only source of first-party playbooks shipped +# inside the app (maintenance tasks, host-agent install). Not operator-editable. +BUILTIN_SOURCE_NAME = "steward-builtin" + + +def _builtin_source() -> dict: + return { + "name": BUILTIN_SOURCE_NAME, + "type": "local", + "path": str((Path(__file__).parent / "bundled").resolve()), + "url": None, + "branch": "main", + "pull_interval_seconds": 0, + "http_token": "", + } + def get_sources(ansible_cfg: dict) -> list[dict]: """Return resolved source list from ansible config section. @@ -33,7 +49,8 @@ def get_sources(ansible_cfg: dict) -> list[dict]: """ sources = ansible_cfg.get("sources", []) cache_dir = ansible_cfg.get("cache_dir", "/var/cache/steward/ansible") - result = [] + # Always expose the bundled first-party playbooks as a source. + result = [_builtin_source()] for src in sources: src_type = src.get("type", "local") if src_type == "git": diff --git a/tests/core/test_builtin_source.py b/tests/core/test_builtin_source.py new file mode 100644 index 0000000..323590a --- /dev/null +++ b/tests/core/test_builtin_source.py @@ -0,0 +1,19 @@ +"""The bundled first-party playbook source is always present and discoverable.""" +from steward.ansible.sources import ( + BUILTIN_SOURCE_NAME, + discover_playbooks, + get_sources, +) + + +def test_builtin_source_is_first_and_local(): + sources = get_sources({"sources": []}) + assert sources[0]["name"] == BUILTIN_SOURCE_NAME + assert sources[0]["type"] == "local" + + +def test_bundled_playbooks_are_discoverable(): + builtin = get_sources({"sources": []})[0] + playbooks = discover_playbooks(builtin["path"]) + assert "maintenance/docker_prune.yml" in playbooks + assert "host_agent/install.yml" in playbooks diff --git a/tests/core/test_git_askpass.py b/tests/core/test_git_askpass.py index a2e9a8e..6fb9c53 100644 --- a/tests/core/test_git_askpass.py +++ b/tests/core/test_git_askpass.py @@ -22,7 +22,8 @@ def test_get_sources_includes_http_token(tmp_path): ], } sources = get_sources(cfg) - assert sources[0]["http_token"] == "mytoken123" + src = next(s for s in sources if s["name"] == "repo") + assert src["http_token"] == "mytoken123" def test_get_sources_missing_http_token_defaults_empty(tmp_path): @@ -36,7 +37,8 @@ def test_get_sources_missing_http_token_defaults_empty(tmp_path): ], } sources = get_sources(cfg) - assert sources[0]["http_token"] == "" + src = next(s for s in sources if s["name"] == "r") + assert src["http_token"] == "" @pytest.mark.asyncio From f3e919892d3e0bd2639d1c40176a7875bf489207 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 11:28:29 -0400 Subject: [PATCH 041/126] =?UTF-8?q?feat(plugins):=20plugin=20capability=20?= =?UTF-8?q?registry=20+=20host=5Fagent=E2=86=92Ansible=20deploy=20synergy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements #253's framework: a small core capability registry (steward/core/capabilities.py) where a module/plugin publishes a named, role-gated action and a consumer discovers it via has_capability() and runs it via invoke_capability() — no hard import, graceful degradation, permission propagation (actor role checked against the capability's required_role). Core publishes "ansible.run_playbook" (operator) wrapping ansible.runner. trigger_run (extended to accept a caller-built inventory). First consumer: the host_agent plugin gains "Deploy via Ansible" on its settings page — pick an inventory target/group and it installs/updates the agent via the bundled host_agent/install.yml, minting a fresh token per host and injecting it as an inventory hostvar (turning per-host curl|sh into one run). Exposed role ordering as middleware.role_meets. Unit tests for the registry + role checks. Task #253 (milestone #37). Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 116 +++++++++++++++++- .../host_agent/templates/settings_list.html | 31 +++++ steward/ansible/runner.py | 9 +- steward/app.py | 13 ++ steward/auth/middleware.py | 5 + steward/core/capabilities.py | 95 ++++++++++++++ tests/core/test_capabilities.py | 57 +++++++++ 7 files changed, 320 insertions(+), 6 deletions(-) create mode 100644 steward/core/capabilities.py create mode 100644 tests/core/test_capabilities.py diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index ed8ef20..07e672b 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -8,7 +8,7 @@ from typing import Any from pathlib import Path import secrets -from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for +from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for, session from steward.auth.middleware import require_role from steward.models.users import UserRole from sqlalchemy import select, func, or_ @@ -535,12 +535,22 @@ def _new_token_pair() -> tuple[str, str]: @host_agent_bp.get("/settings/") @require_role(UserRole.admin) async def settings_list(): - async with current_app.db_sessionmaker() as session: - regs = (await session.execute(select(HostAgentRegistration))).scalars().all() + from steward.core.capabilities import has_capability + from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup + + ansible_available = has_capability("ansible.run_playbook") + async with current_app.db_sessionmaker() as db: + regs = (await db.execute(select(HostAgentRegistration))).scalars().all() hosts_by_id = { - h.id: h for h in (await session.execute( + h.id: h for h in (await db.execute( select(Host).where(Host.id.in_([r.host_id for r in regs])))).scalars().all() } if regs else {} + targets, groups = [], [] + if ansible_available: + targets = (await db.execute( + select(AnsibleTarget).order_by(AnsibleTarget.name))).scalars().all() + groups = (await db.execute( + select(AnsibleGroup).order_by(AnsibleGroup.name))).scalars().all() new_token = request.args.get("new_token") new_host_id = request.args.get("host_id") @@ -555,6 +565,9 @@ async def settings_list(): ], new_token=new_token, install_url=install_url, + ansible_available=ansible_available, + deploy_targets=targets, + deploy_groups=groups, ) @@ -617,3 +630,98 @@ async def delete_registration(host_id: str): await session.delete(reg) await session.commit() return redirect(url_for("host_agent.settings_list")) + + +# ── Deploy via Ansible (plugin↔core synergy via the capability registry) ────── + +async def _ensure_host_for_target(db, target) -> "Host": + """Find or create the Host that an AnsibleTarget should report under.""" + if getattr(target, "host_id", None): + h = await db.get(Host, target.host_id) + if h: + return h + h = (await db.execute( + select(Host).where(Host.name == target.name))).scalar_one_or_none() + if h is None: + h = Host(name=target.name, address=getattr(target, "address", "") or "") + db.add(h) + await db.flush() + return h + + +async def _mint_registration_token(db, host) -> str: + """Create or rotate the host's agent registration; return the raw token. + + Tokens are stored hashed (unrecoverable), so deploying always mints a fresh + token — the run installs the agent with it. Existing agents get rotated. + """ + reg = (await db.execute(select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host.id))).scalar_one_or_none() + raw, hashed = _new_token_pair() + if reg is None: + db.add(HostAgentRegistration(host_id=host.id, token_hash=hashed)) + else: + reg.token_hash = hashed + reg.token_created_at = datetime.now(timezone.utc) + return raw + + +@host_agent_bp.post("/deploy") +@require_role(UserRole.admin) +async def deploy_via_ansible(): + """Install/update the agent on Ansible inventory targets via the bundled + install playbook — the 'curl | sh becomes a button' synergy. + + Uses the core "ansible.run_playbook" capability (no hard import of the + runner) and injects a freshly-minted per-host token as an inventory hostvar. + """ + import json as _json + from steward.core.capabilities import has_capability, invoke_capability + from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory + from steward.ansible.sources import BUILTIN_SOURCE_NAME + + if not has_capability("ansible.run_playbook"): + return _error(400, "ansible_unavailable", "Ansible is not available") + + form = await request.form + scope = (form.get("inventory_scope", "") or "").strip() + try: + interval = max(5, int(form.get("agent_interval", "30"))) + except (TypeError, ValueError): + interval = 30 + if not (scope.startswith("steward:target:") + or scope.startswith("steward:group:") + or scope == "steward:all"): + return _error(400, "bad_scope", "Choose a target or group") + + url = public_base_url(request) + async with current_app.db_sessionmaker() as db: + targets = await fetch_scope_targets(db, scope) + if not targets: + return _error(400, "no_targets", "No Ansible targets in that scope") + tokens: dict[str, str] = {} + async with db.begin(): + for t in targets: + host = await _ensure_host_for_target(db, t) + tokens[t.name] = await _mint_registration_token(db, host) + inv = generate_inventory(targets) + for name, tok in tokens.items(): + hv = inv["_meta"]["hostvars"].setdefault(name, {}) + hv["steward_url"] = url + hv["steward_token"] = tok + inventory_content = _json.dumps(inv) + + actor_role = UserRole(session.get("user_role", "viewer")) + run, _source, err = await invoke_capability( + "ansible.run_playbook", actor_role, + current_app._get_current_object(), # type: ignore[attr-defined] + source_name=BUILTIN_SOURCE_NAME, + playbook_path="host_agent/install.yml", + inventory_content=inventory_content, + inventory_scope=scope, + params={"extra_vars": [f"agent_interval={interval}"]}, + triggered_by=session.get("user_id"), + ) + if err: + return _error(400, "deploy_failed", err) + return redirect(url_for("ansible.run_detail", run_id=run.id)) diff --git a/plugins/host_agent/templates/settings_list.html b/plugins/host_agent/templates/settings_list.html index 49684f7..dc52e79 100644 --- a/plugins/host_agent/templates/settings_list.html +++ b/plugins/host_agent/templates/settings_list.html @@ -28,6 +28,37 @@
+{% if ansible_available %} +
+

Deploy via Ansible

+

+ Install (or update) the agent on Ansible inventory hosts in one run — no per-host curl | sh. + A fresh token is minted per host and injected into the run; existing agents are rotated to the new token. +

+ {% if not (deploy_targets or deploy_groups) %} +

+ No Ansible inventory targets yet. Add some under Ansible → Browse. +

+ {% else %} +
+
+ + +
+
+ + +
+ +
+ {% endif %} +
+{% endif %} +
diff --git a/steward/ansible/runner.py b/steward/ansible/runner.py index 1aada6b..a8ea6cd 100644 --- a/steward/ansible/runner.py +++ b/steward/ansible/runner.py @@ -24,10 +24,14 @@ async def trigger_run( inventory_scope: str = "steward:all", params: dict | None = None, triggered_by: str | None = None, + inventory_content: str | None = None, ): """Resolve inventory for the scope, persist an AnsibleRun, and launch it. triggered_by=None marks a system/automated run (alerts, schedules). + If inventory_content is provided, it is used verbatim and scope resolution + is skipped (the caller built a bespoke inventory — e.g. host_agent deploy + injecting per-host tokens); inventory_scope is still recorded for display. Returns (run, source, error): on success error is None; on failure run is None and error is a short human-readable reason. """ @@ -36,9 +40,10 @@ async def trigger_run( if source is None: return None, None, "Source not found" - inventory_content: str | None = None inventory_path: str | None = None - if inventory_scope.startswith("steward:"): + if inventory_content is not None: + pass # caller-supplied inventory wins; scope kept only for display + elif inventory_scope.startswith("steward:"): async with app.db_sessionmaker() as db: targets = await fetch_scope_targets(db, inventory_scope) inventory_content = json.dumps(generate_inventory(targets)) diff --git a/steward/app.py b/steward/app.py index d5bcf06..a97bd86 100644 --- a/steward/app.py +++ b/steward/app.py @@ -123,6 +123,19 @@ def create_app( register_status_source(ping_status_source) register_status_source(dns_status_source) + # Publish the Ansible "run a playbook" capability so plugins (e.g. host_agent + # auto-deploy) can drive runs without importing the runner. Ansible is core, + # so this is always available; consumers still gate on has_capability(). + from .core.capabilities import register_capability + from .ansible.runner import trigger_run + from .models.users import UserRole as _UserRole + register_capability( + "ansible.run_playbook", trigger_run, + label="Run Ansible playbook", + description="Launch an Ansible playbook run (manual, alert, schedule, or plugin-driven).", + required_role=_UserRole.operator, + ) + # ── 8. Build task registry ───────────────────────────────────────────────── app._task_registry = [] diff --git a/steward/auth/middleware.py b/steward/auth/middleware.py index d57bd34..6aca001 100644 --- a/steward/auth/middleware.py +++ b/steward/auth/middleware.py @@ -6,6 +6,11 @@ from steward.models.users import UserRole _ROLE_ORDER = [UserRole.viewer, UserRole.operator, UserRole.admin] +def role_meets(user_role: UserRole, minimum_role: UserRole) -> bool: + """True when user_role is at least minimum_role in the viewer= _ROLE_ORDER.index(minimum_role) + + def require_role(minimum_role: UserRole): """Decorator: requires authenticated user with at least minimum_role. Also allows access for validated share-token requests (viewer level only). diff --git a/steward/core/capabilities.py b/steward/core/capabilities.py new file mode 100644 index 0000000..df62b11 --- /dev/null +++ b/steward/core/capabilities.py @@ -0,0 +1,95 @@ +# steward/core/capabilities.py +"""Plugin/core capability registry — opportunistic, decoupled synergy. + +A capability is a named, permission-gated action that one part of the system +(core module or plugin) publishes and another can discover + invoke WITHOUT a +hard import. The publisher registers a callable under a string key; a consumer +checks `has_capability(key)` (graceful degradation — the synergy is a bonus, +never a requirement) and calls `invoke_capability(key, actor_role, ...)`. + +First consumer: the host_agent plugin invokes "ansible.run_playbook" to deploy +its agent via Ansible instead of importing the Ansible runner directly. + +Security: every capability declares a required_role; invoke_capability enforces +the caller's role meets it, so a low-privilege context can't drive a privileged +action in another module. +""" +from __future__ import annotations + +import inspect +import logging +from dataclasses import dataclass +from typing import Callable + +from steward.auth.middleware import role_meets +from steward.models.users import UserRole + +logger = logging.getLogger(__name__) + + +class CapabilityUnavailable(Exception): + """Raised when an unknown capability key is invoked.""" + + +class CapabilityForbidden(Exception): + """Raised when the actor's role is below the capability's required_role.""" + + +@dataclass +class Capability: + key: str + fn: Callable + label: str + description: str + required_role: UserRole + + +_CAPABILITIES: dict[str, Capability] = {} + + +def register_capability( + key: str, + fn: Callable, + *, + label: str, + description: str = "", + required_role: UserRole = UserRole.admin, +) -> None: + """Publish a capability. Last registration for a key wins (idempotent re-register).""" + _CAPABILITIES[key] = Capability(key, fn, label, description, required_role) + + +def has_capability(key: str) -> bool: + return key in _CAPABILITIES + + +def get_capability(key: str) -> Capability | None: + return _CAPABILITIES.get(key) + + +def list_capabilities() -> list[Capability]: + return list(_CAPABILITIES.values()) + + +def clear_capabilities() -> None: + """Reset the registry (tests).""" + _CAPABILITIES.clear() + + +async def invoke_capability(key: str, actor_role: UserRole, /, *args, **kwargs): + """Invoke a registered capability after a role check. Awaits async callables. + + Raises CapabilityUnavailable if the key isn't registered, CapabilityForbidden + if actor_role is insufficient. + """ + cap = _CAPABILITIES.get(key) + if cap is None: + raise CapabilityUnavailable(key) + if not role_meets(actor_role, cap.required_role): + raise CapabilityForbidden( + f"capability {key!r} requires role {cap.required_role.value}" + ) + result = cap.fn(*args, **kwargs) + if inspect.isawaitable(result): + result = await result + return result diff --git a/tests/core/test_capabilities.py b/tests/core/test_capabilities.py new file mode 100644 index 0000000..33e2677 --- /dev/null +++ b/tests/core/test_capabilities.py @@ -0,0 +1,57 @@ +"""Unit tests for the plugin/core capability registry.""" +import asyncio + +import pytest + +from steward.auth.middleware import role_meets +from steward.core.capabilities import ( + CapabilityForbidden, + CapabilityUnavailable, + clear_capabilities, + get_capability, + has_capability, + invoke_capability, + register_capability, +) +from steward.models.users import UserRole + + +def setup_function(): + clear_capabilities() + + +def test_register_has_get_defaults_to_admin(): + register_capability("x.do", lambda: "ok", label="Do X") + assert has_capability("x.do") + assert get_capability("x.do").label == "Do X" + assert get_capability("x.do").required_role == UserRole.admin + + +def test_invoke_sync_callable(): + register_capability("x.sum", lambda a, b: a + b, label="Sum", required_role=UserRole.viewer) + assert asyncio.run(invoke_capability("x.sum", UserRole.viewer, 2, 3)) == 5 + + +def test_invoke_async_callable(): + async def double(v): + return v * 2 + + register_capability("x.double", double, label="D", required_role=UserRole.viewer) + assert asyncio.run(invoke_capability("x.double", UserRole.operator, 4)) == 8 + + +def test_invoke_unknown_capability_raises(): + with pytest.raises(CapabilityUnavailable): + asyncio.run(invoke_capability("nope", UserRole.admin)) + + +def test_invoke_forbidden_when_role_too_low(): + register_capability("x.admin", lambda: 1, label="A", required_role=UserRole.admin) + with pytest.raises(CapabilityForbidden): + asyncio.run(invoke_capability("x.admin", UserRole.operator)) + + +def test_role_meets_ordering(): + assert role_meets(UserRole.admin, UserRole.operator) is True + assert role_meets(UserRole.operator, UserRole.operator) is True + assert role_meets(UserRole.viewer, UserRole.operator) is False From c10eae1c746357f20e4b4521e8a250aea64b3da5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 11:39:47 -0400 Subject: [PATCH 042/126] feat(ansible): in-app playbook authoring/editor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds create/edit/delete of playbooks from the UI (admin only), so a homelab user without a git workflow can author automation in-app. A new always-present writable local source "steward-local" (/data/ansible/playbooks, env-overridable, created on first save) is editable alongside operator local-dir sources; the bundled and git sources stay read-only (git is GitOps, clobbered on pull). sources.py: write_playbook / delete_playbook (traversal-guarded, .yml/.yaml only) + validate_playbook_yaml (YAML + play-list check) + is_editable_source. routes.py: /playbooks/new, /edit, /save, /delete (admin). Browse gains a "New playbook" button and per-playbook + view-page Edit/Delete for editable sources. Plain textarea editor with save-time YAML validation. Unit tests for write/delete/guard/validate. Task #579 — completes milestone #37 (Ansible automation). Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/ansible/routes.py | 101 ++++++++++++++++++ steward/ansible/sources.py | 87 ++++++++++++++- steward/templates/ansible/browse.html | 23 +++- .../templates/ansible/playbook_editor.html | 63 +++++++++++ tests/core/test_playbook_editor.py | 53 +++++++++ 5 files changed, 323 insertions(+), 4 deletions(-) create mode 100644 steward/templates/ansible/playbook_editor.html create mode 100644 tests/core/test_playbook_editor.py diff --git a/steward/ansible/routes.py b/steward/ansible/routes.py index 61ae3aa..c71e05d 100644 --- a/steward/ansible/routes.py +++ b/steward/ansible/routes.py @@ -56,6 +56,7 @@ async def browse(): "source": source, "playbooks": playbooks, "inventories": inventories, + "editable": src_module.is_editable_source(source), }) async with current_app.db_sessionmaker() as db: @@ -90,6 +91,7 @@ async def view_playbook(source_name: str, playbook_path: str): view_source=source_name, view_path=playbook_path, view_contents=contents, + view_editable=src_module.is_editable_source(source), ) @@ -354,3 +356,102 @@ async def run_schedule_now(schedule_id: str): if run: return redirect(url_for("ansible.run_detail", run_id=run.id)) return redirect(url_for("ansible.schedules")) + + +# ── In-app playbook authoring/editor (admin only) ───────────────────────────── + +def _editable_sources() -> list[dict]: + return [s for s in _get_sources() if src_module.is_editable_source(s)] + + +def _editable_source(name: str) -> dict | None: + return next((s for s in _editable_sources() if s["name"] == name), None) + + +@ansible_bp.get("/playbooks/new") +@require_role(UserRole.admin) +async def new_playbook(): + sources = _editable_sources() + return await render_template( + "ansible/playbook_editor.html", + editable_sources=sources, editing=False, + source_name=(sources[0]["name"] if sources else ""), + playbook_path="", content=_NEW_PLAYBOOK_TEMPLATE, error=None, + ) + + +@ansible_bp.get("/playbooks/edit//") +@require_role(UserRole.admin) +async def edit_playbook(source_name: str, playbook_path: str): + source = _editable_source(source_name) + if source is None: + return "Source is not editable", 404 + content = src_module.read_playbook(source["path"], playbook_path) + if content is None: + return "Playbook not found", 404 + return await render_template( + "ansible/playbook_editor.html", + editable_sources=_editable_sources(), editing=True, + source_name=source_name, playbook_path=playbook_path, + content=content, error=None, + ) + + +@ansible_bp.post("/playbooks/save") +@require_role(UserRole.admin) +async def save_playbook(): + form = await request.form + source_name = (form.get("source_name", "") or "").strip() + playbook_path = (form.get("playbook_path", "") or "").strip() + content = form.get("content", "") or "" + editing = (form.get("editing", "") == "1") + + source = _editable_source(source_name) + err = None + if source is None: + err = "Source is not editable" + elif not playbook_path: + err = "Filename is required" + else: + ok, verr = src_module.validate_playbook_yaml(content) + if not ok: + err = verr + if err is None: + ok, werr = src_module.write_playbook(source["path"], playbook_path, content) + if not ok: + err = werr + + if err is not None: + return await render_template( + "ansible/playbook_editor.html", + editable_sources=_editable_sources(), editing=editing, + source_name=source_name, playbook_path=playbook_path, + content=content, error=err, + ), 400 + + return redirect(url_for( + "ansible.view_playbook", source_name=source_name, playbook_path=playbook_path)) + + +@ansible_bp.post("/playbooks/delete") +@require_role(UserRole.admin) +async def delete_playbook(): + form = await request.form + source = _editable_source((form.get("source_name", "") or "").strip()) + if source is None: + return "Source is not editable", 400 + ok, err = src_module.delete_playbook(source["path"], (form.get("playbook_path", "") or "").strip()) + if not ok: + return err, 400 + return redirect(url_for("ansible.browse")) + + +_NEW_PLAYBOOK_TEMPLATE = """\ +--- +- name: My playbook + hosts: all + gather_facts: false + tasks: + - name: Ping + ansible.builtin.ping: +""" diff --git a/steward/ansible/sources.py b/steward/ansible/sources.py index 04c0c7a..a74cdf9 100644 --- a/steward/ansible/sources.py +++ b/steward/ansible/sources.py @@ -14,6 +14,12 @@ INVENTORY_NAMES = {"hosts", "inventory", "inventory.yml", "inventory.ini"} # inside the app (maintenance tasks, host-agent install). Not operator-editable. BUILTIN_SOURCE_NAME = "steward-builtin" +# Always-present, WRITABLE local source where the in-app editor saves playbooks, +# so a homelab user with no git workflow can author automation. Lives on the +# persistent /data volume; created lazily on first save. +LOCAL_SOURCE_NAME = "steward-local" +USER_PLAYBOOK_DIR = os.environ.get("STEWARD_PLAYBOOK_DIR", "/data/ansible/playbooks") + def _builtin_source() -> dict: return { @@ -27,6 +33,24 @@ def _builtin_source() -> dict: } +def _local_source() -> dict: + return { + "name": LOCAL_SOURCE_NAME, + "type": "local", + "path": USER_PLAYBOOK_DIR, + "url": None, + "branch": "main", + "pull_interval_seconds": 0, + "http_token": "", + } + + +def is_editable_source(source: dict) -> bool: + """Editable in the in-app editor: local dir sources except the read-only + bundled one. Git sources are GitOps (clobbered on pull) so not editable.""" + return source.get("type") == "local" and source.get("name") != BUILTIN_SOURCE_NAME + + def get_sources(ansible_cfg: dict) -> list[dict]: """Return resolved source list from ansible config section. @@ -49,8 +73,8 @@ def get_sources(ansible_cfg: dict) -> list[dict]: """ sources = ansible_cfg.get("sources", []) cache_dir = ansible_cfg.get("cache_dir", "/var/cache/steward/ansible") - # Always expose the bundled first-party playbooks as a source. - result = [_builtin_source()] + # Always expose the bundled (read-only) + local (writable) first-party sources. + result = [_builtin_source(), _local_source()] for src in sources: src_type = src.get("type", "local") if src_type == "git": @@ -119,6 +143,65 @@ def read_playbook(source_path: str, relative_path: str) -> str | None: return target.read_text(errors="replace") +def _resolve_writable(source_path: str, relative_path: str) -> tuple[Path | None, str | None]: + """Resolve relative_path under source_path, guarding traversal + extension. + + Returns (target_path, None) on success or (None, error). The root need not + exist yet (steward-local is created on first save).""" + rel = relative_path.strip().lstrip("/") + if not rel: + return None, "Filename is required" + if not rel.endswith((".yml", ".yaml")): + return None, "Playbook filename must end in .yml or .yaml" + root = Path(source_path).resolve() + target = (root / rel).resolve() + try: + target.relative_to(root) + except ValueError: + return None, "Invalid path" + return target, None + + +def write_playbook(source_path: str, relative_path: str, content: str) -> tuple[bool, str | None]: + """Write a playbook into a source, creating parent dirs. Traversal-guarded.""" + target, err = _resolve_writable(source_path, relative_path) + if err: + return False, err + try: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + except OSError as exc: + return False, f"Could not write file: {exc}" + return True, None + + +def delete_playbook(source_path: str, relative_path: str) -> tuple[bool, str | None]: + """Delete a playbook from a source. Traversal-guarded; no-op if absent.""" + target, err = _resolve_writable(source_path, relative_path) + if err: + return False, err + try: + if target.exists(): + target.unlink() + except OSError as exc: + return False, f"Could not delete file: {exc}" + return True, None + + +def validate_playbook_yaml(content: str) -> tuple[bool, str | None]: + """Cheap save-time validation: parses as YAML and is a non-empty play list.""" + import yaml + try: + data = yaml.safe_load(content) + except yaml.YAMLError as exc: + return False, f"YAML error: {exc}" + if data is None: + return False, "Playbook is empty" + if not isinstance(data, list): + return False, "A playbook must be a list of plays (top-level YAML list)" + return True, None + + async def git_pull(source: dict) -> None: """Clone the git repo if absent; pull if already present. diff --git a/steward/templates/ansible/browse.html b/steward/templates/ansible/browse.html index 6591d79..bf04039 100644 --- a/steward/templates/ansible/browse.html +++ b/steward/templates/ansible/browse.html @@ -3,14 +3,30 @@ {% block content %}

Browse Playbooks

- ← Run History +
+ {% if session.user_role == 'admin' %} + New playbook + {% endif %} + ← Run History +
{% if view_contents is defined %}

{{ view_path }}

- ← Back to browse +
+ {% if view_editable and session.user_role == 'admin' %} + Edit +
+ + + + + {% endif %} + ← Back to browse + 
{{ view_contents }}
@@ -47,6 +63,9 @@ {% for run in runs %} - {% set status_color = {"running": "var(--yellow)", "success": "var(--green)", "failed": "var(--red)", "interrupted": "var(--orange)"}.get(run.status.value, "var(--text-muted)") %} + {% set status_color = {"queued": "var(--text-dim)", "running": "var(--yellow)", "success": "var(--green)", "failed": "var(--red)", "interrupted": "var(--orange)", "cancelled": "var(--orange)"}.get(run.status.value, "var(--text-muted)") %} diff --git a/steward/templates/ansible/run_detail.html b/steward/templates/ansible/run_detail.html index 8f03613..0545ac8 100644 --- a/steward/templates/ansible/run_detail.html +++ b/steward/templates/ansible/run_detail.html @@ -6,9 +6,17 @@
← Runs

Run Detail

+
+ {% if run.status.value in ("running", "queued") and session.user_role in ("operator", "admin") %} +
+ + + {% endif %} + Download log +
-{% set status_color = {"running": "var(--yellow)", "success": "var(--green)", "failed": "var(--red)", "interrupted": "var(--orange)"}.get(run.status.value, "var(--text-muted)") %} +{% set status_color = {"queued": "var(--text-dim)", "running": "var(--yellow)", "success": "var(--green)", "failed": "var(--red)", "interrupted": "var(--orange)", "cancelled": "var(--orange)"}.get(run.status.value, "var(--text-muted)") %}
ID{{ run.id }} @@ -38,11 +46,41 @@
+{% if run.results and run.results.hosts %} +
+
Host summary
+
{{ pb }} View + {% if sd.editable and session.user_role == 'admin' %} + Edit + {% endif %} +

{{ "Edit playbook" if editing else "New playbook" }}

+ ← Browse + + +{% if error %}
{{ error }}
{% endif %} + +{% if not editable_sources %} +
+ No writable playbook source is available. The built-in steward-local source should appear + automatically; you can also configure a local-dir source under Settings → Ansible. +
+{% else %} +
+ +
+
+ + {% if editing %} + + + {% else %} + + {% endif %} +
+
+ + +
+
+
+ + +

+ Validated as YAML on save. Runs as the configured Ansible user — admin only. +

+
+
+ + Cancel +
+
+ +{% if editing %} +
+ + + +
+{% endif %} +{% endif %} +{% endblock %} diff --git a/tests/core/test_playbook_editor.py b/tests/core/test_playbook_editor.py new file mode 100644 index 0000000..627b826 --- /dev/null +++ b/tests/core/test_playbook_editor.py @@ -0,0 +1,53 @@ +"""Unit tests for in-app playbook authoring helpers (write/delete/validate/guard).""" +from steward.ansible.sources import ( + BUILTIN_SOURCE_NAME, + LOCAL_SOURCE_NAME, + delete_playbook, + get_sources, + is_editable_source, + read_playbook, + validate_playbook_yaml, + write_playbook, +) + + +def test_write_then_read_roundtrip(tmp_path): + ok, err = write_playbook(str(tmp_path), "maint/x.yml", "- name: p\n hosts: all\n") + assert ok and err is None + assert read_playbook(str(tmp_path), "maint/x.yml").startswith("- name: p") + + +def test_write_rejects_non_yaml_extension(tmp_path): + ok, err = write_playbook(str(tmp_path), "evil.sh", "rm -rf /") + assert not ok and "yml" in err + + +def test_write_blocks_path_traversal(tmp_path): + ok, err = write_playbook(str(tmp_path), "../escape.yml", "- {}") + assert not ok and err == "Invalid path" + + +def test_delete_guarded_and_idempotent(tmp_path): + write_playbook(str(tmp_path), "a.yml", "- {}") + assert delete_playbook(str(tmp_path), "a.yml")[0] is True + assert delete_playbook(str(tmp_path), "a.yml")[0] is True # already gone, no error + ok, err = delete_playbook(str(tmp_path), "../x.yml") + assert not ok and err == "Invalid path" + + +def test_validate_playbook_yaml(): + assert validate_playbook_yaml("- name: p\n hosts: all\n")[0] is True + assert validate_playbook_yaml("foo: [unclosed")[0] is False # malformed YAML + assert validate_playbook_yaml("")[0] is False # empty + assert validate_playbook_yaml("key: value")[0] is False # not a play list + + +def test_is_editable_source(): + assert is_editable_source({"type": "local", "name": LOCAL_SOURCE_NAME}) is True + assert is_editable_source({"type": "local", "name": BUILTIN_SOURCE_NAME}) is False + assert is_editable_source({"type": "git", "name": "repo"}) is False + + +def test_get_sources_includes_writable_local(): + names = [s["name"] for s in get_sources({"sources": []})] + assert LOCAL_SOURCE_NAME in names From 71e47242861ebe37c2bea766bb807982e0891c87 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 14:09:58 -0400 Subject: [PATCH 043/126] feat(ansible): cross-link inventory/schedules/browse for discoverability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inventory CRUD UI (/ansible/inventory/targets + /groups) existed but was unreachable from the main Ansible pages — only a buried text hint pointed to it. Add an "Inventory" button to the Runs, Browse, and Schedules headers, and "← Ansible" back-links on the inventory target/group pages, so the targeting features (manual runs, schedules, Deploy-via-Ansible) are findable. Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/templates/ansible/browse.html | 2 ++ steward/templates/ansible/index.html | 1 + steward/templates/ansible/inventory/groups.html | 5 ++++- steward/templates/ansible/inventory/targets.html | 5 ++++- steward/templates/ansible/schedules.html | 6 +++++- 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/steward/templates/ansible/browse.html b/steward/templates/ansible/browse.html index bf04039..578bc64 100644 --- a/steward/templates/ansible/browse.html +++ b/steward/templates/ansible/browse.html @@ -7,6 +7,8 @@ {% if session.user_role == 'admin' %} New playbook {% endif %} + Inventory + Schedules ← Run History diff --git a/steward/templates/ansible/index.html b/steward/templates/ansible/index.html index c77be9a..6a7418b 100644 --- a/steward/templates/ansible/index.html +++ b/steward/templates/ansible/index.html @@ -4,6 +4,7 @@

Ansible Runs

diff --git a/steward/templates/ansible/inventory/groups.html b/steward/templates/ansible/inventory/groups.html index bdb0ec3..c8c8336 100644 --- a/steward/templates/ansible/inventory/groups.html +++ b/steward/templates/ansible/inventory/groups.html @@ -3,7 +3,10 @@ {% block content %}

Inventory Groups

- Targets → +
diff --git a/steward/templates/ansible/inventory/targets.html b/steward/templates/ansible/inventory/targets.html index 17c00a5..ecfb31a 100644 --- a/steward/templates/ansible/inventory/targets.html +++ b/steward/templates/ansible/inventory/targets.html @@ -3,7 +3,10 @@ {% block content %}

Inventory Targets

- ← Groups +
diff --git a/steward/templates/ansible/schedules.html b/steward/templates/ansible/schedules.html index 470fd6f..ecf149b 100644 --- a/steward/templates/ansible/schedules.html +++ b/steward/templates/ansible/schedules.html @@ -9,7 +9,11 @@ {% block content %}

Ansible Schedules

- ← Runs +
{# ── Existing schedules ───────────────────────────────────────────────────── #} From 389002fc6fa134c3b98352c192ad974d93269510 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 14:22:19 -0400 Subject: [PATCH 044/126] feat(ui): breadcrumb navigation across nested pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a breadcrumb trail to nested views for orientation. Mechanism: a {% block breadcrumb %} slot in base.html (+ styling) and a shared crumbs() macro in templates/_macros.html; each nested page fills the block with its trail (root→current, last item is the current page). Pages without the block render no bar, so top-level nav roots stay clean. Applied to: Ansible (browse, schedules, playbook editor, run detail) + inventory (targets/groups + detail), host_agent (fleet, host detail, settings), hosts form, settings tabs (ansible/auth/notifications/plugins/ reports + plugin detail), dashboard (list, edit), and alerts (rule form, maintenance + new). Dynamic labels (host/target/run/dashboard names) come from the page context — no route changes. All 60 templates Jinja-compile. Task #873. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/templates/host_detail.html | 2 ++ plugins/host_agent/templates/host_list.html | 2 ++ .../host_agent/templates/settings_list.html | 2 ++ steward/templates/_macros.html | 18 ++++++++++++++++++ steward/templates/alerts/maintenance.html | 2 ++ steward/templates/alerts/maintenance_form.html | 2 ++ steward/templates/alerts/rules_form.html | 2 ++ steward/templates/ansible/browse.html | 8 ++++++++ .../ansible/inventory/group_detail.html | 2 ++ .../templates/ansible/inventory/groups.html | 2 ++ .../ansible/inventory/target_detail.html | 2 ++ .../templates/ansible/inventory/targets.html | 2 ++ steward/templates/ansible/playbook_editor.html | 2 ++ steward/templates/ansible/run_detail.html | 2 ++ steward/templates/ansible/schedules.html | 2 ++ steward/templates/base.html | 8 ++++++++ steward/templates/dashboard/edit.html | 2 ++ steward/templates/dashboard/list.html | 2 ++ steward/templates/hosts/form.html | 2 ++ steward/templates/settings/ansible.html | 2 ++ steward/templates/settings/auth.html | 2 ++ steward/templates/settings/notifications.html | 2 ++ steward/templates/settings/plugin_detail.html | 2 ++ steward/templates/settings/plugins.html | 2 ++ steward/templates/settings/reports.html | 2 ++ 25 files changed, 78 insertions(+) create mode 100644 steward/templates/_macros.html diff --git a/plugins/host_agent/templates/host_detail.html b/plugins/host_agent/templates/host_detail.html index 1129759..55a0cc3 100644 --- a/plugins/host_agent/templates/host_detail.html +++ b/plugins/host_agent/templates/host_detail.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}{{ host.name }} — Host Agent — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Host Agents", "/plugins/host_agent/"), (host.name, "")]) }}{% endblock %} {% macro fmt_bps(v) %} {%- if v is none -%}— diff --git a/plugins/host_agent/templates/host_list.html b/plugins/host_agent/templates/host_list.html index 69aad8e..79d63a0 100644 --- a/plugins/host_agent/templates/host_list.html +++ b/plugins/host_agent/templates/host_list.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Host Agents — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Dashboard", "/"), ("Host Agents", "")]) }}{% endblock %} {% block content %}

Host Agents

diff --git a/plugins/host_agent/templates/settings_list.html b/plugins/host_agent/templates/settings_list.html index dc52e79..03b6958 100644 --- a/plugins/host_agent/templates/settings_list.html +++ b/plugins/host_agent/templates/settings_list.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Host Agent — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Host Agents", "/plugins/host_agent/"), ("Settings", "")]) }}{% endblock %} {% block content %}

Host Agent — Registered Hosts

diff --git a/steward/templates/_macros.html b/steward/templates/_macros.html new file mode 100644 index 0000000..4cb4b16 --- /dev/null +++ b/steward/templates/_macros.html @@ -0,0 +1,18 @@ +{# Shared template macros. Import with: {% from "_macros.html" import crumbs %} #} + +{# Breadcrumb trail. `items` is a list of (label, href) pairs, ordered root→current. + Intermediate items with a truthy href render as links; the last item (or any + item with an empty href) renders as plain current text. #} +{% macro crumbs(items) -%} + +{%- endmacro %} diff --git a/steward/templates/alerts/maintenance.html b/steward/templates/alerts/maintenance.html index 284c413..6061a7e 100644 --- a/steward/templates/alerts/maintenance.html +++ b/steward/templates/alerts/maintenance.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Maintenance Windows — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Alerts", "/alerts/"), ("Maintenance", "")]) }}{% endblock %} {% block content %}

Maintenance Windows

diff --git a/steward/templates/alerts/maintenance_form.html b/steward/templates/alerts/maintenance_form.html index 2336c2a..860102b 100644 --- a/steward/templates/alerts/maintenance_form.html +++ b/steward/templates/alerts/maintenance_form.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}New Maintenance Window — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Alerts", "/alerts/"), ("Maintenance", "/alerts/maintenance"), ("New window", "")]) }}{% endblock %} {% block content %}

New Maintenance Window

diff --git a/steward/templates/alerts/rules_form.html b/steward/templates/alerts/rules_form.html index 6805827..c44f22d 100644 --- a/steward/templates/alerts/rules_form.html +++ b/steward/templates/alerts/rules_form.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}{% if rule %}Edit Rule{% else %}New Alert Rule{% endif %} — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Alerts", "/alerts/"), ("Edit rule" if rule else "New rule", "")]) }}{% endblock %} {% block content %}

{% if rule %}Edit Rule{% else %}New Alert Rule{% endif %}

diff --git a/steward/templates/ansible/browse.html b/steward/templates/ansible/browse.html index 578bc64..afdcbcc 100644 --- a/steward/templates/ansible/browse.html +++ b/steward/templates/ansible/browse.html @@ -1,5 +1,13 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Browse Playbooks — Steward{% endblock %} +{% block breadcrumb %} +{%- if view_contents is defined -%} +{{ crumbs([("Ansible", "/ansible/"), ("Browse", "/ansible/browse"), (view_path, "")]) }} +{%- else -%} +{{ crumbs([("Ansible", "/ansible/"), ("Browse", "")]) }} +{%- endif -%} +{% endblock %} {% block content %}

Browse Playbooks

diff --git a/steward/templates/ansible/inventory/group_detail.html b/steward/templates/ansible/inventory/group_detail.html index fc35507..84e6975 100644 --- a/steward/templates/ansible/inventory/group_detail.html +++ b/steward/templates/ansible/inventory/group_detail.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Group: {{ group.name }} — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Ansible", "/ansible/"), ("Inventory", "/ansible/inventory/targets"), ("Groups", "/ansible/inventory/groups"), (group.name, "")]) }}{% endblock %} {% block content %}

Group: {{ group.name }}

diff --git a/steward/templates/ansible/inventory/groups.html b/steward/templates/ansible/inventory/groups.html index c8c8336..da2b1e1 100644 --- a/steward/templates/ansible/inventory/groups.html +++ b/steward/templates/ansible/inventory/groups.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Inventory Groups — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Ansible", "/ansible/"), ("Inventory", "/ansible/inventory/targets"), ("Groups", "")]) }}{% endblock %} {% block content %}

Inventory Groups

diff --git a/steward/templates/ansible/inventory/target_detail.html b/steward/templates/ansible/inventory/target_detail.html index 588c756..4837f72 100644 --- a/steward/templates/ansible/inventory/target_detail.html +++ b/steward/templates/ansible/inventory/target_detail.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Target: {{ target.name }} — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Ansible", "/ansible/"), ("Inventory", "/ansible/inventory/targets"), ("Targets", "/ansible/inventory/targets"), (target.name, "")]) }}{% endblock %} {% block content %}

Target: {{ target.name }}

diff --git a/steward/templates/ansible/inventory/targets.html b/steward/templates/ansible/inventory/targets.html index ecfb31a..f92d3d4 100644 --- a/steward/templates/ansible/inventory/targets.html +++ b/steward/templates/ansible/inventory/targets.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Inventory Targets — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Ansible", "/ansible/"), ("Inventory", "/ansible/inventory/targets"), ("Targets", "")]) }}{% endblock %} {% block content %}

Inventory Targets

diff --git a/steward/templates/ansible/playbook_editor.html b/steward/templates/ansible/playbook_editor.html index 24dca51..4d5d51e 100644 --- a/steward/templates/ansible/playbook_editor.html +++ b/steward/templates/ansible/playbook_editor.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}{{ "Edit" if editing else "New" }} Playbook — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Ansible", "/ansible/"), ("Browse", "/ansible/browse"), (("Edit " ~ playbook_path) if editing else "New playbook", "")]) }}{% endblock %} {% block content %}

{{ "Edit playbook" if editing else "New playbook" }}

diff --git a/steward/templates/ansible/run_detail.html b/steward/templates/ansible/run_detail.html index 1aa094a..8f03613 100644 --- a/steward/templates/ansible/run_detail.html +++ b/steward/templates/ansible/run_detail.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Run {{ run.id[:8] }} — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Ansible", "/ansible/"), ("Run " ~ run.id[:8], "")]) }}{% endblock %} {% block content %}
← Runs diff --git a/steward/templates/ansible/schedules.html b/steward/templates/ansible/schedules.html index ecf149b..985da9d 100644 --- a/steward/templates/ansible/schedules.html +++ b/steward/templates/ansible/schedules.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Ansible Schedules — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Ansible", "/ansible/"), ("Schedules", "")]) }}{% endblock %} {% macro fmt_interval(secs) %} {%- set m = {300: "every 5 min", 3600: "hourly", 21600: "every 6h", 43200: "every 12h", 86400: "daily", 604800: "weekly"} -%} diff --git a/steward/templates/base.html b/steward/templates/base.html index a23329d..7c082ca 100644 --- a/steward/templates/base.html +++ b/steward/templates/base.html @@ -145,6 +145,13 @@ textarea { resize: vertical; } /* Empty state */ .empty { color: var(--text-muted); font-size: 0.9rem; padding: 2rem; text-align: center; } +/* Breadcrumbs */ +.breadcrumb { display:flex; align-items:center; flex-wrap:wrap; gap:0.35rem; font-size:0.8rem; color:var(--text-dim); margin-bottom:1rem; } +.breadcrumb a { color:var(--text-muted); } +.breadcrumb a:hover { color:var(--text); } +.breadcrumb-sep { color:var(--border-mid); } +.breadcrumb-cur { color:var(--text-muted); } + /* Stat cards (dashboard) */ .stat-val { font-size: 2rem; font-weight: 700; line-height: 1; } .stat-label { font-size: 0.8rem; color: var(--text-muted); margin-left: 0.3rem; } @@ -228,6 +235,7 @@ function setTimeRange(val) {
+ {% block breadcrumb %}{% endblock %} {% if plugin_failures and session.user_role == 'admin' %}

Edit: {{ dashboard.name }}

diff --git a/steward/templates/dashboard/list.html b/steward/templates/dashboard/list.html index a9372e4..213e1ef 100644 --- a/steward/templates/dashboard/list.html +++ b/steward/templates/dashboard/list.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Dashboards — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Dashboard", "/"), ("Dashboards", "")]) }}{% endblock %} {% block content %}

Dashboards

diff --git a/steward/templates/hosts/form.html b/steward/templates/hosts/form.html index 1ff251f..e39507c 100644 --- a/steward/templates/hosts/form.html +++ b/steward/templates/hosts/form.html @@ -1,5 +1,7 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}{% if host %}Edit Host{% else %}New Host{% endif %} — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), (("Edit " ~ host.name) if host else "New host", "")]) }}{% endblock %} {% block content %}

{% if host %}Edit Host{% else %}Add Host{% endif %}

diff --git a/steward/templates/settings/ansible.html b/steward/templates/settings/ansible.html index 64aa5a2..3861c88 100644 --- a/steward/templates/settings/ansible.html +++ b/steward/templates/settings/ansible.html @@ -1,6 +1,8 @@ {# steward/templates/settings/ansible.html #} {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Settings — Ansible — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Ansible", "")]) }}{% endblock %} {% block content %} {% set active_tab = "ansible" %} {% include "settings/_tabs.html" %} diff --git a/steward/templates/settings/auth.html b/steward/templates/settings/auth.html index 41b69c4..f273db0 100644 --- a/steward/templates/settings/auth.html +++ b/steward/templates/settings/auth.html @@ -1,6 +1,8 @@ {# steward/templates/settings/auth.html #} {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Settings — Auth — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Authentication", "")]) }}{% endblock %} {% block content %} {% set active_tab = "auth" %} {% include "settings/_tabs.html" %} diff --git a/steward/templates/settings/notifications.html b/steward/templates/settings/notifications.html index ed2d4e9..d105694 100644 --- a/steward/templates/settings/notifications.html +++ b/steward/templates/settings/notifications.html @@ -1,6 +1,8 @@ {# steward/templates/settings/notifications.html #} {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Settings — Notifications — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Notifications", "")]) }}{% endblock %} {% block content %} {% set active_tab = "notifications" %} {% include "settings/_tabs.html" %} diff --git a/steward/templates/settings/plugin_detail.html b/steward/templates/settings/plugin_detail.html index aac1c00..b0a1fd7 100644 --- a/steward/templates/settings/plugin_detail.html +++ b/steward/templates/settings/plugin_detail.html @@ -1,6 +1,8 @@ {# steward/templates/settings/plugin_detail.html #} {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}{{ plugin.get('name', plugin._dir) }} Settings — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Plugins", "/settings/plugins/"), (plugin.get('name', plugin._dir), "")]) }}{% endblock %} {% block content %} {% set active_tab = "plugins" %} {% include "settings/_tabs.html" %} diff --git a/steward/templates/settings/plugins.html b/steward/templates/settings/plugins.html index 2b2a5c6..60bd70d 100644 --- a/steward/templates/settings/plugins.html +++ b/steward/templates/settings/plugins.html @@ -1,6 +1,8 @@ {# steward/templates/settings/plugins.html #} {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Settings — Plugins — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Plugins", "")]) }}{% endblock %} {% block content %} {% set active_tab = "plugins" %} {% include "settings/_tabs.html" %} diff --git a/steward/templates/settings/reports.html b/steward/templates/settings/reports.html index 8df780b..c778f71 100644 --- a/steward/templates/settings/reports.html +++ b/steward/templates/settings/reports.html @@ -1,6 +1,8 @@ {# steward/templates/settings/reports.html #} {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Settings — Reports — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Reports", "")]) }}{% endblock %} {% block content %} {% set active_tab = "reports" %} {% include "settings/_tabs.html" %} From 88857be24e65e307b4becb0d6b47f9d0b480417c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 14:51:04 -0400 Subject: [PATCH 045/126] =?UTF-8?q?feat(ansible):=20runner=20robustness=20?= =?UTF-8?q?=E2=80=94=20cancel,=20concurrency,=20structured=20results,=20re?= =?UTF-8?q?tention?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #550 (all four): - Cancellation: track live subprocesses; POST /ansible/runs//cancel (operator) SIGTERMs then SIGKILLs after a grace; new 'cancelled' status (+ migration 0019, ALTER TYPE in autocommit). Queued runs cancel cleanly before launch. Cancel button on run detail. - Concurrency: global semaphore (ansible.max_concurrent_runs, default 3, Settings→Ansible) caps simultaneous runs; excess show 'queued' (new status) until a slot frees. Semaphore bound lazily per running loop. - Structured results: parse PLAY RECAP into per-host ok/changed/unreachable/ failed/skipped + capture failed-task lines, stored in new results JSON column (migration 0020); rendered as a host-summary table on run detail. Keeps live streaming (no json-callback swap). - Retention: full output written to a persistent log artifact (/data/ansible/runs/.log, env-overridable) beyond the 1 MB DB cap and across restarts; in-memory replay buffer bounded + GC'd after completion; Download-log route. Boot reconciliation now also sweeps stale 'queued'. Unit tests for recap parsing + cancel flagging. Status colors updated across run list / detail / schedules. Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/ansible/executor.py | 272 ++++++++++++++---- steward/ansible/routes.py | 40 ++- steward/app.py | 3 +- steward/core/settings.py | 3 + .../versions/0019_ansible_run_statuses.py | 26 ++ .../versions/0020_ansible_run_results.py | 22 ++ steward/models/ansible.py | 7 +- steward/settings/routes.py | 6 + steward/templates/ansible/index.html | 2 +- steward/templates/ansible/run_detail.html | 42 ++- steward/templates/ansible/schedules.html | 2 +- steward/templates/settings/ansible.html | 7 + tests/core/test_ansible_executor.py | 35 +++ 13 files changed, 406 insertions(+), 61 deletions(-) create mode 100644 steward/migrations/versions/0019_ansible_run_statuses.py create mode 100644 steward/migrations/versions/0020_ansible_run_results.py create mode 100644 tests/core/test_ansible_executor.py diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py index d194b67..dcfa9b6 100644 --- a/steward/ansible/executor.py +++ b/steward/ansible/executor.py @@ -4,6 +4,7 @@ import asyncio import json import logging import os +import re import shutil import tempfile import time @@ -17,9 +18,19 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -_OUTPUT_CAP_BYTES = 1024 * 1024 # 1 MB DB cap +_OUTPUT_CAP_BYTES = 1024 * 1024 # 1 MB DB cap (full output also goes to a log artifact) _FLUSH_LINES = 50 _FLUSH_SECS = 5.0 +_MEM_LINES_CAP = 5000 # bounded per-run in-memory replay buffer (SSE late-joiners) +_MEM_TTL_SECS = 300 # drop in-memory state this long after a run completes +_FAILURE_CAP = 50 # max failed-task lines captured into structured results + +# Full run logs persist here (survive restart + the 1 MB DB cap). Override via env. +ARTIFACT_DIR = os.environ.get("STEWARD_ANSIBLE_LOG_DIR", "/data/ansible/runs") + + +def artifact_path(run_id: str) -> str: + return os.path.join(ARTIFACT_DIR, f"{run_id}.log") @dataclass @@ -27,11 +38,19 @@ class _Done: status: str -# Per-run broadcast state (lives until process restarts) -_run_lines: dict[str, list[str]] = {} # all output lines received so far +class _Aborted(Exception): + """Raised internally when a queued run is cancelled before it launches.""" + + +# Per-run broadcast state (lives until shortly after completion, then GC'd) +_run_lines: dict[str, list[str]] = {} # bounded tail of output lines _run_done: dict[str, _Done] = {} # set when run completes _run_listeners: dict[str, list[asyncio.Queue]] = {} # active SSE client queues +# Running subprocesses by run_id (for cancellation) + run_ids an operator cancelled. +_run_procs: dict[str, asyncio.subprocess.Process] = {} +_cancelled: set[str] = set() + def register_listener(run_id: str) -> asyncio.Queue: """Create a listener queue for a run. Replays existing lines immediately.""" @@ -52,15 +71,31 @@ def deregister_listener(run_id: str, q: asyncio.Queue) -> None: listeners.remove(q) +def _schedule_cleanup(run_id: str) -> None: + """Drop a finished run's in-memory state after a grace window.""" + async def _cleanup() -> None: + await asyncio.sleep(_MEM_TTL_SECS) + _run_lines.pop(run_id, None) + _run_done.pop(run_id, None) + try: + asyncio.create_task(_cleanup()) + except RuntimeError: + pass # no running loop (e.g. tests) — fine to skip + + def _broadcast(run_id: str, item: str | _Done) -> None: if isinstance(item, str): - _run_lines.setdefault(run_id, []).append(item) + buf = _run_lines.setdefault(run_id, []) + buf.append(item) + if len(buf) > _MEM_LINES_CAP: # keep only the most recent lines in memory + del buf[: len(buf) - _MEM_LINES_CAP] else: _run_done[run_id] = item for q in _run_listeners.get(run_id, []): q.put_nowait(item) if isinstance(item, _Done): _run_listeners.pop(run_id, None) + _schedule_cleanup(run_id) def build_ansible_command( @@ -127,6 +162,86 @@ def ansible_env(creds: dict | None, base_env) -> dict: return env +# ── Structured results (parsed from the streamed output) ────────────────────── + +_RECAP_RE = re.compile( + r"^(?P\S+)\s*:\s*ok=(?P\d+)\s+changed=(?P\d+)\s+" + r"unreachable=(?P\d+)\s+failed=(?P\d+)" + r"(?:\s+skipped=(?P\d+))?" +) + + +def parse_recap(lines: list[str]) -> dict[str, dict[str, int]]: + """Parse Ansible's PLAY RECAP into {host: {ok,changed,unreachable,failed,skipped}}. + + Streaming-friendly: we keep the default stdout callback (live output) and + derive the per-host summary from the recap rather than swapping to the json + callback (which would buffer everything to the end and kill live streaming). + """ + hosts: dict[str, dict[str, int]] = {} + in_recap = False + for line in lines: + if "PLAY RECAP" in line: + in_recap = True + continue + if not in_recap: + continue + m = _RECAP_RE.match(line.strip()) + if m: + hosts[m.group("host")] = { + "ok": int(m.group("ok")), + "changed": int(m.group("changed")), + "unreachable": int(m.group("unreachable")), + "failed": int(m.group("failed")), + "skipped": int(m.group("skipped") or 0), + } + return hosts + + +# ── Cancellation ────────────────────────────────────────────────────────────── + +async def _set_status(app: "Quart", run_id: str, status) -> None: + from sqlalchemy import update + from steward.models.ansible import AnsibleRun + async with app.db_sessionmaker() as session: + async with session.begin(): + await session.execute( + update(AnsibleRun).where(AnsibleRun.id == run_id).values(status=status) + ) + + +async def _terminate_then_kill(proc: asyncio.subprocess.Process, grace: float = 10.0) -> None: + try: + proc.terminate() + except ProcessLookupError: + return + try: + await asyncio.wait_for(proc.wait(), timeout=grace) + except asyncio.TimeoutError: + try: + proc.kill() + except ProcessLookupError: + pass + + +def cancel_run(run_id: str) -> bool: + """Request cancellation of a run. Terminates the process if it's running; + a queued run is flagged so start_run aborts before launch. Returns False + only when the run isn't tracked in this process (already finished).""" + if run_id not in _run_procs and run_id not in _cancelled: + # Not obviously in-flight. Still flag it (covers a just-queued run whose + # proc hasn't been registered yet); caller should gate on run status. + _cancelled.add(run_id) + return run_id in _run_procs or True + _cancelled.add(run_id) + proc = _run_procs.get(run_id) + if proc is not None: + asyncio.create_task(_terminate_then_kill(proc)) + return True + + +# ── Run execution ───────────────────────────────────────────────────────────── + async def start_run( app: "Quart", run_id: str, @@ -138,26 +253,30 @@ async def start_run( ) -> None: """Execute ansible-playbook as a subprocess and update the DB run row. - If inventory_content is given (host-targeted runs), it's written to a temp - file and used as the inventory instead of inventory_path. + Respects a global concurrency cap (app._ansible_semaphore): if no slot is + free the run shows as 'queued' until one frees. Supports cancellation, a + persistent full-log artifact, and a parsed per-host result summary. """ - _run_lines[run_id] = [] + from steward.models.ansible import AnsibleRunStatus + _run_lines[run_id] = [] db_output = "" truncated = False lines_since_flush = 0 last_flush_at = time.monotonic() + failures: list[str] = [] - async def _flush(final_status: str | None = None) -> None: + async def _flush(final_status: str | None = None, results: dict | None = None) -> None: nonlocal lines_since_flush, last_flush_at from sqlalchemy import update - from steward.models.ansible import AnsibleRun, AnsibleRunStatus + from steward.models.ansible import AnsibleRun values: dict = {"output": db_output} + if results is not None: + values["results"] = results if final_status is not None: values["status"] = AnsibleRunStatus(final_status) values["finished_at"] = datetime.now(timezone.utc) - async with app.db_sessionmaker() as session: async with session.begin(): await session.execute( @@ -166,61 +285,106 @@ async def start_run( lines_since_flush = 0 last_flush_at = time.monotonic() - cwd = source_path if Path(source_path).exists() else None - creds = app.config.get("ANSIBLE", {}) + # ── Concurrency gate (semaphore lazily bound to the running loop so the + # cap is honoured in prod's single loop without breaking multi-loop tests). + loop = asyncio.get_running_loop() + sem = getattr(app, "_ansible_semaphore", None) + if sem is None or getattr(app, "_ansible_sem_loop", None) is not loop: + cap = int((app.config.get("ANSIBLE") or {}).get("max_concurrent_runs", 3) or 3) + sem = asyncio.Semaphore(max(1, cap)) + app._ansible_semaphore = sem + app._ansible_sem_loop = loop + queued = sem.locked() + if queued: + await _set_status(app, run_id, AnsibleRunStatus.queued) + _broadcast(run_id, "[queued — waiting for an Ansible run slot]") + await sem.acquire() - # Temp dir holds the ephemeral inventory (if any) + credential files; removed in finally. - tmpdir = tempfile.mkdtemp(prefix="steward-ansible-") - os.chmod(tmpdir, 0o700) + final_status = "failed" + logf = None try: - if inventory_content: - inv_target = os.path.join(tmpdir, "inventory") - with open(inv_target, "w", encoding="utf-8") as invf: - invf.write(inventory_content) - else: - inv_target = inventory_path - cmd = build_ansible_command(playbook_path, inv_target, params) + if run_id in _cancelled: # cancelled while queued — abort before launching + raise _Aborted + if queued: + await _set_status(app, run_id, AnsibleRunStatus.running) - cred_args, cred_files = build_credentials(creds, tmpdir) - for cred_path, content in cred_files: - with open(cred_path, "w", encoding="utf-8") as cf: - cf.write(content) - os.chmod(cred_path, 0o600) + cwd = source_path if Path(source_path).exists() else None + creds = app.config.get("ANSIBLE", {}) + try: + os.makedirs(ARTIFACT_DIR, exist_ok=True) + logf = open(artifact_path(run_id), "w", encoding="utf-8") + except OSError: + logf = None # artifact is best-effort; the DB still holds capped output - proc = await asyncio.create_subprocess_exec( - *cmd, *cred_args, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.STDOUT, - cwd=cwd, - env=ansible_env(creds, os.environ), - ) - assert proc.stdout is not None + # Temp dir holds the ephemeral inventory (if any) + credential files. + tmpdir = tempfile.mkdtemp(prefix="steward-ansible-") + os.chmod(tmpdir, 0o700) + try: + if inventory_content: + inv_target = os.path.join(tmpdir, "inventory") + with open(inv_target, "w", encoding="utf-8") as invf: + invf.write(inventory_content) + else: + inv_target = inventory_path + cmd = build_ansible_command(playbook_path, inv_target, params) - async for raw_line in proc.stdout: - line = raw_line.decode(errors="replace").rstrip("\n\r") - _broadcast(run_id, line) + cred_args, cred_files = build_credentials(creds, tmpdir) + for cred_path, content in cred_files: + with open(cred_path, "w", encoding="utf-8") as cf: + cf.write(content) + os.chmod(cred_path, 0o600) - if not truncated: - candidate = (db_output + "\n" + line) if db_output else line - if len(candidate.encode()) > _OUTPUT_CAP_BYTES: - db_output += "\n[output truncated]" - truncated = True - else: - db_output = candidate + proc = await asyncio.create_subprocess_exec( + *cmd, *cred_args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + cwd=cwd, + env=ansible_env(creds, os.environ), + ) + _run_procs[run_id] = proc + assert proc.stdout is not None - lines_since_flush += 1 - elapsed = time.monotonic() - last_flush_at - if lines_since_flush >= _FLUSH_LINES or elapsed >= _FLUSH_SECS: - await _flush() + async for raw_line in proc.stdout: + line = raw_line.decode(errors="replace").rstrip("\n\r") + _broadcast(run_id, line) + if logf is not None: + logf.write(line + "\n") + if (line.startswith("fatal:") or "FAILED!" in line) and len(failures) < _FAILURE_CAP: + failures.append(line[:500]) - await proc.wait() - final_status = "success" if proc.returncode == 0 else "failed" + if not truncated: + candidate = (db_output + "\n" + line) if db_output else line + if len(candidate.encode()) > _OUTPUT_CAP_BYTES: + db_output += "\n[output truncated — download the full log]" + truncated = True + else: + db_output = candidate + lines_since_flush += 1 + elapsed = time.monotonic() - last_flush_at + if lines_since_flush >= _FLUSH_LINES or elapsed >= _FLUSH_SECS: + await _flush() + + await proc.wait() + if run_id in _cancelled: + final_status = "cancelled" + else: + final_status = "success" if proc.returncode == 0 else "failed" + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + except _Aborted: + final_status = "cancelled" except Exception: logger.exception("Ansible run %s failed with exception", run_id) - final_status = "failed" + final_status = "cancelled" if run_id in _cancelled else "failed" finally: - shutil.rmtree(tmpdir, ignore_errors=True) + if logf is not None: + logf.close() + _run_procs.pop(run_id, None) + _cancelled.discard(run_id) + sem.release() - await _flush(final_status) + results = {"hosts": parse_recap(_run_lines.get(run_id, [])), "failures": failures} + await _flush(final_status, results=results) _broadcast(run_id, _Done(status=final_status)) diff --git a/steward/ansible/routes.py b/steward/ansible/routes.py index c71e05d..5e206b0 100644 --- a/steward/ansible/routes.py +++ b/steward/ansible/routes.py @@ -166,7 +166,7 @@ async def stream_run(run_id: str): return "Not found", 404 async def generate(): - if run.status != AnsibleRunStatus.running: + if run.status not in (AnsibleRunStatus.running, AnsibleRunStatus.queued): yield f"event: done\ndata: {run.status.value}\n\n" return @@ -208,6 +208,44 @@ async def run_detail(run_id: str): "ansible/run_detail.html", run=run, triggered_label=triggered_label) +@ansible_bp.post("/runs//cancel") +@require_role(UserRole.operator) +async def cancel_run(run_id: str): + async with current_app.db_sessionmaker() as db: + run = (await db.execute( + select(AnsibleRun).where(AnsibleRun.id == run_id))).scalar_one_or_none() + if run is None: + return "Not found", 404 + if run.status in (AnsibleRunStatus.running, AnsibleRunStatus.queued): + executor.cancel_run(run_id) + return redirect(url_for("ansible.run_detail", run_id=run_id)) + + +@ansible_bp.get("/runs//log") +@require_role(UserRole.viewer) +async def run_log(run_id: str): + """Download the full run log (persistent artifact; falls back to DB output).""" + import os + from steward.ansible.executor import artifact_path + + async with current_app.db_sessionmaker() as db: + run = (await db.execute( + select(AnsibleRun).where(AnsibleRun.id == run_id))).scalar_one_or_none() + if run is None: + return "Not found", 404 + + path = artifact_path(run_id) + if os.path.exists(path): + with open(path, encoding="utf-8", errors="replace") as f: + body = f.read() + else: + body = run.output or "" + return Response( + body, mimetype="text/plain", + headers={"Content-Disposition": f'attachment; filename="ansible-run-{run_id[:8]}.log"'}, + ) + + # ── Scheduled recurring runs ────────────────────────────────────────────────── @ansible_bp.get("/schedules") diff --git a/steward/app.py b/steward/app.py index a97bd86..9984424 100644 --- a/steward/app.py +++ b/steward/app.py @@ -322,7 +322,8 @@ async def _mark_interrupted_runs(app: Quart) -> None: async with session.begin(): await session.execute( update(AnsibleRun) - .where(AnsibleRun.status == AnsibleRunStatus.running) + .where(AnsibleRun.status.in_( + [AnsibleRunStatus.running, AnsibleRunStatus.queued])) .values( status=AnsibleRunStatus.interrupted, finished_at=datetime.now(timezone.utc), diff --git a/steward/core/settings.py b/steward/core/settings.py index f8cf0d1..8fa3c2c 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -57,6 +57,8 @@ DEFAULTS: dict[str, Any] = { "ansible.become_password": "", "ansible.vault_password": "", "ansible.host_key_checking": False, + # Max simultaneous playbook runs; extra runs queue. Applied at app start. + "ansible.max_concurrent_runs": 3, "ping.threshold.good_ms": 50, "ping.threshold.warn_ms": 200, "plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml", @@ -175,6 +177,7 @@ def to_ansible_cfg(settings: dict[str, Any]) -> dict: "become_password": settings.get("ansible.become_password", ""), "vault_password": settings.get("ansible.vault_password", ""), "host_key_checking": settings.get("ansible.host_key_checking", False), + "max_concurrent_runs": settings.get("ansible.max_concurrent_runs", 3), } diff --git a/steward/migrations/versions/0019_ansible_run_statuses.py b/steward/migrations/versions/0019_ansible_run_statuses.py new file mode 100644 index 0000000..ebbd5e7 --- /dev/null +++ b/steward/migrations/versions/0019_ansible_run_statuses.py @@ -0,0 +1,26 @@ +"""Add 'queued' + 'cancelled' values to the ansiblerunstatus enum + +Revision ID: 0019_ansible_run_statuses +Revises: 0018_ansible_schedules +Create Date: 2026-06-16 +""" +from typing import Sequence, Union +from alembic import op + +revision: str = "0019_ansible_run_statuses" +down_revision: Union[str, None] = "0018_ansible_schedules" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ALTER TYPE ... ADD VALUE cannot run inside a transaction block; use an + # autocommit block. IF NOT EXISTS keeps it idempotent across re-runs. + with op.get_context().autocommit_block(): + op.execute("ALTER TYPE ansiblerunstatus ADD VALUE IF NOT EXISTS 'queued'") + op.execute("ALTER TYPE ansiblerunstatus ADD VALUE IF NOT EXISTS 'cancelled'") + + +def downgrade() -> None: + # Postgres has no DROP VALUE for enums; leaving the values is harmless. + pass diff --git a/steward/migrations/versions/0020_ansible_run_results.py b/steward/migrations/versions/0020_ansible_run_results.py new file mode 100644 index 0000000..9dfdf79 --- /dev/null +++ b/steward/migrations/versions/0020_ansible_run_results.py @@ -0,0 +1,22 @@ +"""Add results JSON column to ansible_runs (parsed per-host summary) + +Revision ID: 0020_ansible_run_results +Revises: 0019_ansible_run_statuses +Create Date: 2026-06-16 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "0020_ansible_run_results" +down_revision: Union[str, None] = "0019_ansible_run_statuses" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("ansible_runs", sa.Column("results", sa.JSON(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("ansible_runs", "results") diff --git a/steward/models/ansible.py b/steward/models/ansible.py index 36731dc..f0ace14 100644 --- a/steward/models/ansible.py +++ b/steward/models/ansible.py @@ -8,10 +8,12 @@ from .base import Base class AnsibleRunStatus(str, enum.Enum): + queued = "queued" # waiting for a concurrency slot running = "running" success = "success" failed = "failed" - interrupted = "interrupted" + interrupted = "interrupted" # process/host died mid-run (e.g. app restart) + cancelled = "cancelled" # stopped by an operator class AnsibleRun(Base): @@ -39,3 +41,6 @@ class AnsibleRun(Base): output: Mapped[str | None] = mapped_column(Text, nullable=True) # Optional run parameters: {extra_vars: [..], limit: str, tags: str, check: bool}. params: Mapped[dict | None] = mapped_column(JSON, nullable=True) + # Parsed per-host summary + captured failures: + # {hosts: {name: {ok,changed,unreachable,failed,skipped}}, failures: [str]}. + results: Mapped[dict | None] = mapped_column(JSON, nullable=True) diff --git a/steward/settings/routes.py b/steward/settings/routes.py index 805b5a1..fcba65b 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -179,6 +179,7 @@ async def ansible(): become_set=bool(settings.get("ansible.become_password")), vault_set=bool(settings.get("ansible.vault_password")), host_key_checking=settings.get("ansible.host_key_checking", False), + max_concurrent_runs=settings.get("ansible.max_concurrent_runs", 3), ) @@ -203,6 +204,11 @@ async def ansible_save_credentials(): if val: await set_setting(db, key, val) await set_setting(db, "ansible.host_key_checking", "host_key_checking" in form) + try: + mcr = max(1, min(50, int(form.get("max_concurrent_runs", 3)))) + except (TypeError, ValueError): + mcr = 3 + await set_setting(db, "ansible.max_concurrent_runs", mcr) await _reload_app_config() await log_audit(current_app, session.get("user_id"), session.get("username", ""), "settings.saved", detail={"section": "ansible.credentials"}) diff --git a/steward/templates/ansible/index.html b/steward/templates/ansible/index.html index 6a7418b..ad747a3 100644 --- a/steward/templates/ansible/index.html +++ b/steward/templates/ansible/index.html @@ -27,7 +27,7 @@
{{ run.playbook_path }} {{ run.source_name }}
+ + + + + + + + {% for host, c in run.results.hosts.items() %} + + + + + + + + + {% endfor %} + +
Hostokchangedunreachablefailedskipped
{{ host }}{{ c.ok }}{{ c.changed }}{{ c.unreachable }}{{ c.failed }}{{ c.skipped }}
+ {% if run.results.failures %} +
Failures
+ 
{{ run.results.failures | join("\n") }}
+ {% endif %} +
+{% endif %} +
Output
- {% if run.status.value == "running" %} + {% if run.status.value in ("running", "queued") %} 
{{ run.output or "" }}
diff --git a/steward/templates/ansible/schedules.html b/steward/templates/ansible/schedules.html index 985da9d..0fe51ee 100644 --- a/steward/templates/ansible/schedules.html +++ b/steward/templates/ansible/schedules.html @@ -42,7 +42,7 @@ {% if s.last_error %} launch error {% elif row.last_run %} - {% set sc = {"running":"var(--yellow)","success":"var(--green)","failed":"var(--red)","interrupted":"var(--orange)"}.get(row.last_run.status.value, "var(--text-muted)") %} + {% set sc = {"queued":"var(--text-dim)","running":"var(--yellow)","success":"var(--green)","failed":"var(--red)","interrupted":"var(--orange)","cancelled":"var(--orange)"}.get(row.last_run.status.value, "var(--text-muted)") %} {{ row.last_run.status.value }} · {{ s.last_run_at.strftime("%m-%d %H:%M") }} {% else %} diff --git a/steward/templates/settings/ansible.html b/steward/templates/settings/ansible.html index 3861c88..27d0602 100644 --- a/steward/templates/settings/ansible.html +++ b/steward/templates/settings/ansible.html @@ -62,6 +62,13 @@ Enforce SSH host-key checking (off by default for homelab convenience)
+
+ + +
diff --git a/tests/core/test_ansible_executor.py b/tests/core/test_ansible_executor.py new file mode 100644 index 0000000..8fde46c --- /dev/null +++ b/tests/core/test_ansible_executor.py @@ -0,0 +1,35 @@ +"""Unit tests for executor structured-results parsing + cancel flagging.""" +from steward.ansible.executor import _cancelled, cancel_run, parse_recap + + +def test_parse_recap_extracts_per_host_counts(): + lines = [ + "PLAY RECAP *****************************************************************", + "web01 : ok=5 changed=2 unreachable=0 failed=0 skipped=1 rescued=0", + "db01 : ok=3 changed=0 unreachable=1 failed=0 skipped=0", + ] + hosts = parse_recap(lines) + assert hosts["web01"] == {"ok": 5, "changed": 2, "unreachable": 0, "failed": 0, "skipped": 1} + assert hosts["db01"]["unreachable"] == 1 + assert hosts["db01"]["ok"] == 3 + + +def test_parse_recap_ignores_lines_before_recap(): + lines = [ + "TASK [whatever] ****", + "ok: [web01]", + "PLAY RECAP ****", + "web01 : ok=1 changed=0 unreachable=0 failed=0 skipped=0", + ] + assert set(parse_recap(lines)) == {"web01"} + + +def test_parse_recap_empty_without_recap(): + assert parse_recap(["TASK [x] ***", "ok: [h]"]) == {} + + +def test_cancel_run_flags_untracked_run(): + rid = "nonexistent-run-id" + assert cancel_run(rid) is True + assert rid in _cancelled + _cancelled.discard(rid) From 6e91bdc82be95cda90bf323a240a0529870d6068 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 15:22:43 -0400 Subject: [PATCH 046/126] feat(security): encrypt sensitive settings at rest (Fernet) Secrets (smtp.password, oidc.client_secret, ldap.bind_password, ansible ssh_private_key/become_password/vault_password) were stored plaintext in app_settings. Add transparent encryption-at-rest: - steward/core/crypto.py: Fernet keyed off the app secret (/data/secret.key), enc:v1: prefix marks ciphertext; passthrough for plaintext/empty/no-key, never reveals plaintext on a wrong key. - settings.py: SECRET_KEYS registry; set_setting encrypts on write; all read paths (get_setting / get_all_settings / load_settings_sync) decrypt transparently; migrate_plaintext_secrets() converts legacy rows in place. - app.py startup: init_crypto(SECRET_KEY) + one-time legacy-secret migration before settings load. - Add cryptography dependency. UI masking is unchanged (it checks decrypted truthiness). Key-loss caveat documented: secrets are unrecoverable if the app secret key is lost. Unit tests cover round-trip, empty/plaintext passthrough, and wrong-key safety. Task #580. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 1 + steward/app.py | 7 +++ steward/core/crypto.py | 93 +++++++++++++++++++++++++++++++++++++++ steward/core/settings.py | 65 ++++++++++++++++++++++++--- tests/core/test_crypto.py | 46 +++++++++++++++++++ 5 files changed, 206 insertions(+), 6 deletions(-) create mode 100644 steward/core/crypto.py create mode 100644 tests/core/test_crypto.py diff --git a/pyproject.toml b/pyproject.toml index 1a4aecc..c10b66a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "packaging>=24.0", "click>=8.0", "httpx>=0.27", + "cryptography>=42", ] [project.optional-dependencies] diff --git a/steward/app.py b/steward/app.py index 9984424..92b1c2c 100644 --- a/steward/app.py +++ b/steward/app.py @@ -47,6 +47,13 @@ def create_app( plugin_dirs=[Path(d).resolve() for d in app.config["PLUGIN_DIRS"]], ) + # ── 3b. Encryption-at-rest: bind the encryptor, then encrypt legacy secrets ─ + if not testing: + from .core.crypto import init_crypto + from .core.settings import migrate_plaintext_secrets + init_crypto(app.config["SECRET_KEY"]) + migrate_plaintext_secrets(app.config["DATABASE_URL"]) + # ── 4. Load all settings from DB → populate app.config ──────────────────── if not testing: from .core.settings import ( diff --git a/steward/core/crypto.py b/steward/core/crypto.py new file mode 100644 index 0000000..2d5a1fb --- /dev/null +++ b/steward/core/crypto.py @@ -0,0 +1,93 @@ +# steward/core/crypto.py +"""Encryption-at-rest for sensitive settings (smtp/oidc/ldap/ansible secrets). + +Values are encrypted with Fernet (AES-128-CBC + HMAC) using a key derived from +the app secret key (the same /data/secret.key used for sessions). Encrypted +values are stored with an ``enc:v1:`` prefix so reads can transparently tell +ciphertext from legacy plaintext during/after migration. + +Key-loss caveat: if the app secret key is lost or changed, encrypted secrets +become unrecoverable — they must be re-entered. This ties secret recovery to +the same key the rest of the app already depends on; back it up with the data. +""" +from __future__ import annotations + +import base64 +import hashlib +import logging +import os +from pathlib import Path + +from cryptography.fernet import Fernet, InvalidToken + +logger = logging.getLogger(__name__) + +ENC_PREFIX = "enc:v1:" +_SECRET_KEY_FILE = Path("/data/secret.key") +_fernet: Fernet | None = None + + +def _make_fernet(secret_key: str) -> Fernet: + # Derive a stable 32-byte Fernet key from the app secret (any length string). + digest = hashlib.sha256(secret_key.encode("utf-8")).digest() + return Fernet(base64.urlsafe_b64encode(digest)) + + +def init_crypto(secret_key: str) -> None: + """Bind the encryptor to the app's secret key (called once at startup).""" + global _fernet + if secret_key: + _fernet = _make_fernet(secret_key) + + +def _resolve_secret_key() -> str | None: + """Fallback key resolution mirroring config (env → /data/secret.key).""" + k = os.environ.get("STEWARD_SECRET_KEY") + if k: + return k + try: + if _SECRET_KEY_FILE.exists(): + v = _SECRET_KEY_FILE.read_text().strip() + if v: + return v + except OSError: + pass + return None + + +def _get() -> Fernet | None: + global _fernet + if _fernet is None: + k = _resolve_secret_key() + if k: + _fernet = _make_fernet(k) + return _fernet + + +def is_encrypted(value) -> bool: + return isinstance(value, str) and value.startswith(ENC_PREFIX) + + +def encrypt_secret(plaintext: str) -> str: + """Return an ``enc:v1:`` token, or the input unchanged if empty / no key.""" + if not plaintext: + return plaintext + f = _get() + if f is None: + logger.warning("No secret key available — storing a secret in PLAINTEXT") + return plaintext + return ENC_PREFIX + f.encrypt(plaintext.encode("utf-8")).decode("ascii") + + +def decrypt_secret(value: str) -> str: + """Decrypt an ``enc:v1:`` token; pass through plaintext / undecryptable values.""" + if not is_encrypted(value): + return value + f = _get() + if f is None: + return value + try: + return f.decrypt(value[len(ENC_PREFIX):].encode("ascii")).decode("utf-8") + except InvalidToken: + logger.error("Could not decrypt a stored secret (wrong/rotated key?)") + return value diff --git a/steward/core/settings.py b/steward/core/settings.py index 8fa3c2c..465d563 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -104,6 +104,24 @@ DEFAULTS: dict[str, Any] = { "reports.last_sent_at": "", } +# Settings encrypted at rest (transparent encrypt-on-write / decrypt-on-read). +# Adding a key here makes new writes ciphertext; run migrate_plaintext_secrets +# to convert any existing plaintext rows. +SECRET_KEYS: set[str] = { + "smtp.password", + "oidc.client_secret", + "ldap.bind_password", + "ansible.ssh_private_key", + "ansible.become_password", + "ansible.vault_password", +} + + +def _decode(value: Any) -> Any: + """Decrypt a stored value if it's an encrypted token; else pass through.""" + from steward.core.crypto import decrypt_secret, is_encrypted + return decrypt_secret(value) if is_encrypted(value) else value + # ───────────────────────────────────────────────────────────────────────────── # Async helpers (use inside request handlers / scheduled tasks) @@ -117,27 +135,31 @@ async def get_setting(session: AsyncSession, key: str) -> Any: row = result.scalar_one_or_none() if row is None: return DEFAULTS.get(key) - return json.loads(row.value_json) + return _decode(json.loads(row.value_json)) async def set_setting(session: AsyncSession, key: str, value: Any) -> None: - """Upsert a setting. Call inside an active transaction.""" + """Upsert a setting (encrypting secret keys at rest). Call in a transaction.""" + to_store = value + if key in SECRET_KEYS and isinstance(value, str) and value: + from steward.core.crypto import encrypt_secret + to_store = encrypt_secret(value) result = await session.execute( select(AppSetting).where(AppSetting.key == key) ) row = result.scalar_one_or_none() now = datetime.now(timezone.utc) if row is None: - session.add(AppSetting(key=key, value_json=json.dumps(value), updated_at=now)) + session.add(AppSetting(key=key, value_json=json.dumps(to_store), updated_at=now)) else: - row.value_json = json.dumps(value) + row.value_json = json.dumps(to_store) row.updated_at = now async def get_all_settings(session: AsyncSession) -> dict[str, Any]: """Return flat key→value dict with defaults filled in for missing keys.""" result = await session.execute(select(AppSetting)) - stored = {row.key: json.loads(row.value_json) for row in result.scalars()} + stored = {row.key: _decode(json.loads(row.value_json)) for row in result.scalars()} out: dict[str, Any] = {} for key, default in DEFAULTS.items(): out[key] = stored.get(key, default) @@ -216,7 +238,7 @@ def load_settings_sync(db_url: str) -> dict[str, Any]: try: async with factory() as session: result = await session.execute(select(AppSetting)) - return {row.key: json.loads(row.value_json) for row in result.scalars()} + return {row.key: _decode(json.loads(row.value_json)) for row in result.scalars()} finally: await engine.dispose() @@ -230,6 +252,37 @@ def load_settings_sync(db_url: str) -> dict[str, Any]: return out +def migrate_plaintext_secrets(db_url: str) -> int: + """Encrypt any existing plaintext secret rows in place. Idempotent. + + Returns the number of values converted. Run once at startup after the + encryptor is initialised (already-encrypted rows are skipped). + """ + from steward.core.crypto import encrypt_secret, is_encrypted + + async def _run() -> int: + from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker + engine = create_async_engine(db_url, echo=False) + factory = async_sessionmaker(engine, expire_on_commit=False) + converted = 0 + try: + async with factory() as session: + async with session.begin(): + result = await session.execute( + select(AppSetting).where(AppSetting.key.in_(SECRET_KEYS)) + ) + for row in result.scalars(): + val = json.loads(row.value_json) + if isinstance(val, str) and val and not is_encrypted(val): + row.value_json = json.dumps(encrypt_secret(val)) + converted += 1 + finally: + await engine.dispose() + return converted + + return asyncio.run(_run()) + + # ───────────────────────────────────────────────────────────────────────────── # External URL helper # ───────────────────────────────────────────────────────────────────────────── diff --git a/tests/core/test_crypto.py b/tests/core/test_crypto.py new file mode 100644 index 0000000..1cf6d57 --- /dev/null +++ b/tests/core/test_crypto.py @@ -0,0 +1,46 @@ +"""Unit tests for settings encryption-at-rest (Fernet).""" +from steward.core.crypto import ( + ENC_PREFIX, + decrypt_secret, + encrypt_secret, + init_crypto, + is_encrypted, +) + + +def setup_function(): + init_crypto("unit-test-secret-key") + + +def test_roundtrip(): + tok = encrypt_secret("hunter2") + assert tok.startswith(ENC_PREFIX) + assert tok != "hunter2" + assert is_encrypted(tok) + assert decrypt_secret(tok) == "hunter2" + + +def test_empty_passes_through(): + assert encrypt_secret("") == "" + assert is_encrypted("") is False + + +def test_plaintext_decrypt_passthrough(): + assert decrypt_secret("not-encrypted") == "not-encrypted" + + +def test_is_encrypted_type_guard(): + assert is_encrypted(encrypt_secret("x")) is True + assert is_encrypted("plain") is False + assert is_encrypted(None) is False + assert is_encrypted(123) is False + + +def test_wrong_key_does_not_reveal_plaintext(): + init_crypto("key-A") + tok = encrypt_secret("secret") + init_crypto("key-B") + # Wrong key -> InvalidToken -> token returned unchanged, never the plaintext. + assert decrypt_secret(tok) != "secret" + init_crypto("key-A") + assert decrypt_secret(tok) == "secret" From 0318f6423f30fbffea01e3f95b6afb1f2796f243 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 17:17:38 -0400 Subject: [PATCH 047/126] feat(ansible): host provisioning via steward managed SSH identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turn the agent-install playbook into a full provisioning + maintenance path. Solves the bootstrap chicken-and-egg: first contact uses an operator-supplied password (one run, never stored), which creates a dedicated `steward` login account with NOPASSWD sudo + Steward's managed public key. Every run thereafter connects as `steward` with the managed key — fully unattended (scheduled prune, agent updates). - core/crypto: generate_ssh_keypair() — ed25519, OpenSSH formats. - settings: ansible.ssh_public_key (non-secret, displayed) + ansible.ssh_user (default steward); to_ansible_cfg extended. - settings UI + route: "Generate managed key" (private encrypted, public shown to copy) + SSH-user field. - executor: build_bootstrap() writes a 0600 vars file (-e @file) for the per-run user/password — never argv, never DB, never logged; drops the managed key when a bootstrap password is given; --user floor from the global ssh_user when no override. - runner.trigger_run: pass-through `connection` kwarg, deliberately NOT persisted on AnsibleRun.params (password stays out of the DB). - bundled/host_agent/provision.yml: create steward user + authorized_keys + /etc/sudoers.d/steward (visudo-validated) + agent install. - host_agent: /provision route + "Provision a fresh host" card (bootstrap user/password; injects pubkey + user + token as space-safe JSON hostvars). - Dockerfile: add sshpass (Ansible shells out to it for password SSH). - tests: keypair generation + build_bootstrap (secret stays off argv). Co-Authored-By: Claude Opus 4.8 (1M context) --- Dockerfile | 3 + plugins/host_agent/routes.py | 84 ++++++++++ .../host_agent/templates/settings_list.html | 45 ++++++ .../ansible/bundled/host_agent/provision.yml | 143 ++++++++++++++++++ steward/ansible/executor.py | 59 +++++++- steward/ansible/runner.py | 5 + steward/core/crypto.py | 25 +++ steward/core/settings.py | 11 +- steward/settings/routes.py | 28 ++++ steward/templates/settings/ansible.html | 35 +++++ tests/core/test_crypto.py | 23 +++ tests/test_ansible_credentials.py | 36 ++++- 12 files changed, 492 insertions(+), 5 deletions(-) create mode 100644 steward/ansible/bundled/host_agent/provision.yml diff --git a/Dockerfile b/Dockerfile index c74d42d..e74ceca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \ gosu \ # openssh-client — ansible-playbook reaches remote hosts over ssh openssh-client \ + # sshpass — Ansible shells out to it for first-contact password SSH + # (provisioning a fresh host before the managed key is installed) + sshpass \ && rm -rf /var/lib/apt/lists/* # Upgrade pip to suppress the version notice diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 07e672b..269296b 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -558,6 +558,9 @@ async def settings_list(): if new_token and new_host_id: install_url = f"{public_base_url(request)}/plugins/host_agent/install.sh?token={new_token}" + managed_key_set = bool( + (current_app.config.get("ANSIBLE", {}).get("ssh_public_key") or "").strip()) + return await render_template( "settings_list.html", registrations=[ @@ -568,6 +571,7 @@ async def settings_list(): ansible_available=ansible_available, deploy_targets=targets, deploy_groups=groups, + managed_key_set=managed_key_set, ) @@ -725,3 +729,83 @@ async def deploy_via_ansible(): if err: return _error(400, "deploy_failed", err) return redirect(url_for("ansible.run_detail", run_id=run.id)) + + +@host_agent_bp.post("/provision") +@require_role(UserRole.admin) +async def provision_via_ansible(): + """First-contact provisioning: create the steward account + install the + managed key + NOPASSWD sudo, then install the agent — over bootstrap + (password) auth. After this, hosts are reachable with the managed key. + + The bootstrap password is passed to the runner as a connection override and + is NOT persisted on the run row. + """ + import json as _json + from steward.core.capabilities import has_capability, invoke_capability + from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory + from steward.ansible.sources import BUILTIN_SOURCE_NAME + + if not has_capability("ansible.run_playbook"): + return _error(400, "ansible_unavailable", "Ansible is not available") + + ansible_cfg = current_app.config.get("ANSIBLE", {}) + pubkey = (ansible_cfg.get("ssh_public_key") or "").strip() + if not pubkey: + return _error(400, "no_managed_key", + "Generate a managed SSH key in Settings → Ansible first") + steward_user = (ansible_cfg.get("ssh_user") or "steward").strip() or "steward" + + form = await request.form + scope = (form.get("inventory_scope", "") or "").strip() + boot_user = (form.get("bootstrap_user", "") or "").strip() + boot_password = form.get("bootstrap_password", "") or "" + try: + interval = max(5, int(form.get("agent_interval", "30"))) + except (TypeError, ValueError): + interval = 30 + if not (scope.startswith("steward:target:") + or scope.startswith("steward:group:") + or scope == "steward:all"): + return _error(400, "bad_scope", "Choose a target or group") + if not boot_user or not boot_password: + return _error(400, "missing_bootstrap", + "Bootstrap user and password are required for first contact") + + url = public_base_url(request) + async with current_app.db_sessionmaker() as db: + targets = await fetch_scope_targets(db, scope) + if not targets: + return _error(400, "no_targets", "No Ansible targets in that scope") + tokens: dict[str, str] = {} + async with db.begin(): + for t in targets: + host = await _ensure_host_for_target(db, t) + tokens[t.name] = await _mint_registration_token(db, host) + inv = generate_inventory(targets) + for name, tok in tokens.items(): + hv = inv["_meta"]["hostvars"].setdefault(name, {}) + hv["steward_url"] = url + hv["steward_token"] = tok + # Pubkey carries a space (the comment) and steward_user is free text — + # injected as JSON hostvars (space-safe), NOT -e k=v strings which + # Ansible would shlex-split on whitespace. + hv["steward_pubkey"] = pubkey + hv["steward_user"] = steward_user + inventory_content = _json.dumps(inv) + + actor_role = UserRole(session.get("user_role", "viewer")) + run, _source, err = await invoke_capability( + "ansible.run_playbook", actor_role, + current_app._get_current_object(), # type: ignore[attr-defined] + source_name=BUILTIN_SOURCE_NAME, + playbook_path="host_agent/provision.yml", + inventory_content=inventory_content, + inventory_scope=scope, + params={"extra_vars": [f"agent_interval={interval}"]}, + triggered_by=session.get("user_id"), + connection={"user": boot_user, "password": boot_password}, + ) + if err: + return _error(400, "provision_failed", err) + return redirect(url_for("ansible.run_detail", run_id=run.id)) diff --git a/plugins/host_agent/templates/settings_list.html b/plugins/host_agent/templates/settings_list.html index 03b6958..77fdf71 100644 --- a/plugins/host_agent/templates/settings_list.html +++ b/plugins/host_agent/templates/settings_list.html @@ -31,6 +31,51 @@
{% if ansible_available %} +
+

Provision a fresh host

+

+ First contact for a host Steward can't yet reach by key. Creates a + steward login account with passwordless sudo, installs the managed + key, then installs the agent — over a one-time bootstrap password (used for this + run only, never stored). After this, use Deploy below for updates. +

+ {% if not managed_key_set %} +

+ No managed key yet. Generate one under Settings → Ansible first. +

+ {% elif not (deploy_targets or deploy_groups) %} +

+ No Ansible inventory targets yet. Add some under Ansible → Inventory. +

+ {% else %} +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+ {% endif %} +
+

Deploy via Ansible

diff --git a/steward/ansible/bundled/host_agent/provision.yml b/steward/ansible/bundled/host_agent/provision.yml new file mode 100644 index 0000000..db35589 --- /dev/null +++ b/steward/ansible/bundled/host_agent/provision.yml @@ -0,0 +1,143 @@ +--- +# First-contact provisioning: bring a fresh host under Steward management, then +# install the host agent — all in one idempotent run. +# +# Run this ONCE per host with bootstrap credentials (an existing login + sudo, +# typically password auth via the "Provision host" button). It: +# 1. creates a dedicated `steward` login account, +# 2. installs Steward's managed public key into its authorized_keys, +# 3. grants it passwordless sudo, +# 4. installs/updates the host agent. +# After this, every future run (agent updates, maintenance) connects as +# `steward` with the managed key — no operator credentials needed. +# +# Extra-vars (injected by the Provision card): +# steward_url, steward_token — agent → Steward auth (per-host) +# steward_pubkey — managed public key to authorize +# steward_user (default steward), agent_interval (default 30) +- name: Provision host for Steward + install agent + hosts: all + become: true + gather_facts: false + vars: + steward_url: "" + steward_token: "" + steward_pubkey: "" + steward_user: steward + agent_interval: 30 + tasks: + - name: Require provisioning vars + ansible.builtin.assert: + that: + - steward_url | length > 0 + - steward_token | length > 0 + - steward_pubkey | length > 0 + fail_msg: "Pass steward_url, steward_token and steward_pubkey as extra-vars." + + # ── Managed login account ──────────────────────────────────────────────── + - name: Create the steward management account + ansible.builtin.user: + name: "{{ steward_user }}" + shell: /bin/sh + create_home: true + state: present + + - name: Ensure ~/.ssh exists + ansible.builtin.file: + path: "/home/{{ steward_user }}/.ssh" + state: directory + owner: "{{ steward_user }}" + group: "{{ steward_user }}" + mode: "0700" + + - name: Authorize the managed public key + ansible.builtin.lineinfile: + path: "/home/{{ steward_user }}/.ssh/authorized_keys" + line: "{{ steward_pubkey }}" + create: true + owner: "{{ steward_user }}" + group: "{{ steward_user }}" + mode: "0600" + + - name: Grant passwordless sudo to the steward account + ansible.builtin.copy: + dest: "/etc/sudoers.d/steward" + content: "{{ steward_user }} ALL=(ALL) NOPASSWD:ALL\n" + owner: root + group: root + mode: "0440" + validate: "visudo -cf %s" + + # ── Host agent (same as install.yml) ───────────────────────────────────── + - name: Create steward-agent system user + ansible.builtin.user: + name: steward-agent + system: true + shell: /usr/sbin/nologin + create_home: false + + - name: Create agent directory + ansible.builtin.file: + path: /usr/local/lib/steward-agent + state: directory + mode: "0755" + + - name: Download agent.py + ansible.builtin.get_url: + url: "{{ steward_url }}/plugins/host_agent/agent.py" + dest: /usr/local/lib/steward-agent/agent.py + mode: "0755" + force: true + notify: restart steward-agent + + - name: Write agent config + ansible.builtin.copy: + dest: /etc/steward-agent.conf + owner: root + group: steward-agent + mode: "0640" + content: | + url = {{ steward_url }} + token = {{ steward_token }} + interval_seconds = {{ agent_interval }} + notify: restart steward-agent + + - name: Install systemd unit + ansible.builtin.copy: + dest: /etc/systemd/system/steward-agent.service + mode: "0644" + content: | + [Unit] + Description=Steward host agent + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + User=steward-agent + Environment=STEWARD_AGENT_CONFIG=/etc/steward-agent.conf + ExecStart=/usr/bin/env python3 /usr/local/lib/steward-agent/agent.py + Restart=on-failure + RestartSec=5 + NoNewPrivileges=true + ProtectSystem=strict + ProtectHome=true + PrivateTmp=true + + [Install] + WantedBy=multi-user.target + notify: restart steward-agent + + - name: Enable and start the agent + ansible.builtin.systemd: + name: steward-agent + enabled: true + state: started + daemon_reload: true + + handlers: + - name: restart steward-agent + ansible.builtin.systemd: + name: steward-agent + state: restarted + daemon_reload: true diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py index dcfa9b6..f8c7ec8 100644 --- a/steward/ansible/executor.py +++ b/steward/ansible/executor.py @@ -155,6 +155,37 @@ def build_credentials(creds: dict | None, tmpdir: str) -> tuple[list[str], list[ return args, files +def build_bootstrap(connection: dict | None, tmpdir: str) -> tuple[list[str], list[tuple[str, str]]]: + """Per-run connection override for first-contact provisioning. + + Pure: returns (extra argv, files to write). The bootstrap user/password are + written to a temp vars file (``-e @file``) — never argv, never the DB — so + they don't leak via the process list or persist on the AnsibleRun row. + A bare connection password doubles as the become password unless one is + given explicitly. connection keys: user, password, become_password. + """ + args: list[str] = [] + files: list[tuple[str, str]] = [] + connection = connection or {} + user = (connection.get("user") or "").strip() + password = connection.get("password") or "" + if not user and not password: + return args, files + + lines: dict[str, str] = {} + if user: + lines["ansible_user"] = user + if password: + lines["ansible_password"] = password + lines["ansible_become_password"] = connection.get("become_password") or password + # JSON values are valid YAML and keep arbitrary characters safe. + content = "".join(f"{k}: {json.dumps(v)}\n" for k, v in lines.items()) + path = os.path.join(tmpdir, "bootstrap.yml") + files.append((path, content)) + args += ["-e", "@" + path] + return args, files + + def ansible_env(creds: dict | None, base_env) -> dict: """Return a copy of base_env with ANSIBLE_HOST_KEY_CHECKING set from creds.""" env = dict(base_env) @@ -250,12 +281,17 @@ async def start_run( source_path: str, params: dict | None = None, inventory_content: str | None = None, + connection: dict | None = None, ) -> None: """Execute ansible-playbook as a subprocess and update the DB run row. Respects a global concurrency cap (app._ansible_semaphore): if no slot is free the run shows as 'queued' until one frees. Supports cancellation, a persistent full-log artifact, and a parsed per-host result summary. + + connection is an optional per-run SSH override (user/password) for + first-contact provisioning. It is applied to the subprocess only — it is + never persisted on the AnsibleRun row, never placed on argv, and not logged. """ from steward.models.ansible import AnsibleRunStatus @@ -328,14 +364,31 @@ async def start_run( inv_target = inventory_path cmd = build_ansible_command(playbook_path, inv_target, params) - cred_args, cred_files = build_credentials(creds, tmpdir) - for cred_path, content in cred_files: + # First-contact provisioning supplies a password and a bootstrap + # user; the managed key isn't on the host yet, so don't offer it + # (avoids a failed key attempt before the password fallback). + conn = connection or {} + effective_creds = dict(creds) + if conn.get("password"): + effective_creds.pop("ssh_private_key", None) + + cred_args, cred_files = build_credentials(effective_creds, tmpdir) + boot_args, boot_files = build_bootstrap(conn, tmpdir) + for cred_path, content in cred_files + boot_files: with open(cred_path, "w", encoding="utf-8") as cf: cf.write(content) os.chmod(cred_path, 0o600) + # Steady-state floor: connect as the managed account unless a target + # var or the bootstrap override (extra-vars, higher precedence) wins. + user_args: list[str] = [] + if not conn.get("user"): + ssh_user = (creds.get("ssh_user") or "").strip() + if ssh_user: + user_args += ["--user", ssh_user] + proc = await asyncio.create_subprocess_exec( - *cmd, *cred_args, + *cmd, *cred_args, *boot_args, *user_args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, cwd=cwd, diff --git a/steward/ansible/runner.py b/steward/ansible/runner.py index a8ea6cd..5b031ef 100644 --- a/steward/ansible/runner.py +++ b/steward/ansible/runner.py @@ -25,6 +25,7 @@ async def trigger_run( params: dict | None = None, triggered_by: str | None = None, inventory_content: str | None = None, + connection: dict | None = None, ): """Resolve inventory for the scope, persist an AnsibleRun, and launch it. @@ -32,6 +33,9 @@ async def trigger_run( If inventory_content is provided, it is used verbatim and scope resolution is skipped (the caller built a bespoke inventory — e.g. host_agent deploy injecting per-host tokens); inventory_scope is still recorded for display. + connection is an optional per-run SSH override (user/password) for + first-contact provisioning — passed to the executor but deliberately NOT + stored on the AnsibleRun row (params), so the password never lands in the DB. Returns (run, source, error): on success error is None; on failure run is None and error is a short human-readable reason. """ @@ -73,6 +77,7 @@ async def trigger_run( executor.start_run( app, run_id, playbook_path, inventory_path or "", source["path"], params or None, inventory_content, + connection=connection, ) ) task.add_done_callback( diff --git a/steward/core/crypto.py b/steward/core/crypto.py index 2d5a1fb..44595a7 100644 --- a/steward/core/crypto.py +++ b/steward/core/crypto.py @@ -19,6 +19,8 @@ import os from pathlib import Path from cryptography.fernet import Fernet, InvalidToken +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey logger = logging.getLogger(__name__) @@ -91,3 +93,26 @@ def decrypt_secret(value: str) -> str: except InvalidToken: logger.error("Could not decrypt a stored secret (wrong/rotated key?)") return value + + +def generate_ssh_keypair(comment: str = "steward-managed") -> tuple[str, str]: + """Mint a fresh ed25519 keypair for Steward's managed Ansible identity. + + Returns (private_openssh_pem, public_openssh_line). ed25519 is small, fast, + and universally supported by modern OpenSSH. The private key is unencrypted + OpenSSH PEM (Steward stores it encrypted-at-rest itself); the public key is + the single-line ``ssh-ed25519 AAAA… comment`` form for authorized_keys. + """ + key = Ed25519PrivateKey.generate() + private_pem = key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.OpenSSH, + encryption_algorithm=serialization.NoEncryption(), + ).decode("ascii") + public_line = key.public_key().public_bytes( + encoding=serialization.Encoding.OpenSSH, + format=serialization.PublicFormat.OpenSSH, + ).decode("ascii") + if comment: + public_line = f"{public_line} {comment}" + return private_pem, public_line diff --git a/steward/core/settings.py b/steward/core/settings.py index 465d563..e08b03d 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -52,10 +52,17 @@ DEFAULTS: dict[str, Any] = { "webhook.template": _DEFAULT_WEBHOOK_TEMPLATE, "ansible.sources": [], # Ansible credentials — global, used by every run (manual + alert-triggered). - # Plaintext at rest, masked in the UI (encryption-at-rest tracked separately). + # ssh_private_key/become/vault are encrypted at rest (see SECRET_KEYS). "ansible.ssh_private_key": "", "ansible.become_password": "", "ansible.vault_password": "", + # Public half of the managed keypair — non-secret, displayed so it can be + # sprayed onto hosts (provisioning installs it into ~steward/.ssh). + "ansible.ssh_public_key": "", + # Default SSH login for steady-state runs — the dedicated account + # provisioning creates. Acts as a floor (--user); a target's ansible_user + # or a per-run bootstrap override still wins. + "ansible.ssh_user": "steward", "ansible.host_key_checking": False, # Max simultaneous playbook runs; extra runs queue. Applied at app start. "ansible.max_concurrent_runs": 3, @@ -198,6 +205,8 @@ def to_ansible_cfg(settings: dict[str, Any]) -> dict: "ssh_private_key": settings.get("ansible.ssh_private_key", ""), "become_password": settings.get("ansible.become_password", ""), "vault_password": settings.get("ansible.vault_password", ""), + "ssh_public_key": settings.get("ansible.ssh_public_key", ""), + "ssh_user": settings.get("ansible.ssh_user", "steward"), "host_key_checking": settings.get("ansible.host_key_checking", False), "max_concurrent_runs": settings.get("ansible.max_concurrent_runs", 3), } diff --git a/steward/settings/routes.py b/steward/settings/routes.py index fcba65b..8944f58 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -178,11 +178,36 @@ async def ansible(): ssh_key_set=bool(settings.get("ansible.ssh_private_key")), become_set=bool(settings.get("ansible.become_password")), vault_set=bool(settings.get("ansible.vault_password")), + ssh_public_key=settings.get("ansible.ssh_public_key", ""), + ssh_user=settings.get("ansible.ssh_user", "steward"), host_key_checking=settings.get("ansible.host_key_checking", False), max_concurrent_runs=settings.get("ansible.max_concurrent_runs", 3), ) +@settings_bp.post("/ansible/generate-key") +@require_role(UserRole.admin) +async def ansible_generate_key(): + """Mint a fresh managed ed25519 keypair (private encrypted, public shown). + + This is the reusable Steward identity: provisioning installs the public half + on every host, and steady-state runs authenticate with the private half. + Regenerating invalidates access to already-provisioned hosts until they are + re-provisioned, so the UI confirms before calling this. + """ + from steward.core.crypto import generate_ssh_keypair + + private_pem, public_line = generate_ssh_keypair() + async with current_app.db_sessionmaker() as db: + async with db.begin(): + await set_setting(db, "ansible.ssh_private_key", private_pem) + await set_setting(db, "ansible.ssh_public_key", public_line) + await _reload_app_config() + await log_audit(current_app, session.get("user_id"), session.get("username", ""), + "settings.saved", detail={"section": "ansible.managed_key"}) + return redirect(url_for("settings.ansible")) + + @settings_bp.post("/ansible/credentials") @require_role(UserRole.admin) async def ansible_save_credentials(): @@ -203,6 +228,9 @@ async def ansible_save_credentials(): val = raw.strip("\n") if field == "ssh_private_key" else raw.strip() if val: await set_setting(db, key, val) + ssh_user = (form.get("ssh_user", "") or "").strip() + if ssh_user: + await set_setting(db, "ansible.ssh_user", ssh_user) await set_setting(db, "ansible.host_key_checking", "host_key_checking" in form) try: mcr = max(1, min(50, int(form.get("max_concurrent_runs", 3)))) diff --git a/steward/templates/settings/ansible.html b/steward/templates/settings/ansible.html index 27d0602..2bf1611 100644 --- a/steward/templates/settings/ansible.html +++ b/steward/templates/settings/ansible.html @@ -21,6 +21,35 @@ {# ── Credentials ──────────────────────────────────────────────────────────── #} {% set cfg_badge = '(configured)' %} {% set not_set_badge = '(not set)' %} + + {# ── Managed identity (provisioning) ───────────────────────────────────────── #} +

+
Managed SSH identity
+

+ Steward's reusable Ansible identity. Provisioning installs the public + key into each host's {{ ssh_user }} account; steady-state runs authenticate + with the private half (stored encrypted). Generate it once, then provision hosts from + Host Agents. +

+ {% if ssh_public_key %} +
+ + + Click to select — add to a host's authorized_keys to authorize Steward manually. +
+
+ +
+ {% else %} +
+ + ed25519 — created server-side, nothing to paste. +
+ {% endif %} +
+
Credentials

@@ -56,6 +85,12 @@ Clear {% endif %}

+
+ + +
{% endfor %} diff --git a/tests/test_playbook_variables.py b/tests/test_playbook_variables.py new file mode 100644 index 0000000..560cf03 --- /dev/null +++ b/tests/test_playbook_variables.py @@ -0,0 +1,110 @@ +"""Unit tests for playbook variable discovery + run-time extra-vars file.""" +import json + +from steward.ansible.executor import build_extra_vars_file +from steward.ansible.sources import discover_playbook_variables + + +def _names(variables): + return [v["name"] for v in variables] + + +def test_vars_block_defaults_surface_as_fields(): + content = """ +- hosts: all + vars: + steward_url: "" + agent_interval: 30 + enabled: true + tasks: [] +""" + variables = discover_playbook_variables(content) + assert _names(variables) == ["steward_url", "agent_interval", "enabled"] + interval = next(v for v in variables if v["name"] == "agent_interval") + assert interval["default"] == 30 + assert interval["required"] is False + assert interval["secret"] is False + + +def test_secretish_names_flagged(): + content = """ +- hosts: all + vars: + db_password: "" + api_token: "" + vault_secret: "" + plain_value: "x" +""" + by = {v["name"]: v for v in discover_playbook_variables(content)} + assert by["db_password"]["secret"] is True + assert by["api_token"]["secret"] is True + assert by["vault_secret"]["secret"] is True + assert by["plain_value"]["secret"] is False + + +def test_vars_prompt_required_and_private(): + content = """ +- hosts: all + vars_prompt: + - name: release_tag + prompt: "Which release?" + - name: admin_pw + prompt: "Admin password" + private: true + default: "" + tasks: [] +""" + by = {v["name"]: v for v in discover_playbook_variables(content)} + # No default → required; vars_prompt defaults to private=yes → secret. + assert by["release_tag"]["required"] is True + assert by["release_tag"]["secret"] is True + assert by["release_tag"]["prompt"] == "Which release?" + # Explicit private + has default → secret but not required. + assert by["admin_pw"]["secret"] is True + assert by["admin_pw"]["required"] is False + + +def test_vars_prompt_wins_on_name_collision(): + content = """ +- hosts: all + vars_prompt: + - name: dup + prompt: "from prompt" + private: false + vars: + dup: "from vars" +""" + variables = discover_playbook_variables(content) + assert _names(variables) == ["dup"] + assert variables[0]["prompt"] == "from prompt" + + +def test_non_scalar_vars_skipped(): + content = """ +- hosts: all + vars: + scalar: 1 + a_list: [1, 2] + a_map: {k: v} +""" + assert _names(discover_playbook_variables(content)) == ["scalar"] + + +def test_malformed_yaml_returns_empty(): + assert discover_playbook_variables("this: : : not valid") == [] + assert discover_playbook_variables("just a string") == [] + + +def test_extra_vars_file_roundtrip_and_space_safe(): + merged = {"agent_interval": "30", "msg": "hello world", "q": 'a "quote"'} + args, files = build_extra_vars_file(merged, "/td") + assert args == ["-e", "@/td/extravars.json"] + path, content = files[0] + assert path == "/td/extravars.json" + # JSON keeps spaces/quotes intact — no shlex hazard like -e key=value. + assert json.loads(content) == merged + + +def test_extra_vars_file_empty_is_noop(): + assert build_extra_vars_file({}, "/td") == ([], []) + assert build_extra_vars_file(None, "/td") == ([], []) From 6a8146b54412749ce80bbb7b9a48562f038937ed Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 19:22:12 -0400 Subject: [PATCH 049/126] fix(ansible): replace steward key on reprovision + distinct agent update path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provisioning review corrections + the matching frontend, plus breadcrumb header integration. - provision.yml: authorize the managed pubkey with a regexp match on the ' steward-managed' comment so rotating the key REPLACES the host's steward key in place instead of stacking a second authorized entry. Hand-added keys (other comments) are untouched. - update.yml (new): refresh agent.py + restart only. Does NOT rotate the token or rewrite /etc/steward-agent.conf — the host keeps its identity. Asserts the agent is already installed and fails clearly otherwise. - host_agent /update route: runs update.yml as the managed steward user (no token minting). Token rotation stays a deliberate action. - settings/ansible/generate-key honors a safe relative `next` redirect, so an inline trigger elsewhere returns to its page. - Host Agents settings: reworked into a clear lifecycle — an intro card that explains it runs Ansible to deploy the agent (+ inline "generate managed key" warning/trigger when none exists), then three labelled cards: 1 Provision, 2 Install/enroll, 3 Update. Each explains what it does. - base.html: breadcrumb now renders as a kicker line directly above the page title (moved below alerts, tightened margin) so nested and top-level views share one consistent header. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 35 ++++++ .../host_agent/templates/settings_list.html | 105 +++++++++++------- .../ansible/bundled/host_agent/provision.yml | 6 +- steward/ansible/bundled/host_agent/update.yml | 47 ++++++++ steward/settings/routes.py | 5 + steward/templates/base.html | 6 +- 6 files changed, 161 insertions(+), 43 deletions(-) create mode 100644 steward/ansible/bundled/host_agent/update.yml diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 269296b..da49482 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -809,3 +809,38 @@ async def provision_via_ansible(): if err: return _error(400, "provision_failed", err) return redirect(url_for("ansible.run_detail", run_id=run.id)) + + +@host_agent_bp.post("/update") +@require_role(UserRole.admin) +async def update_via_ansible(): + """Refresh agent.py on already-installed hosts via the bundled update + playbook. Connects as the managed steward account — no token rotation, no + config rewrite (the host keeps its existing identity).""" + from steward.core.capabilities import has_capability, invoke_capability + from steward.ansible.sources import BUILTIN_SOURCE_NAME + + if not has_capability("ansible.run_playbook"): + return _error(400, "ansible_unavailable", "Ansible is not available") + + form = await request.form + scope = (form.get("inventory_scope", "") or "").strip() + if not (scope.startswith("steward:target:") + or scope.startswith("steward:group:") + or scope == "steward:all"): + return _error(400, "bad_scope", "Choose a target or group") + + url = public_base_url(request) + actor_role = UserRole(session.get("user_role", "viewer")) + run, _source, err = await invoke_capability( + "ansible.run_playbook", actor_role, + current_app._get_current_object(), # type: ignore[attr-defined] + source_name=BUILTIN_SOURCE_NAME, + playbook_path="host_agent/update.yml", + inventory_scope=scope, + params={"extra_vars": [f"steward_url={url}"]}, + triggered_by=session.get("user_id"), + ) + if err: + return _error(400, "update_failed", err) + return redirect(url_for("ansible.run_detail", run_id=run.id)) diff --git a/plugins/host_agent/templates/settings_list.html b/plugins/host_agent/templates/settings_list.html index 77fdf71..7cbff8f 100644 --- a/plugins/host_agent/templates/settings_list.html +++ b/plugins/host_agent/templates/settings_list.html @@ -31,41 +31,63 @@
{% if ansible_available %} +{% set scope_select %} + + +{% endset %} +
-

Provision a fresh host

-

- First contact for a host Steward can't yet reach by key. Creates a - steward login account with passwordless sudo, installs the managed - key, then installs the agent — over a one-time bootstrap password (used for this - run only, never stored). After this, use Deploy below for updates. +

Agent lifecycle via Ansible

+

+ These actions run bundled Ansible playbooks to deploy and maintain the + Steward monitoring agent on your inventory hosts. + Steward generates each host's API token automatically and connects over SSH as the managed + steward account — you'll be dropped into the live Ansible run to watch it.

+ {% if not managed_key_set %} -

- No managed key yet. Generate one under Settings → Ansible first. -

+
+ ⚠ + + No managed SSH key yet — Steward needs one to log into hosts as steward. + +
+ + +
+
{% elif not (deploy_targets or deploy_groups) %} -

+

No Ansible inventory targets yet. Add some under Ansible → Inventory.

- {% else %} + {% endif %} +
+ +{% if managed_key_set and (deploy_targets or deploy_groups) %} +
+

1 · Provision a fresh host

+

+ First contact for a brand-new host. Connects with a one-time bootstrap + user + password (used for this run only, never stored), creates the steward + login account with passwordless sudo, installs the managed SSH key, then installs the + agent. Run this once per host. +

-
- - -
+
{{ scope_select }}
- +
- +
@@ -73,37 +95,40 @@
- {% endif %}
-

Deploy via Ansible

+

2 · Install / enroll agent

- Install (or update) the agent on Ansible inventory hosts in one run — no per-host curl | sh. - A fresh token is minted per host and injected into the run; existing agents are rotated to the new token. + For a host that's already provisioned (has the steward account) but has no + agent yet — or to re-enroll one. Connects as steward with the managed key, + mints a fresh token, and installs the agent. No curl | sh needed.

- {% if not (deploy_targets or deploy_groups) %} -

- No Ansible inventory targets yet. Add some under Ansible → Browse. -

- {% else %}
-
- - -
+
{{ scope_select }}
- +
- {% endif %}
+ +
+

3 · Update agents

+

+ Refresh the agent binary on hosts already running it. Connects as steward + with the managed key and restarts the service — the token and config are left untouched, + so identity is preserved. +

+
+
{{ scope_select }}
+ +
+
+{% endif %} {% endif %}
diff --git a/steward/ansible/bundled/host_agent/provision.yml b/steward/ansible/bundled/host_agent/provision.yml index db35589..73be606 100644 --- a/steward/ansible/bundled/host_agent/provision.yml +++ b/steward/ansible/bundled/host_agent/provision.yml @@ -50,9 +50,13 @@ group: "{{ steward_user }}" mode: "0700" - - name: Authorize the managed public key + - name: Authorize the managed public key (replace any prior steward-managed key) ansible.builtin.lineinfile: path: "/home/{{ steward_user }}/.ssh/authorized_keys" + # Match on the ' steward-managed' comment so a rotated key REPLACES the + # old one in place rather than stacking a second authorized key. Any + # keys the operator added by hand (different comment) are left untouched. + regexp: ' steward-managed$' line: "{{ steward_pubkey }}" create: true owner: "{{ steward_user }}" diff --git a/steward/ansible/bundled/host_agent/update.yml b/steward/ansible/bundled/host_agent/update.yml new file mode 100644 index 0000000..8dff5ff --- /dev/null +++ b/steward/ansible/bundled/host_agent/update.yml @@ -0,0 +1,47 @@ +--- +# Update an already-installed Steward host agent: refresh agent.py and restart. +# +# Deliberately does NOT touch the registration token or /etc/steward-agent.conf +# — the host already has a working token, so an update leaves identity alone and +# only swaps the agent binary. Connects as the managed `steward` account (set +# globally as the SSH user); no bootstrap password or token injection needed. +# +# For a fresh host with no agent, use provision.yml (new host) or install.yml +# (already-provisioned host) instead — those mint and install the token. +- name: Update Steward host agent + hosts: all + become: true + gather_facts: false + vars: + steward_url: "" + tasks: + - name: Require steward_url + ansible.builtin.assert: + that: + - steward_url | length > 0 + fail_msg: "Pass steward_url as an extra-var (the Update action sets it)." + + - name: Confirm the agent is already installed + ansible.builtin.stat: + path: /usr/local/lib/steward-agent/agent.py + register: agent_file + + - name: Fail clearly if the agent is missing + ansible.builtin.fail: + msg: "No agent at /usr/local/lib/steward-agent — provision or install this host first." + when: not agent_file.stat.exists + + - name: Refresh agent.py + ansible.builtin.get_url: + url: "{{ steward_url }}/plugins/host_agent/agent.py" + dest: /usr/local/lib/steward-agent/agent.py + mode: "0755" + force: true + notify: restart steward-agent + + handlers: + - name: restart steward-agent + ansible.builtin.systemd: + name: steward-agent + state: restarted + daemon_reload: true diff --git a/steward/settings/routes.py b/steward/settings/routes.py index 8944f58..64d623c 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -197,6 +197,7 @@ async def ansible_generate_key(): """ from steward.core.crypto import generate_ssh_keypair + form = await request.form private_pem, public_line = generate_ssh_keypair() async with current_app.db_sessionmaker() as db: async with db.begin(): @@ -205,6 +206,10 @@ async def ansible_generate_key(): await _reload_app_config() await log_audit(current_app, session.get("user_id"), session.get("username", ""), "settings.saved", detail={"section": "ansible.managed_key"}) + # Allow the inline trigger on other pages (e.g. Host Agents) to return there. + nxt = (form.get("next", "") or "").strip() + if nxt.startswith("/") and not nxt.startswith("//"): + return redirect(nxt) return redirect(url_for("settings.ansible")) diff --git a/steward/templates/base.html b/steward/templates/base.html index 7c082ca..2e81c39 100644 --- a/steward/templates/base.html +++ b/steward/templates/base.html @@ -146,7 +146,9 @@ textarea { resize: vertical; } .empty { color: var(--text-muted); font-size: 0.9rem; padding: 2rem; text-align: center; } /* Breadcrumbs */ -.breadcrumb { display:flex; align-items:center; flex-wrap:wrap; gap:0.35rem; font-size:0.8rem; color:var(--text-dim); margin-bottom:1rem; } +/* Breadcrumb renders as a kicker line tucked directly above the page header + (.page-title), so nested and top-level views share one consistent header. */ +.breadcrumb { display:flex; align-items:center; flex-wrap:wrap; gap:0.35rem; font-size:0.78rem; color:var(--text-dim); margin:0 0 0.45rem; } .breadcrumb a { color:var(--text-muted); } .breadcrumb a:hover { color:var(--text); } .breadcrumb-sep { color:var(--border-mid); } @@ -235,7 +237,6 @@ function setTimeRange(val) {
- {% block breadcrumb %}{% endblock %} {% if plugin_failures and session.user_role == 'admin' %}
{{ error }}
{% endif %} + {% block breadcrumb %}{% endblock %} {% block content %}{% endblock %}
From 8bdf07f709bfa5ca99e47dfb4a134f75e07a0ba3 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 20:29:34 -0400 Subject: [PATCH 050/126] =?UTF-8?q?feat(hosts):=20Phase=201=20=E2=80=94=20?= =?UTF-8?q?host=20detail=20hub=20page=20(unify=20host=20IA)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make Hosts the front-and-center hub. A host now has a real detail page at /hosts/ that pulls its facets into one view, instead of management being scattered across a nav-less Host-Agents area and the edit form. - hosts: new GET /hosts/ detail route + hosts/detail.html. Shows the monitors summary (ping/DNS status + latency + uptime 24h/7d/30d), an Ansible section (linked target, link/create, run-playbook), and an embedded Agent panel. Hosts list name links here; ansible-link redirects here. - host_agent: GET /plugins/host_agent/panel/ — a self-contained HTMX fragment embedded into the core hub across the plugin boundary (core never imports plugin models). Shows live agent metrics + Update/Rotate/Remove when installed, or the provisioning path when not: inline "generate managed key" warning, a prompt to link an Ansible target first, then Provision (bootstrap password) / Install (managed key) tied to the host's target scope. Part of milestone 70 (Hosts hub). Phase 2+ will enrich the list, redirect the old fleet/settings pages, and re-taxonomize plugins into capabilities vs integrations. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 37 ++++++ plugins/host_agent/templates/panel.html | 117 ++++++++++++++++++ steward/hosts/routes.py | 36 +++++- steward/templates/hosts/detail.html | 156 ++++++++++++++++++++++++ steward/templates/hosts/list.html | 2 +- 5 files changed, 346 insertions(+), 2 deletions(-) create mode 100644 plugins/host_agent/templates/panel.html create mode 100644 steward/templates/hosts/detail.html diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index da49482..1863141 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -532,6 +532,43 @@ def _new_token_pair() -> tuple[str, str]: return raw, _hash_token(raw) +@host_agent_bp.get("/panel/") +@require_role(UserRole.viewer) +async def host_panel(host_id: str): + """Per-host agent panel embedded into the core Hosts hub via HTMX. + + Shows live agent metrics if the host reports, otherwise the provisioning + actions — tied to the host's linked Ansible target (the SSH connection). + Self-contained fragment (no base.html) so it can be hx-swapped in. + """ + from steward.core.capabilities import has_capability + from steward.models.ansible_inventory import AnsibleTarget + + async with current_app.db_sessionmaker() as db: + host = (await db.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return "", 404 + reg = (await db.execute(select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + latest = await _latest_metrics_for_host(db, host.name) if reg else {} + target = (await db.execute(select(AnsibleTarget).where( + AnsibleTarget.host_id == host_id))).scalar_one_or_none() + + hostlvl = latest.get(host.name, {}) + ls = reg.last_seen_at if reg else None + stale = bool(reg) and ( + ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()) + + ansible_cfg = current_app.config.get("ANSIBLE", {}) + return await render_template( + "panel.html", + host=host, reg=reg, hostlvl=hostlvl, stale=stale, target=target, + ansible_available=has_capability("ansible.run_playbook"), + managed_key_set=bool((ansible_cfg.get("ssh_public_key") or "").strip()), + ) + + @host_agent_bp.get("/settings/") @require_role(UserRole.admin) async def settings_list(): diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html new file mode 100644 index 0000000..6391d19 --- /dev/null +++ b/plugins/host_agent/templates/panel.html @@ -0,0 +1,117 @@ +{# Per-host agent panel — embedded into /hosts/ via HTMX. Self-contained. #} +{% set scope = "steward:target:" ~ target.id if target else "" %} +
+
+

Agent

+ {% if reg %} + + {{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %} + {% if reg.last_seen_at %} · last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }}{% endif %} + + {% endif %} +
+ + {% if reg %} + {# ── Installed: at-a-glance metrics + lifecycle ── #} +
+ {% set cpu = hostlvl.get('cpu_pct') %} + {% set mem = hostlvl.get('mem_used_pct') %} + {% set disk = hostlvl.get('disk_used_pct_worst') %} + {% set load1 = hostlvl.get('load_1m') %} +
CPU
+
{{ '%.0f%%'|format(cpu) if cpu is not none else '—' }}
+
Memory
+
{{ '%.0f%%'|format(mem) if mem is not none else '—' }}
+
Disk (worst)
+
{{ '%.0f%%'|format(disk) if disk is not none else '—' }}
+
Load 1m
+
{{ '%.2f'|format(load1) if load1 is not none else '—' }}
+
+ +
+ Full metrics → + {% if session.user_role == 'admin' %} + {% if ansible_available and target %} +
+ + +
+ {% endif %} +
+ +
+
+ +
+ {% endif %} +
+ {% if ansible_available and not target %} +

+ Link an Ansible target (above) to enable one-click agent updates. +

+ {% endif %} + + {% else %} + {# ── Not installed: provisioning path ── #} +

+ No agent installed. The agent reports CPU, memory, disk, network and more back to Steward. +

+ + {% if session.user_role != 'admin' %} +

An admin can install the agent here.

+ + {% elif not ansible_available %} +

+ Ansible isn't available, so the agent can't be deployed from here. Install it manually with the + curl install command. +

+ + {% elif not managed_key_set %} +
+ ⚠ + No managed SSH key yet — Steward needs one to log into hosts. +
+ + +
+
+ + {% elif not target %} +

+ Link or create an Ansible target for this host (in the Ansible section below) first — provisioning + needs an SSH connection. +

+ + {% else %} +

+ Deploys the agent to {{ target.name }} by running an Ansible playbook. Steward mints + the host's API token automatically; you'll watch the run live. +

+ {# Provision: brand-new host (creates steward account + key over a one-time password) #} +
+ +
Provision (first contact — fresh host):
+
+ + +
+
+ + +
+ +
+ {# Install: host already has the steward account (managed key works) #} +
+ + Already provisioned? + +
+ {% endif %} + {% endif %} +
diff --git a/steward/hosts/routes.py b/steward/hosts/routes.py index 56b66d1..c9a903e 100644 --- a/steward/hosts/routes.py +++ b/steward/hosts/routes.py @@ -115,6 +115,40 @@ async def list_hosts(): ) +@hosts_bp.get("/") +@require_role(UserRole.viewer) +async def host_detail(host_id: str): + """The host hub: monitors + agent (embedded fragment) + Ansible, one view.""" + from steward.models.ansible_inventory import AnsibleTarget + from sqlalchemy.orm import selectinload + + async with current_app.db_sessionmaker() as db: + host = (await db.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return "Not found", 404 + ping = (await db.execute( + select(PingResult).where(PingResult.host_id == host_id) + .order_by(PingResult.probed_at.desc()).limit(1))).scalar_one_or_none() + dns = (await db.execute( + select(DnsResult).where(DnsResult.host_id == host_id) + .order_by(DnsResult.resolved_at.desc()).limit(1))).scalar_one_or_none() + uptime = (await _compute_uptime(db)).get(host_id) + linked_target = (await db.execute( + select(AnsibleTarget).where(AnsibleTarget.host_id == host_id) + .options(selectinload(AnsibleTarget.groups)))).scalar_one_or_none() + linkable_targets = (await db.execute( + select(AnsibleTarget).where(AnsibleTarget.host_id.is_(None)) + .order_by(AnsibleTarget.name))).scalars().all() + + return await render_template( + "hosts/detail.html", + host=host, ping=ping, dns=dns, uptime=uptime, + linked_target=linked_target, linkable_targets=linkable_targets, + ansible_sources=_ansible_source_names(), + ) + + @hosts_bp.get("/new") @require_role(UserRole.operator) async def new_host(): @@ -250,7 +284,7 @@ async def ansible_link(host_id: str): ) db.add(new_target) - return redirect(f"/hosts/{host_id}/edit") + return redirect(f"/hosts/{host_id}") @hosts_bp.post("//run-playbook") diff --git a/steward/templates/hosts/detail.html b/steward/templates/hosts/detail.html new file mode 100644 index 0000000..81217c5 --- /dev/null +++ b/steward/templates/hosts/detail.html @@ -0,0 +1,156 @@ +{% extends "base.html" %} +{% from "_macros.html" import crumbs %} +{% block title %}{{ host.name }} — Hosts — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), (host.name, "")]) }}{% endblock %} +{% block content %} + +
+
+

{{ host.name }}

+
+ {{ host.address or "no address — resolves by name" }} +
+
+
+ Edit + {% if session.user_role == 'admin' %} +
+ +
+ {% endif %} +
+
+ +{# ── Monitors ─────────────────────────────────────────────────────────────── #} +
+
+

Monitors

+ Configure → +
+
+
+
Ping
+ {% if not host.ping_enabled %} +
off
+ {% elif ping is none %} +
no data yet
+ {% else %} +
+ {{ ping.status.value }} +
+ {% if ping.response_time_ms is not none %} +
{{ '%.0f'|format(ping.response_time_ms) }} ms
+ {% endif %} + {% endif %} +
+
+
DNS
+ {% if not host.dns_enabled %} +
off
+ {% elif dns is none %} +
no data yet
+ {% else %} +
+ {{ dns.status.value }} +
+ {% if dns.resolved_ip %} +
{{ dns.resolved_ip }}
+ {% endif %} + {% endif %} +
+
+
Uptime
+ {% if uptime %} +
+ 24h {{ uptime['24h'] if uptime['24h'] is not none else '—' }}{{ '%' if uptime['24h'] is not none }} + · 7d {{ uptime['7d'] if uptime['7d'] is not none else '—' }}{{ '%' if uptime['7d'] is not none }} + · 30d {{ uptime['30d'] if uptime['30d'] is not none else '—' }}{{ '%' if uptime['30d'] is not none }} +
+ {% else %} +
—
+ {% endif %} +
+
+
+ +{# ── Agent (host_agent plugin fragment, embedded across the plugin boundary) ── #} +
+
Loading agent…
+
+ +{# ── Ansible ──────────────────────────────────────────────────────────────── #} +
+

Ansible

+ {% if linked_target %} +
+
+ {{ linked_target.name }} + {{ linked_target.address }} + {% if linked_target.groups %} +
+ {% for grp in linked_target.groups %} + {{ grp.name }} + {% endfor %} +
+ {% endif %} +
+ Edit target +
+ + +
+
+ + {% if ansible_sources %} +
+ Run a playbook against this host +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ +
+
+ {% endif %} + {% else %} +

+ No Ansible target linked. A target gives this host an SSH connection so Steward can + run playbooks and provision the agent. +

+
+ {% if linkable_targets %} +
+ + + +
+ {% endif %} +
+ + +
+
+ {% endif %} +
+ +{% endblock %} diff --git a/steward/templates/hosts/list.html b/steward/templates/hosts/list.html index 1cc0815..5c3a2a8 100644 --- a/steward/templates/hosts/list.html +++ b/steward/templates/hosts/list.html @@ -31,7 +31,7 @@ {% set dns = latest_dns.get(host.id) %} {% set ut = uptime.get(host.id, {}) %} - {{ host.name }} + {{ host.name }} {{ host.address }} {{ host.probe_type.value | upper }}{% if host.probe_type.value == 'tcp' %}:{{ host.probe_port }}{% endif %} From f29255039d0317bee32d8f645cef91a6a1f31796 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 20:51:33 -0400 Subject: [PATCH 051/126] =?UTF-8?q?feat(hosts):=20Phase=202+3=20=E2=80=94?= =?UTF-8?q?=20agent=20column,=20fold=20fleet=20+=20management=20into=20Hos?= =?UTF-8?q?ts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Hosts list: new Agent column (latest CPU/mem read from the generic PluginMetric table — no host_agent import) + an admin "Agent fleet" button. - /plugins/host_agent/ (old fleet page) now redirects to /hosts/ (folded into the hub; kept as a redirect so widgets/links don't 404). - Agent management moved off the "settings" URL: the management page is now /plugins/host_agent/fleet/ ("Agent fleet" — bulk provision/install/update + registrations + curl install), reachable from the Hosts list. Old /plugins/host_agent/settings/ redirects there. Per-host management lives on the host detail page; this page is now explicitly the bulk/fleet view. Milestone 70 phases 2-3. Phase 4 (plugins capability/integration split + nav cleanup) next. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 22 +++++++---- .../host_agent/templates/settings_list.html | 15 ++++++-- steward/hosts/routes.py | 38 +++++++++++++++++++ steward/templates/hosts/list.html | 14 +++++++ 4 files changed, 77 insertions(+), 12 deletions(-) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 1863141..c15f177 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -375,10 +375,9 @@ async def _fleet_rows(session) -> list[dict]: @host_agent_bp.get("/") @require_role(UserRole.viewer) async def index(): - """Fleet overview page — cards per host linking to the detail view.""" - async with current_app.db_sessionmaker() as session: - rows = await _fleet_rows(session) - return await render_template("host_list.html", rows=rows) + """The fleet view folded into the Hosts hub — kept as a redirect so old + links/widgets don't 404.""" + return redirect("/hosts/") @host_agent_bp.get("/widget") @@ -571,7 +570,14 @@ async def host_panel(host_id: str): @host_agent_bp.get("/settings/") @require_role(UserRole.admin) -async def settings_list(): +async def settings_redirect(): + """Old management URL — agent management moved into the Hosts section.""" + return redirect(url_for("host_agent.fleet")) + + +@host_agent_bp.get("/fleet/") +@require_role(UserRole.admin) +async def fleet(): from steward.core.capabilities import has_capability from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup @@ -641,7 +647,7 @@ async def add_host(): await session.commit() host_id = host.id - return redirect(url_for("host_agent.settings_list") + f"?new_token={raw}&host_id={host_id}") + return redirect(url_for("host_agent.fleet") + f"?new_token={raw}&host_id={host_id}") @host_agent_bp.post("/settings//rotate-token") @@ -657,7 +663,7 @@ async def rotate_token(host_id: str): reg.token_hash = hashed reg.token_created_at = datetime.now(timezone.utc) await session.commit() - return redirect(url_for("host_agent.settings_list") + f"?new_token={raw}&host_id={host_id}") + return redirect(url_for("host_agent.fleet") + f"?new_token={raw}&host_id={host_id}") @host_agent_bp.post("/settings//delete") @@ -670,7 +676,7 @@ async def delete_registration(host_id: str): if reg is not None: await session.delete(reg) await session.commit() - return redirect(url_for("host_agent.settings_list")) + return redirect(url_for("host_agent.fleet")) # ── Deploy via Ansible (plugin↔core synergy via the capability registry) ────── diff --git a/plugins/host_agent/templates/settings_list.html b/plugins/host_agent/templates/settings_list.html index 7cbff8f..866cad4 100644 --- a/plugins/host_agent/templates/settings_list.html +++ b/plugins/host_agent/templates/settings_list.html @@ -1,9 +1,16 @@ {% extends "base.html" %} {% from "_macros.html" import crumbs %} -{% block title %}Host Agent — Steward{% endblock %} -{% block breadcrumb %}{{ crumbs([("Host Agents", "/plugins/host_agent/"), ("Settings", "")]) }}{% endblock %} +{% block title %}Agent fleet — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), ("Agent fleet", "")]) }}{% endblock %} {% block content %} -

Host Agent — Registered Hosts

+
+

Agent fleet

+ ← Hosts +
+

+ Bulk agent operations across your inventory. For a single host, use its + host page — provisioning and metrics live there. +

{% if new_token %}
@@ -58,7 +65,7 @@ No managed SSH key yet — Steward needs one to log into hosts as steward.
- +
diff --git a/steward/hosts/routes.py b/steward/hosts/routes.py index c9a903e..95e0ab2 100644 --- a/steward/hosts/routes.py +++ b/steward/hosts/routes.py @@ -68,6 +68,42 @@ async def _compute_uptime(db) -> dict[str, dict]: return stats +async def _agent_metrics_by_host(db, host_names: list[str]) -> dict[str, dict]: + """Latest agent cpu/mem per host name from the generic PluginMetric table. + + Read directly (PluginMetric is a core model) so the Hosts list can show an + at-a-glance agent column without importing the host_agent plugin. + """ + from steward.models.metrics import PluginMetric + if not host_names: + return {} + wanted = ("cpu_pct", "mem_used_pct") + subq = ( + select( + PluginMetric.resource_name, PluginMetric.metric_name, + func.max(PluginMetric.recorded_at).label("m"), + ) + .where( + PluginMetric.source_module == "host_agent", + PluginMetric.metric_name.in_(wanted), + PluginMetric.resource_name.in_(host_names), + ) + .group_by(PluginMetric.resource_name, PluginMetric.metric_name) + ).subquery() + rows = (await db.execute( + select(PluginMetric).join( + subq, + (PluginMetric.resource_name == subq.c.resource_name) + & (PluginMetric.metric_name == subq.c.metric_name) + & (PluginMetric.recorded_at == subq.c.m), + ) + )).scalars().all() + out: dict[str, dict] = {} + for r in rows: + out.setdefault(r.resource_name, {})[r.metric_name] = r.value + return out + + @hosts_bp.get("/") @require_role(UserRole.viewer) async def list_hosts(): @@ -105,6 +141,7 @@ async def list_hosts(): ) latest_dns = {r.host_id: r for r in dr.scalars()} uptime = await _compute_uptime(db) + agent_metrics = await _agent_metrics_by_host(db, [h.name for h in hosts]) return await render_template( "hosts/list.html", @@ -112,6 +149,7 @@ async def list_hosts(): latest_pings=latest_pings, latest_dns=latest_dns, uptime=uptime, + agent_metrics=agent_metrics, ) diff --git a/steward/templates/hosts/list.html b/steward/templates/hosts/list.html index 5c3a2a8..dde6693 100644 --- a/steward/templates/hosts/list.html +++ b/steward/templates/hosts/list.html @@ -5,6 +5,9 @@

Hosts

SLA + {% if session.user_role == 'admin' %} + Agent fleet + {% endif %} Add Host
@@ -22,6 +25,7 @@ 24h 7d 30d + Agent @@ -89,6 +93,16 @@ {% endif %} {% endfor %} + {% set am = agent_metrics.get(host.name) %} + + {% if am %} + + {{ '%.0f'|format(am.get('cpu_pct', 0)) }}% / {{ '%.0f'|format(am.get('mem_used_pct', 0)) }}% + + {% else %} + — + {% endif %} + Edit
From 7ef1af218429622c5ebb14db46fcdc6e67bfe10f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 20:57:10 -0400 Subject: [PATCH 052/126] =?UTF-8?q?feat(settings):=20Phase=204=20=E2=80=94?= =?UTF-8?q?=20capabilities=20vs=20integrations=20+=20nav=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final phase of the host-IA unification (milestone 70). - Settings → Plugins split into two tiers: "Monitoring capabilities" (host_agent, http, snmp, docker — built-in host facets, surfaced via Hosts/Status, on by default) and "Integrations" (traefik, unifi — external systems, off until configured). Presentation only: a CAPABILITY_PLUGINS set (overridable by plugin.yaml `kind:`) tags each plugin; module loading, optional deps, and migrations are untouched. - Drop the "default-enable a plugin" framing in the UI copy — capabilities are described as built-in, not optional add-ons. - Nav: remove the standalone "Uptime" item (folded into Hosts; still reachable via the SLA button on the Hosts list). Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/settings/routes.py | 12 +++ steward/templates/base.html | 1 - steward/templates/settings/plugins.html | 123 +++++++++++++----------- 3 files changed, 78 insertions(+), 58 deletions(-) diff --git a/steward/settings/routes.py b/steward/settings/routes.py index 64d623c..69e3c0e 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -505,6 +505,13 @@ def _build_plugin_cfg_from_form(plugin: dict, form) -> dict: # ── Plugins list ────────────────────────────────────────────────────────────── +# Bundled plugins that are really built-in *host/monitoring capabilities* (facets +# of a host), not discrete vendor integrations. Presentation-only split — they +# still load through the normal plugin mechanism. A plugin.yaml may set +# kind: capability|integration to override. +CAPABILITY_PLUGINS = {"host_agent", "http", "snmp", "docker"} + + @settings_bp.get("/plugins/") @require_role(UserRole.admin) async def plugins(): @@ -512,9 +519,14 @@ async def plugins(): settings = await get_all_settings(db) discovered = _discover_plugins() _merge_plugin_config(discovered, to_plugins_cfg(settings)) + for p in discovered: + p["_kind"] = p.get("kind") or ( + "capability" if p["_dir"] in CAPABILITY_PLUGINS else "integration") repos = _get_plugin_repos(settings) return await render_template( "settings/plugins.html", + capabilities=[p for p in discovered if p["_kind"] == "capability"], + integrations=[p for p in discovered if p["_kind"] != "capability"], discovered_plugins=discovered, repos=repos, settings=settings, diff --git a/steward/templates/base.html b/steward/templates/base.html index 2e81c39..49dff1f 100644 --- a/steward/templates/base.html +++ b/steward/templates/base.html @@ -209,7 +209,6 @@ textarea { resize: vertical; } Dashboard Status Hosts - Uptime Ping DNS Alerts diff --git a/steward/templates/settings/plugins.html b/steward/templates/settings/plugins.html index 60bd70d..1116d3d 100644 --- a/steward/templates/settings/plugins.html +++ b/steward/templates/settings/plugins.html @@ -10,72 +10,81 @@ {# ── Restart banner ────────────────────────────────────────────────────────── #}
-{# ── Installed Plugins ─────────────────────────────────────────────────────── #} -
-
Installed Plugins
+{% macro plugin_card(plugin) %} +{% set fail_reason = plugin_failures.get(plugin._dir) %} +
+ {% if fail_reason %} + + {% elif plugin._enabled %} + + {% else %} + + {% endif %} +
+
+ {{ plugin.get('name', plugin._dir) }} + v{{ plugin.get('version', '?') }} + {% if fail_reason %} + Error + {% elif plugin._enabled %} + Enabled + {% else %} + Disabled + {% endif %} +
+ {% if plugin.get('description') %} +
{{ plugin.description }}
+ {% endif %} + {% if fail_reason %} +
{{ fail_reason }}
+ {% endif %} +
+ Settings → +
+{% endmacro %} + +{# ── Monitoring capabilities (host facets) ─────────────────────────────────── #} +
+
Monitoring capabilities
- -{% if not discovered_plugins %} -
- No plugins installed. The table awaits new seats. +

+ Built-in ways to monitor your hosts (agent metrics, HTTP/uptime, SNMP, Docker). These are + facets of a host — you'll see their data in the Hosts and + Status sections, not as separate areas. On by default. +

+{% if capabilities %} +
+ {% for plugin in capabilities %}{{ plugin_card(plugin) }}{% endfor %}
{% else %} +
None installed.
+{% endif %} + +{# ── Integrations (external systems) ───────────────────────────────────────── #} +
Integrations
+

+ External systems Steward pulls data from (e.g. Traefik, UniFi). Enable and configure each one + to surface its own dashboard. Off until you set them up. +

+{% if integrations %}
- {% for plugin in discovered_plugins %} - {% set fail_reason = plugin_failures.get(plugin._dir) %} -
- - {# Status indicator #} - {% if fail_reason %} - - {% elif plugin._enabled %} - - {% else %} - - {% endif %} - - {# Name + version + description #} -
-
- {{ plugin.get('name', plugin._dir) }} - v{{ plugin.get('version', '?') }} - {% if fail_reason %} - Error - {% elif plugin._enabled %} - Enabled - {% else %} - Disabled - {% endif %} -
- {% if plugin.get('description') %} -
{{ plugin.description }}
- {% endif %} - {% if fail_reason %} -
{{ fail_reason }}
- {% endif %} -
- - {# Settings button #} - Settings → -
- {% endfor %} + {% for plugin in integrations %}{{ plugin_card(plugin) }}{% endfor %} +
+{% else %} +
+ No integrations installed. Browse the catalog below.
{% endif %} From 17c9c875e492956d3394aa5bdc4d983df06605b4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 21:20:11 -0400 Subject: [PATCH 053/126] refactor(hosts): remove legacy host_agent redirects/paths (no back-compat) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dev-only instance, no bookmarks — per family rule 22, fully remove old paths instead of shimming them. - Delete the /plugins/host_agent/ (index) and /plugins/host_agent/settings/ redirect routes; delete the now-dead host_list.html fleet template. - Move the remaining management POST routes off /settings/ to /fleet/ (add-host, rotate-token, delete) — single canonical prefix. - Repoint real callers to canonical URLs: dashboard widgets (host resources → /hosts/, history → /plugins/host_agent/fleet/), the full-metrics page breadcrumb + back link (→ the host hub), Settings→Ansible link, and the agent panel's curl-install link. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 21 ++------- plugins/host_agent/templates/host_detail.html | 4 +- plugins/host_agent/templates/host_list.html | 45 ------------------- plugins/host_agent/templates/panel.html | 6 +-- .../host_agent/templates/settings_list.html | 6 +-- steward/core/widgets.py | 4 +- steward/templates/settings/ansible.html | 2 +- 7 files changed, 14 insertions(+), 74 deletions(-) delete mode 100644 plugins/host_agent/templates/host_list.html diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index c15f177..ab88034 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -372,14 +372,6 @@ async def _fleet_rows(session) -> list[dict]: return rows -@host_agent_bp.get("/") -@require_role(UserRole.viewer) -async def index(): - """The fleet view folded into the Hosts hub — kept as a redirect so old - links/widgets don't 404.""" - return redirect("/hosts/") - - @host_agent_bp.get("/widget") async def widget_table(): """Fleet-glance dashboard widget: one row per monitored host, latest metrics.""" @@ -568,13 +560,6 @@ async def host_panel(host_id: str): ) -@host_agent_bp.get("/settings/") -@require_role(UserRole.admin) -async def settings_redirect(): - """Old management URL — agent management moved into the Hosts section.""" - return redirect(url_for("host_agent.fleet")) - - @host_agent_bp.get("/fleet/") @require_role(UserRole.admin) async def fleet(): @@ -618,7 +603,7 @@ async def fleet(): ) -@host_agent_bp.post("/settings/add-host") +@host_agent_bp.post("/fleet/add-host") @require_role(UserRole.admin) async def add_host(): form = await request.form @@ -650,7 +635,7 @@ async def add_host(): return redirect(url_for("host_agent.fleet") + f"?new_token={raw}&host_id={host_id}") -@host_agent_bp.post("/settings//rotate-token") +@host_agent_bp.post("/fleet//rotate-token") @require_role(UserRole.admin) async def rotate_token(host_id: str): async with current_app.db_sessionmaker() as session: @@ -666,7 +651,7 @@ async def rotate_token(host_id: str): return redirect(url_for("host_agent.fleet") + f"?new_token={raw}&host_id={host_id}") -@host_agent_bp.post("/settings//delete") +@host_agent_bp.post("/fleet//delete") @require_role(UserRole.admin) async def delete_registration(host_id: str): async with current_app.db_sessionmaker() as session: diff --git a/plugins/host_agent/templates/host_detail.html b/plugins/host_agent/templates/host_detail.html index 55a0cc3..c34fda9 100644 --- a/plugins/host_agent/templates/host_detail.html +++ b/plugins/host_agent/templates/host_detail.html @@ -1,7 +1,7 @@ {% extends "base.html" %} {% from "_macros.html" import crumbs %} {% block title %}{{ host.name }} — Host Agent — Steward{% endblock %} -{% block breadcrumb %}{{ crumbs([("Host Agents", "/plugins/host_agent/"), (host.name, "")]) }}{% endblock %} +{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), (host.name, "/hosts/" ~ host.id), ("Metrics", "")]) }}{% endblock %} {% macro fmt_bps(v) %} {%- if v is none -%}— @@ -27,7 +27,7 @@ {% block content %}
- ← Fleet + ← Host

{{ host.name }}

{% if stale %}stale{% else %}live{% endif %}
diff --git a/plugins/host_agent/templates/host_list.html b/plugins/host_agent/templates/host_list.html deleted file mode 100644 index 79d63a0..0000000 --- a/plugins/host_agent/templates/host_list.html +++ /dev/null @@ -1,45 +0,0 @@ -{% extends "base.html" %} -{% from "_macros.html" import crumbs %} -{% block title %}Host Agents — Steward{% endblock %} -{% block breadcrumb %}{{ crumbs([("Dashboard", "/"), ("Host Agents", "")]) }}{% endblock %} -{% block content %} -
-

Host Agents

- {% if session.user_role == 'admin' %} - Manage agents → - {% endif %} -
- -{% if not rows %} -
- No hosts are reporting agent metrics yet. - {% if session.user_role == 'admin' %} - Add one from Host Agent settings and run the install command on the target. - {% else %} - Ask an admin to install the agent on a host. - {% endif %} -
-{% else %} - -{% endif %} -{% endblock %} diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html index 6391d19..4e8cc5c 100644 --- a/plugins/host_agent/templates/panel.html +++ b/plugins/host_agent/templates/panel.html @@ -37,11 +37,11 @@ {% endif %} -
-
@@ -65,7 +65,7 @@ {% elif not ansible_available %}

Ansible isn't available, so the agent can't be deployed from here. Install it manually with the - curl install command. + curl install command.

{% elif not managed_key_set %} diff --git a/plugins/host_agent/templates/settings_list.html b/plugins/host_agent/templates/settings_list.html index 866cad4..80c027f 100644 --- a/plugins/host_agent/templates/settings_list.html +++ b/plugins/host_agent/templates/settings_list.html @@ -23,7 +23,7 @@ {% endif %}
-
@@ -155,12 +155,12 @@ {{ item.reg.distro or "—" }} {{ item.reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") if item.reg.last_seen_at else "never" }} - -
diff --git a/steward/core/widgets.py b/steward/core/widgets.py index 3b7740e..d637f1f 100644 --- a/steward/core/widgets.py +++ b/steward/core/widgets.py @@ -262,7 +262,7 @@ WIDGET_REGISTRY: dict[str, dict] = { "label": "Host Agent — Resources", "description": "Fleet view: CPU, memory, disk, and load per monitored host", "hx_url": "/plugins/host_agent/widget", - "detail_url": "/plugins/host_agent/", + "detail_url": "/hosts/", "plugin": "host_agent", "poll": True, "params": [], @@ -272,7 +272,7 @@ WIDGET_REGISTRY: dict[str, dict] = { "label": "Host Agent — History", "description": "CPU/memory/disk history for one host", "hx_url": "/plugins/host_agent/widget/history", - "detail_url": "/plugins/host_agent/settings/", + "detail_url": "/plugins/host_agent/fleet/", "plugin": "host_agent", "poll": True, "params": [ diff --git a/steward/templates/settings/ansible.html b/steward/templates/settings/ansible.html index 2bf1611..dd1021f 100644 --- a/steward/templates/settings/ansible.html +++ b/steward/templates/settings/ansible.html @@ -29,7 +29,7 @@ Steward's reusable Ansible identity. Provisioning installs the public key into each host's {{ ssh_user }} account; steady-state runs authenticate with the private half (stored encrypted). Generate it once, then provision hosts from - Host Agents. + Host Agents.

{% if ssh_public_key %}
From cfe6b4c25f70af5f926dd3a10cc43ff88cc47ca0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 22:53:13 -0400 Subject: [PATCH 054/126] =?UTF-8?q?fix(host=5Fagent):=20deploy/provision?= =?UTF-8?q?=20500=20=E2=80=94=20double-begin=20on=20session?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetch_scope_targets() runs a SELECT that autobegins the session transaction (SQLAlchemy 2.0), so the following `async with db.begin()` raised "A transaction is already begun on this Session" → 500 on both /plugins/host_agent/deploy and /provision. Mint registrations directly into the autobegun transaction, build the inventory while the ORM objects are still live, then await db.commit(). Escaped CI because unit tests use a mock db_sessionmaker; no integration test exercises these routes (follow-up noted in Scribe issue #884). Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index ab88034..0519873 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -731,17 +731,19 @@ async def deploy_via_ansible(): targets = await fetch_scope_targets(db, scope) if not targets: return _error(400, "no_targets", "No Ansible targets in that scope") + # The read above autobegins the session transaction (SQLAlchemy 2.0), so + # mint into it directly and commit — a nested db.begin() would double-begin. tokens: dict[str, str] = {} - async with db.begin(): - for t in targets: - host = await _ensure_host_for_target(db, t) - tokens[t.name] = await _mint_registration_token(db, host) - inv = generate_inventory(targets) + for t in targets: + host = await _ensure_host_for_target(db, t) + tokens[t.name] = await _mint_registration_token(db, host) + inv = generate_inventory(targets) # built while ORM objects are still live for name, tok in tokens.items(): hv = inv["_meta"]["hostvars"].setdefault(name, {}) hv["steward_url"] = url hv["steward_token"] = tok inventory_content = _json.dumps(inv) + await db.commit() actor_role = UserRole(session.get("user_role", "viewer")) run, _source, err = await invoke_capability( @@ -805,12 +807,13 @@ async def provision_via_ansible(): targets = await fetch_scope_targets(db, scope) if not targets: return _error(400, "no_targets", "No Ansible targets in that scope") + # The read above autobegins the session transaction (SQLAlchemy 2.0), so + # mint into it directly and commit — a nested db.begin() would double-begin. tokens: dict[str, str] = {} - async with db.begin(): - for t in targets: - host = await _ensure_host_for_target(db, t) - tokens[t.name] = await _mint_registration_token(db, host) - inv = generate_inventory(targets) + for t in targets: + host = await _ensure_host_for_target(db, t) + tokens[t.name] = await _mint_registration_token(db, host) + inv = generate_inventory(targets) # built while ORM objects are still live for name, tok in tokens.items(): hv = inv["_meta"]["hostvars"].setdefault(name, {}) hv["steward_url"] = url @@ -821,6 +824,7 @@ async def provision_via_ansible(): hv["steward_pubkey"] = pubkey hv["steward_user"] = steward_user inventory_content = _json.dumps(inv) + await db.commit() actor_role = UserRole(session.get("user_role", "viewer")) run, _source, err = await invoke_capability( From 9ce4cce5c5a3357174752bf8c172dd6e1cc6e204 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 08:05:47 -0400 Subject: [PATCH 055/126] feat(crypto): name the failing setting in wrong-key decrypt log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "could not decrypt a stored secret" warning was generic, so an operator couldn't tell which of the six secret settings was encrypted under an old key. Thread the setting key through _decode → decrypt_secret(context=...) so the log now reads e.g. "Could not decrypt stored secret smtp.password (wrong/rotated key — re-enter it)". Pure diagnostic; decrypt behaviour unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/core/crypto.py | 11 ++++++++--- steward/core/settings.py | 16 ++++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/steward/core/crypto.py b/steward/core/crypto.py index 44595a7..e677b9b 100644 --- a/steward/core/crypto.py +++ b/steward/core/crypto.py @@ -81,8 +81,12 @@ def encrypt_secret(plaintext: str) -> str: return ENC_PREFIX + f.encrypt(plaintext.encode("utf-8")).decode("ascii") -def decrypt_secret(value: str) -> str: - """Decrypt an ``enc:v1:`` token; pass through plaintext / undecryptable values.""" +def decrypt_secret(value: str, *, context: str = "") -> str: + """Decrypt an ``enc:v1:`` token; pass through plaintext / undecryptable values. + + context (e.g. the setting key) is only used to name the value in the + wrong-key log line, so an operator knows exactly which secret to re-enter. + """ if not is_encrypted(value): return value f = _get() @@ -91,7 +95,8 @@ def decrypt_secret(value: str) -> str: try: return f.decrypt(value[len(ENC_PREFIX):].encode("ascii")).decode("utf-8") except InvalidToken: - logger.error("Could not decrypt a stored secret (wrong/rotated key?)") + logger.error("Could not decrypt stored secret %s (wrong/rotated key — re-enter it)", + context or "(unknown setting)") return value diff --git a/steward/core/settings.py b/steward/core/settings.py index e08b03d..e636e2b 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -124,10 +124,14 @@ SECRET_KEYS: set[str] = { } -def _decode(value: Any) -> Any: - """Decrypt a stored value if it's an encrypted token; else pass through.""" +def _decode(value: Any, key: str = "") -> Any: + """Decrypt a stored value if it's an encrypted token; else pass through. + + key is passed through to the decrypt log so a wrong-key failure names the + exact setting that needs re-entering. + """ from steward.core.crypto import decrypt_secret, is_encrypted - return decrypt_secret(value) if is_encrypted(value) else value + return decrypt_secret(value, context=key) if is_encrypted(value) else value # ───────────────────────────────────────────────────────────────────────────── @@ -142,7 +146,7 @@ async def get_setting(session: AsyncSession, key: str) -> Any: row = result.scalar_one_or_none() if row is None: return DEFAULTS.get(key) - return _decode(json.loads(row.value_json)) + return _decode(json.loads(row.value_json), key) async def set_setting(session: AsyncSession, key: str, value: Any) -> None: @@ -166,7 +170,7 @@ async def set_setting(session: AsyncSession, key: str, value: Any) -> None: async def get_all_settings(session: AsyncSession) -> dict[str, Any]: """Return flat key→value dict with defaults filled in for missing keys.""" result = await session.execute(select(AppSetting)) - stored = {row.key: _decode(json.loads(row.value_json)) for row in result.scalars()} + stored = {row.key: _decode(json.loads(row.value_json), row.key) for row in result.scalars()} out: dict[str, Any] = {} for key, default in DEFAULTS.items(): out[key] = stored.get(key, default) @@ -247,7 +251,7 @@ def load_settings_sync(db_url: str) -> dict[str, Any]: try: async with factory() as session: result = await session.execute(select(AppSetting)) - return {row.key: _decode(json.loads(row.value_json)) for row in result.scalars()} + return {row.key: _decode(json.loads(row.value_json), row.key) for row in result.scalars()} finally: await engine.dispose() From a92d1995d5fdc55f2040c0f0b79f5d3c0994af0c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 09:28:31 -0400 Subject: [PATCH 056/126] fix(ansible): write DB inventory as static YAML, not dynamic --list JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit generate_inventory() emits Ansible's dynamic --list shape (all.hosts is a LIST, vars under _meta) — valid only as an executable inventory script's stdout. We were writing it to a static file and passing -i, so Ansible's yaml plugin rejected it ("Invalid 'hosts' entry for 'all' group, requires a dictionary, found ...list...") and fell back to implicit localhost → "no hosts matched". Affected every steward:* scope run; surfaced on the first real provision. - New inventory_to_yaml(inv): convert the --list dict → a valid static YAML inventory (all.hosts dict keyed by host, groups under all.children, group vars preserved, injected per-host vars like steward_token retained). - Wire it into runner.trigger_run, host_agent deploy + provision. - executor writes the file as inventory.yml so the yaml plugin's extension check reliably claims it. - generate_inventory unchanged (still the --list dict); conversion happens at write time. Unit tests added. Scribe issue #885. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 12 +++++----- steward/ansible/executor.py | 4 +++- steward/ansible/inventory_gen.py | 27 ++++++++++++++++++++++ steward/ansible/runner.py | 7 +++--- tests/core/test_inventory_gen.py | 39 +++++++++++++++++++++++++++++++- 5 files changed, 78 insertions(+), 11 deletions(-) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 0519873..0eeec1f 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -707,9 +707,9 @@ async def deploy_via_ansible(): Uses the core "ansible.run_playbook" capability (no hard import of the runner) and injects a freshly-minted per-host token as an inventory hostvar. """ - import json as _json from steward.core.capabilities import has_capability, invoke_capability - from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory + from steward.ansible.inventory_gen import ( + fetch_scope_targets, generate_inventory, inventory_to_yaml) from steward.ansible.sources import BUILTIN_SOURCE_NAME if not has_capability("ansible.run_playbook"): @@ -742,7 +742,7 @@ async def deploy_via_ansible(): hv = inv["_meta"]["hostvars"].setdefault(name, {}) hv["steward_url"] = url hv["steward_token"] = tok - inventory_content = _json.dumps(inv) + inventory_content = inventory_to_yaml(inv) await db.commit() actor_role = UserRole(session.get("user_role", "viewer")) @@ -771,9 +771,9 @@ async def provision_via_ansible(): The bootstrap password is passed to the runner as a connection override and is NOT persisted on the run row. """ - import json as _json from steward.core.capabilities import has_capability, invoke_capability - from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory + from steward.ansible.inventory_gen import ( + fetch_scope_targets, generate_inventory, inventory_to_yaml) from steward.ansible.sources import BUILTIN_SOURCE_NAME if not has_capability("ansible.run_playbook"): @@ -823,7 +823,7 @@ async def provision_via_ansible(): # Ansible would shlex-split on whitespace. hv["steward_pubkey"] = pubkey hv["steward_user"] = steward_user - inventory_content = _json.dumps(inv) + inventory_content = inventory_to_yaml(inv) await db.commit() actor_role = UserRole(session.get("user_role", "viewer")) diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py index b70d694..dcc2038 100644 --- a/steward/ansible/executor.py +++ b/steward/ansible/executor.py @@ -375,7 +375,9 @@ async def start_run( os.chmod(tmpdir, 0o700) try: if inventory_content: - inv_target = os.path.join(tmpdir, "inventory") + # .yml so Ansible's yaml inventory plugin reliably claims it + # (the plugin's verify_file checks the extension). + inv_target = os.path.join(tmpdir, "inventory.yml") with open(inv_target, "w", encoding="utf-8") as invf: invf.write(inventory_content) else: diff --git a/steward/ansible/inventory_gen.py b/steward/ansible/inventory_gen.py index d6ab7ff..2938625 100644 --- a/steward/ansible/inventory_gen.py +++ b/steward/ansible/inventory_gen.py @@ -37,6 +37,33 @@ def generate_inventory(targets: list) -> dict: return inv +def inventory_to_yaml(inv: dict) -> str: + """Convert the ``--list`` dict from generate_inventory into a STATIC YAML + inventory string. + + The ``--list`` shape (``all.hosts`` is a *list*, vars under ``_meta``) is only + valid as the stdout of an executable dynamic-inventory script. When written to + a plain file and passed via ``-i``, Ansible's yaml plugin requires + ``all.hosts`` to be a *dict* keyed by hostname — otherwise it fails to parse + and silently falls back to implicit localhost. This emits that valid form. + """ + import yaml + hostvars = (inv.get("_meta") or {}).get("hostvars") or {} + all_hosts = (inv.get("all") or {}).get("hosts") or [] + root: dict = {"hosts": {h: (hostvars.get(h) or {}) for h in all_hosts}} + children: dict = {} + for key, val in inv.items(): + if key in ("all", "_meta") or not isinstance(val, dict): + continue + grp: dict = {"hosts": {h: {} for h in (val.get("hosts") or [])}} + if val.get("vars"): + grp["vars"] = val["vars"] + children[key] = grp + if children: + root["children"] = children + return yaml.safe_dump({"all": root}, default_flow_style=False, sort_keys=True) + + async def fetch_scope_targets(db, scope: str) -> list: """Query AnsibleTarget objects for a given inventory scope string. diff --git a/steward/ansible/runner.py b/steward/ansible/runner.py index 27dfaf7..7d751d9 100644 --- a/steward/ansible/runner.py +++ b/steward/ansible/runner.py @@ -7,12 +7,13 @@ manual, alert-triggered, and scheduled runs all take the exact same path. from __future__ import annotations import asyncio -import json import uuid from datetime import datetime, timezone from steward.ansible import executor, sources as src_module -from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory +from steward.ansible.inventory_gen import ( + fetch_scope_targets, generate_inventory, inventory_to_yaml, +) from steward.models.ansible import AnsibleRun, AnsibleRunStatus @@ -53,7 +54,7 @@ async def trigger_run( elif inventory_scope.startswith("steward:"): async with app.db_sessionmaker() as db: targets = await fetch_scope_targets(db, inventory_scope) - inventory_content = json.dumps(generate_inventory(targets)) + inventory_content = inventory_to_yaml(generate_inventory(targets)) elif inventory_scope.startswith("repo:"): parts = inventory_scope.split(":", 2) inventory_path = parts[2] if len(parts) == 3 else "" diff --git a/tests/core/test_inventory_gen.py b/tests/core/test_inventory_gen.py index 5952e3a..b0b1811 100644 --- a/tests/core/test_inventory_gen.py +++ b/tests/core/test_inventory_gen.py @@ -3,7 +3,10 @@ Uses SimpleNamespace duck-typed objects so no DB or SQLAlchemy is required. """ from types import SimpleNamespace -from steward.ansible.inventory_gen import generate_inventory + +import yaml + +from steward.ansible.inventory_gen import generate_inventory, inventory_to_yaml def _group(name, ansible_vars=None): @@ -91,3 +94,37 @@ def test_target_in_multiple_groups(): assert "web-prod-01" in result["prod"]["hosts"] assert result["_meta"]["hostvars"]["web-prod-01"]["role"] == "web" assert result["_meta"]["hostvars"]["web-prod-01"]["env"] == "production" + + +# ── inventory_to_yaml: STATIC inventory (hosts is a dict, not a list) ────────── + +def test_yaml_hosts_is_dict_with_hostvars(): + target = _target("web1", "192.168.1.10", {"ansible_user": "ubuntu"}) + parsed = yaml.safe_load(inventory_to_yaml(generate_inventory([target]))) + # The bug was a LIST here; a static YAML inventory requires a dict keyed by host. + assert isinstance(parsed["all"]["hosts"], dict) + assert parsed["all"]["hosts"]["web1"]["ansible_host"] == "192.168.1.10" + assert parsed["all"]["hosts"]["web1"]["ansible_user"] == "ubuntu" + + +def test_yaml_groups_under_children(): + grp = _group("db", {"db_port": 5432}) + t1 = _target("db1", "10.0.0.1", groups=[grp]) + t2 = _target("db2", "10.0.0.2", groups=[grp]) + parsed = yaml.safe_load(inventory_to_yaml(generate_inventory([t1, t2]))) + assert set(parsed["all"]["hosts"]) == {"db1", "db2"} + assert set(parsed["all"]["children"]["db"]["hosts"]) == {"db1", "db2"} + assert parsed["all"]["children"]["db"]["vars"]["db_port"] == 5432 + + +def test_yaml_injected_hostvars_survive(): + """Extra per-host vars injected after generate_inventory (e.g. steward_token).""" + inv = generate_inventory([_target("h1", "1.2.3.4")]) + inv["_meta"]["hostvars"]["h1"]["steward_token"] = "tok with space" + parsed = yaml.safe_load(inventory_to_yaml(inv)) + assert parsed["all"]["hosts"]["h1"]["steward_token"] == "tok with space" + + +def test_yaml_empty_targets(): + parsed = yaml.safe_load(inventory_to_yaml(generate_inventory([]))) + assert parsed == {"all": {"hosts": {}}} From bb90411f00d30109d08a62f9d0a7fb3266d97635 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 09:50:11 -0400 Subject: [PATCH 057/126] fix(host_agent): gate "deployed" on agent check-in; fail no-op runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed provision looked successful in two ways: 1. The host panel showed the agent as deployed (metrics + Update/Rotate/Remove) because provision/deploy mint the registration row BEFORE the playbook runs and the panel keyed "installed" on that row. Now gated on the agent actually checking in (reg.last_seen_at). Three states: reporting (metrics + lifecycle), pending (token minted but no check-in → "no metrics yet, deploy may be running/failed" banner + retry + Clear pending registration), and none (install path). 2. The run reported success though nothing ran — ansible-playbook exits 0 on "no hosts matched"/empty inventory. The executor now treats an empty PLAY RECAP (returncode 0 but no hosts executed) as failed, with a clear failure note. Non-zero exits and recap failed/unreachable were already caught. Scribe issue #887. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 9 +++++--- plugins/host_agent/templates/panel.html | 30 ++++++++++++++++++++----- steward/ansible/executor.py | 13 ++++++++++- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 0eeec1f..e25c1da 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -548,13 +548,16 @@ async def host_panel(host_id: str): hostlvl = latest.get(host.name, {}) ls = reg.last_seen_at if reg else None - stale = bool(reg) and ( - ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()) + # "Deployed" is gated on the agent actually checking in (last_seen_at), NOT on + # the registration row — that row is minted before the playbook runs, so a + # failed provision would otherwise look deployed. + reporting = bool(reg) and ls is not None + stale = reporting and (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds() ansible_cfg = current_app.config.get("ANSIBLE", {}) return await render_template( "panel.html", - host=host, reg=reg, hostlvl=hostlvl, stale=stale, target=target, + host=host, reg=reg, hostlvl=hostlvl, reporting=reporting, stale=stale, target=target, ansible_available=has_capability("ansible.run_playbook"), managed_key_set=bool((ansible_cfg.get("ssh_public_key") or "").strip()), ) diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html index 4e8cc5c..5279945 100644 --- a/plugins/host_agent/templates/panel.html +++ b/plugins/host_agent/templates/panel.html @@ -3,16 +3,18 @@

Agent

- {% if reg %} + {% if reporting %} {{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %} - {% if reg.last_seen_at %} · last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }}{% endif %} + · last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }} + {% elif reg %} + pending — no check-in yet {% endif %}
- {% if reg %} - {# ── Installed: at-a-glance metrics + lifecycle ── #} + {% if reporting %} + {# ── Reporting: at-a-glance metrics + lifecycle ── #}
{% set cpu = hostlvl.get('cpu_pct') %} {% set mem = hostlvl.get('mem_used_pct') %} @@ -54,10 +56,21 @@ {% endif %} {% else %} - {# ── Not installed: provisioning path ── #} + {# ── Not reporting: pending (token minted, no check-in) or never installed ── #} + {% if reg %} +
+ A token was minted for this host but no metrics have arrived yet — the + deploy may still be running, or it failed. Open the latest + Ansible run to check, then retry below. Live metrics appear here + once the agent checks in. +
+ {% else %}

No agent installed. The agent reports CPU, memory, disk, network and more back to Steward.

+ {% endif %} {% if session.user_role != 'admin' %}

An admin can install the agent here.

@@ -113,5 +126,12 @@ {% endif %} + + {% if reg and session.user_role == 'admin' %} +
+ +
+ {% endif %} {% endif %}
diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py index dcc2038..3f51975 100644 --- a/steward/ansible/executor.py +++ b/steward/ansible/executor.py @@ -442,10 +442,21 @@ async def start_run( await _flush() await proc.wait() + recap = parse_recap(_run_lines.get(run_id, [])) if run_id in _cancelled: final_status = "cancelled" + elif proc.returncode != 0: + final_status = "failed" + elif not recap: + # ansible-playbook exits 0 on "no hosts matched" / empty inventory + # — nothing actually ran, so don't report it as a success. + final_status = "failed" + if len(failures) < _FAILURE_CAP: + failures.append( + "No hosts matched — nothing executed " + "(check the host's Ansible target and inventory).") else: - final_status = "success" if proc.returncode == 0 else "failed" + final_status = "success" finally: shutil.rmtree(tmpdir, ignore_errors=True) From ad726e65f32172930cac2b4b3f4ce7831cec1c78 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 10:02:15 -0400 Subject: [PATCH 058/126] feat(ansible): dropdown playbook selection + auto-populated variable fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop making operators type playbook paths and guess extra-var names. - Reusable infra: shared ansible/_playbook_vars.html (the discovered-variable fields) + ansible/_playbook_options.html; two HTMX endpoints — /ansible/playbook-options (a source's playbooks, optional ?selected for edit) and /ansible/playbook-vars (a playbook's vars:/vars_prompt: as fill-in fields). browse _run_form.html refactored to include the shared partial. - Host "Run a playbook against this host": source dropdown → playbook dropdown → variable fields, all chained via HTMX. Handler reuses _parse_run_params so var__/secret__ fields flow through extra_vars_map + the unpersisted secret_vars channel. - Schedules: playbook free-text+datalist → source-dependent dropdown; fixed the extra-vars edit pre-fill to read extra_vars_map (stale list key after the earlier JSON-extra-vars change). Scribe #895. Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/ansible/routes.py | 29 ++++++++++++++ steward/hosts/routes.py | 26 ++++--------- .../templates/ansible/_playbook_options.html | 6 +++ steward/templates/ansible/_playbook_vars.html | 32 +++++++++++++++ steward/templates/ansible/_run_form.html | 31 +-------------- steward/templates/ansible/schedules.html | 20 ++++++---- steward/templates/hosts/detail.html | 39 +++++++++++++------ 7 files changed, 114 insertions(+), 69 deletions(-) create mode 100644 steward/templates/ansible/_playbook_options.html create mode 100644 steward/templates/ansible/_playbook_vars.html diff --git a/steward/ansible/routes.py b/steward/ansible/routes.py index ccf5fce..d88dca8 100644 --- a/steward/ansible/routes.py +++ b/steward/ansible/routes.py @@ -143,6 +143,35 @@ def _parse_run_params(form) -> tuple[dict | None, dict | None, str | None]: return (params or None), (secret_vars or None), None +@ansible_bp.get("/playbook-options") +@require_role(UserRole.operator) +async def playbook_options(): + """HTMX fragment:
+{% endfor %} +{% else %} +

+ No declared variables found in this playbook. +

+{% endif %} diff --git a/steward/templates/ansible/_run_form.html b/steward/templates/ansible/_run_form.html index 088f0f1..d706683 100644 --- a/steward/templates/ansible/_run_form.html +++ b/steward/templates/ansible/_run_form.html @@ -34,36 +34,7 @@
- {% if variables %} -
- Playbook variables -
-

- Leave a field blank to use the playbook/inventory default. Secret fields are - never stored. -

- {% for v in variables %} -
- - {% if v.secret %}{% endif %} - -
- {% endfor %} - {% else %} -

- No declared variables found in this playbook. -

- {% endif %} + {% include "ansible/_playbook_vars.html" %}
Advanced diff --git a/steward/templates/ansible/schedules.html b/steward/templates/ansible/schedules.html index 0fe51ee..f9c9d9e 100644 --- a/steward/templates/ansible/schedules.html +++ b/steward/templates/ansible/schedules.html @@ -86,7 +86,9 @@
- {% for sd in source_data %} @@ -94,12 +96,13 @@
- - - - {% for pb in all_playbooks %}{% endfor %} - + +
@@ -132,7 +135,8 @@
- +
diff --git a/steward/templates/hosts/detail.html b/steward/templates/hosts/detail.html index 81217c5..50eddd9 100644 --- a/steward/templates/hosts/detail.html +++ b/steward/templates/hosts/detail.html @@ -109,23 +109,38 @@
- + {% for s in ansible_sources %}{% endfor %}
- - -
-
- - -
-
- + +
+
+
+ Advanced +
+ + +
+
+ + +
+
+ +
+
From 7e6e63521b939330d2c3a50e1164dd4d453719ad Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 10:11:30 -0400 Subject: [PATCH 059/126] fix(host_agent): provision/deploy vars shadowed by play-var precedence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assertion failed ("Pass steward_url/token/pubkey") because those were injected as inventory HOST vars, but the playbooks declared them in the play `vars:` block — and play vars OUTRANK inventory host vars, so the empty defaults won and the injected values never reached the play. - Pass globals (steward_url, steward_pubkey, steward_user, agent_interval) as extra-vars via the JSON -e @file (highest precedence, space-safe). Keep only the per-host steward_token as an inventory host var. - provision.yml / install.yml: drop steward_token from `vars:` so the host var isn't shadowed; assertions use `| default('')` for the un-defaulted token. This was the next layer under the inventory-format fix — first the inventory wouldn't parse, now the injected vars actually apply. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 28 ++++++++++--------- .../ansible/bundled/host_agent/install.yml | 9 +++--- .../ansible/bundled/host_agent/provision.yml | 13 +++++---- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index e25c1da..a377f71 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -741,10 +741,9 @@ async def deploy_via_ansible(): host = await _ensure_host_for_target(db, t) tokens[t.name] = await _mint_registration_token(db, host) inv = generate_inventory(targets) # built while ORM objects are still live + # Per-host token as a host var; steward_url goes in as an extra-var below. for name, tok in tokens.items(): - hv = inv["_meta"]["hostvars"].setdefault(name, {}) - hv["steward_url"] = url - hv["steward_token"] = tok + inv["_meta"]["hostvars"].setdefault(name, {})["steward_token"] = tok inventory_content = inventory_to_yaml(inv) await db.commit() @@ -756,7 +755,8 @@ async def deploy_via_ansible(): playbook_path="host_agent/install.yml", inventory_content=inventory_content, inventory_scope=scope, - params={"extra_vars": [f"agent_interval={interval}"]}, + # extra-vars outrank the playbook's `vars:` defaults; token stays per-host. + params={"extra_vars_map": {"steward_url": url, "agent_interval": interval}}, triggered_by=session.get("user_id"), ) if err: @@ -817,15 +817,10 @@ async def provision_via_ansible(): host = await _ensure_host_for_target(db, t) tokens[t.name] = await _mint_registration_token(db, host) inv = generate_inventory(targets) # built while ORM objects are still live + # Only the per-host token is a host var; the globals go in as extra-vars + # below (extra-vars outrank play vars, host vars do not). for name, tok in tokens.items(): - hv = inv["_meta"]["hostvars"].setdefault(name, {}) - hv["steward_url"] = url - hv["steward_token"] = tok - # Pubkey carries a space (the comment) and steward_user is free text — - # injected as JSON hostvars (space-safe), NOT -e k=v strings which - # Ansible would shlex-split on whitespace. - hv["steward_pubkey"] = pubkey - hv["steward_user"] = steward_user + inv["_meta"]["hostvars"].setdefault(name, {})["steward_token"] = tok inventory_content = inventory_to_yaml(inv) await db.commit() @@ -837,7 +832,14 @@ async def provision_via_ansible(): playbook_path="host_agent/provision.yml", inventory_content=inventory_content, inventory_scope=scope, - params={"extra_vars": [f"agent_interval={interval}"]}, + # extra_vars_map → a JSON -e @file: highest precedence (beats play vars) + # and space-safe (steward_pubkey carries a comment with a space). + params={"extra_vars_map": { + "steward_url": url, + "steward_pubkey": pubkey, + "steward_user": steward_user, + "agent_interval": interval, + }}, triggered_by=session.get("user_id"), connection={"user": boot_user, "password": boot_password}, ) diff --git a/steward/ansible/bundled/host_agent/install.yml b/steward/ansible/bundled/host_agent/install.yml index bb39dd4..ce4acdd 100644 --- a/steward/ansible/bundled/host_agent/install.yml +++ b/steward/ansible/bundled/host_agent/install.yml @@ -7,17 +7,18 @@ hosts: all become: true gather_facts: false + # steward_token is a per-host inventory var — not declared here, or a play var + # would outrank and shadow it. steward_url arrives as an extra-var. vars: steward_url: "" - steward_token: "" agent_interval: 30 tasks: - name: Require steward_url and steward_token ansible.builtin.assert: that: - - steward_url | length > 0 - - steward_token | length > 0 - fail_msg: "Pass steward_url and steward_token as extra-vars." + - steward_url | default('') | length > 0 + - steward_token | default('') | length > 0 + fail_msg: "Pass steward_url and steward_token (Steward injects these)." - name: Create steward-agent system user ansible.builtin.user: diff --git a/steward/ansible/bundled/host_agent/provision.yml b/steward/ansible/bundled/host_agent/provision.yml index 73be606..49883eb 100644 --- a/steward/ansible/bundled/host_agent/provision.yml +++ b/steward/ansible/bundled/host_agent/provision.yml @@ -19,9 +19,12 @@ hosts: all become: true gather_facts: false + # steward_token is injected as a per-host inventory var (it differs per host), + # so it must NOT be declared here — a play var would outrank the inventory var + # and shadow it. The globals below arrive as extra-vars (highest precedence), + # which override these defaults. vars: steward_url: "" - steward_token: "" steward_pubkey: "" steward_user: steward agent_interval: 30 @@ -29,10 +32,10 @@ - name: Require provisioning vars ansible.builtin.assert: that: - - steward_url | length > 0 - - steward_token | length > 0 - - steward_pubkey | length > 0 - fail_msg: "Pass steward_url, steward_token and steward_pubkey as extra-vars." + - steward_url | default('') | length > 0 + - steward_token | default('') | length > 0 + - steward_pubkey | default('') | length > 0 + fail_msg: "Pass steward_url, steward_token and steward_pubkey (Steward injects these)." # ── Managed login account ──────────────────────────────────────────────── - name: Create the steward management account From f80f6c87e829b9e60040a9ea8956d618ab3a8d46 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 10:15:28 -0400 Subject: [PATCH 060/126] feat(auth): capture Steward URL during first-run admin setup (OOBE) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fresh install had no prompt for general.public_base_url, so first-run agent installs/share links silently used the request Host header. Add a "Steward URL" field to the first-run /setup page (create-admin), pre-filled with the current address, with help text. setup_post saves general.public_base_url and applies it to app.config immediately (no restart). Editable later in Settings → General. Scribe #896. Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/auth/routes.py | 12 +++++++++++- steward/templates/auth/setup.html | 11 +++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/steward/auth/routes.py b/steward/auth/routes.py index e646c6b..f1cd510 100644 --- a/steward/auth/routes.py +++ b/steward/auth/routes.py @@ -184,7 +184,10 @@ async def setup(): from quart import current_app if await get_user_count(current_app) > 0: return redirect(url_for("auth.login")) - return await render_template("auth/setup.html") + # Pre-fill the public URL with however the operator reached this page — the + # common case is correct, and it's what agent installs / share links use. + return await render_template( + "auth/setup.html", default_url=request.host_url.rstrip("/")) @auth_bp.post("/setup") @@ -201,9 +204,16 @@ async def setup_post(): return await render_template("auth/setup.html", error="All fields required"), 400 pw_hash = bcrypt.hashpw(password, bcrypt.gensalt()).decode() user = User(username=username, email=email, password_hash=pw_hash, role=UserRole.admin) + steward_url = (form.get("public_base_url", "") or "").strip().rstrip("/") async with current_app.db_sessionmaker() as db: async with db.begin(): db.add(user) + if steward_url: + from steward.core.settings import set_setting + await set_setting(db, "general.public_base_url", steward_url) + if steward_url: + # Apply immediately so agent installs / share links use it without a restart. + current_app.config["PUBLIC_BASE_URL"] = steward_url login_user(user) await log_audit(current_app, user.id, user.username, "auth.setup", detail={"username": username}) diff --git a/steward/templates/auth/setup.html b/steward/templates/auth/setup.html index 48a076a..00c3db7 100644 --- a/steward/templates/auth/setup.html +++ b/steward/templates/auth/setup.html @@ -18,6 +18,17 @@
+
+ + + + Used for agent installs, share links, and alert links. Pre-filled from your current + address — change it if hosts reach Steward by a different name. Editable later in Settings → General. + +
From 36212dc58b5cf5037a3cecf2ac32fd18522d7bc6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 11:05:43 -0400 Subject: [PATCH 061/126] feat(host_agent): at-a-glance disk = root (/) not "worst"; metric tooltips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "Disk (worst)" number was opaque at a glance. Show the root filesystem (/) usage instead — what people actually care about — on the host panel and the dashboard widget. "Worst" is kept only for the widget's health dot (so a full /var or /data still warns) and on the full-metrics page (per-mount + worst trend). Added hover tooltips defining CPU / Memory / Disk / Load. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 12 ++++++++++-- plugins/host_agent/templates/panel.html | 15 +++++++++------ plugins/host_agent/templates/widget_table.html | 4 ++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index a377f71..a52167e 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -341,7 +341,10 @@ async def _fleet_rows(session) -> list[dict]: )).scalars().all() latest: dict[str, dict[str, float]] = {} + root_disk: dict[str, float] = {} # host_name → root (/) disk % for row in latest_rows: + if row.resource_name.endswith(":/") and row.metric_name == "disk_used_pct": + root_disk[row.resource_name[:-2]] = row.value if ":" in row.resource_name: continue latest.setdefault(row.resource_name, {})[row.metric_name] = row.value @@ -362,7 +365,8 @@ async def _fleet_rows(session) -> list[dict]: "stale": stale, "cpu_pct": m.get("cpu_pct"), "mem_used_pct": m.get("mem_used_pct"), - "disk_worst": m.get("disk_used_pct_worst"), + "disk_worst": m.get("disk_used_pct_worst"), # health dot only + "disk_root": root_disk.get(host.name), # displayed at-a-glance "load_1m": m.get("load_1m"), "temp_max": m.get("temp_c_max"), "net_rx_bps": m.get("net_rx_bps"), @@ -547,6 +551,9 @@ async def host_panel(host_id: str): AnsibleTarget.host_id == host_id))).scalar_one_or_none() hostlvl = latest.get(host.name, {}) + # At-a-glance disk = the root filesystem (what people actually care about), + # not the "worst" mount which is opaque. Worst stays on the full-metrics page. + disk_root = (latest.get(host.name + ":/", {}) or {}).get("disk_used_pct") ls = reg.last_seen_at if reg else None # "Deployed" is gated on the agent actually checking in (last_seen_at), NOT on # the registration row — that row is minted before the playbook runs, so a @@ -557,7 +564,8 @@ async def host_panel(host_id: str): ansible_cfg = current_app.config.get("ANSIBLE", {}) return await render_template( "panel.html", - host=host, reg=reg, hostlvl=hostlvl, reporting=reporting, stale=stale, target=target, + host=host, reg=reg, hostlvl=hostlvl, disk_root=disk_root, + reporting=reporting, stale=stale, target=target, ansible_available=has_capability("ansible.run_playbook"), managed_key_set=bool((ansible_cfg.get("ssh_public_key") or "").strip()), ) diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html index 5279945..0c7d866 100644 --- a/plugins/host_agent/templates/panel.html +++ b/plugins/host_agent/templates/panel.html @@ -18,15 +18,18 @@
{% set cpu = hostlvl.get('cpu_pct') %} {% set mem = hostlvl.get('mem_used_pct') %} - {% set disk = hostlvl.get('disk_used_pct_worst') %} {% set load1 = hostlvl.get('load_1m') %} -
CPU
+
+
CPU
{{ '%.0f%%'|format(cpu) if cpu is not none else '—' }}
-
Memory
+
+
Memory
{{ '%.0f%%'|format(mem) if mem is not none else '—' }}
-
Disk (worst)
-
{{ '%.0f%%'|format(disk) if disk is not none else '—' }}
-
Load 1m
+
+
Disk /
+
{{ '%.0f%%'|format(disk_root) if disk_root is not none else '—' }}
+
+
Load 1m
{{ '%.2f'|format(load1) if load1 is not none else '—' }}
diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html index c391b0c..1250740 100644 --- a/plugins/host_agent/templates/widget_table.html +++ b/plugins/host_agent/templates/widget_table.html @@ -16,8 +16,8 @@ {% if r.mem_used_pct is not none %} mem {{ "%.0f"|format(r.mem_used_pct) }}% {% endif %} - {% if r.disk_worst is not none %} - disk {{ "%.0f"|format(r.disk_worst) }}% + {% if r.disk_root is not none %} + disk / {{ "%.0f"|format(r.disk_root) }}% {% endif %} {% if r.load_1m is not none %} load {{ "%.2f"|format(r.load_1m) }} From a0d1c5f07c46e8db08e0c34bc6cf7509e8f15efa Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 11:10:46 -0400 Subject: [PATCH 062/126] feat(host_agent): sparklines + load/core + PSI on the host panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Each at-a-glance metric (CPU, Memory, Disk /, Load) now shows a 6h sparkline (reused core.status.sparkline_svg + a recent-series query; disk uses the root mount sub-resource) so trend/consistency is visible, not just the instant. - Load is now normalized: "Load /core" = 1m load ÷ CPU cores as % (100% = run queue matches capacity), comparable across different hardware. Cores derived from the per-core CPU metrics already collected — no agent change. Raw load in the tooltip. - Added a "Pressure 10s" line: PSI cpu/mem/io (some, avg10), the hardware-independent saturation signal already collected by the agent. Scribe #898. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 40 +++++++++++++++++++++++- plugins/host_agent/templates/panel.html | 41 ++++++++++++++++--------- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index a52167e..199c552 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -11,7 +11,7 @@ import secrets from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for, session from steward.auth.middleware import require_role from steward.models.users import UserRole -from sqlalchemy import select, func, or_ +from sqlalchemy import select, func, or_, and_ from datetime import timedelta from steward.core.settings import public_base_url @@ -550,10 +550,47 @@ async def host_panel(host_id: str): target = (await db.execute(select(AnsibleTarget).where( AnsibleTarget.host_id == host_id))).scalar_one_or_none() + # Recent series for the at-a-glance sparklines (cpu/mem/load host-level, + # disk from the root mount sub-resource). + series_raw: dict[str, list[float]] = {"cpu": [], "mem": [], "disk": [], "load": []} + if reg: + since = datetime.now(timezone.utc) - timedelta(hours=6) + spark_rows = (await db.execute( + select(PluginMetric).where( + PluginMetric.source_module == SOURCE_MODULE, + PluginMetric.recorded_at >= since, + or_( + and_(PluginMetric.resource_name == host.name, + PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))), + and_(PluginMetric.resource_name == host.name + ":/", + PluginMetric.metric_name == "disk_used_pct"), + ), + ).order_by(PluginMetric.recorded_at) + )).scalars().all() + for r in spark_rows: + if r.resource_name == host.name: + key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name) + if key: + series_raw[key].append(r.value) + else: + series_raw["disk"].append(r.value) + + from steward.core.status import sparkline_svg hostlvl = latest.get(host.name, {}) # At-a-glance disk = the root filesystem (what people actually care about), # not the "worst" mount which is opaque. Worst stays on the full-metrics page. disk_root = (latest.get(host.name + ":/", {}) or {}).get("disk_used_pct") + sparks = {k: sparkline_svg(v[-120:]) for k, v in series_raw.items()} + # Normalize load by core count so it's comparable across hardware (load/core + # as %). Cores derived from the per-core CPU sub-resources already collected. + cores = sum(1 for k in latest if k.startswith(host.name + ":core")) + load1 = hostlvl.get("load_1m") + load_per_core = round(load1 / cores * 100) if (cores and load1 is not None) else None + psi = { + "cpu": hostlvl.get("psi_cpu_some_avg10"), + "mem": hostlvl.get("psi_mem_some_avg10"), + "io": hostlvl.get("psi_io_some_avg10"), + } ls = reg.last_seen_at if reg else None # "Deployed" is gated on the agent actually checking in (last_seen_at), NOT on # the registration row — that row is minted before the playbook runs, so a @@ -565,6 +602,7 @@ async def host_panel(host_id: str): return await render_template( "panel.html", host=host, reg=reg, hostlvl=hostlvl, disk_root=disk_root, + sparks=sparks, cores=cores, load1=load1, load_per_core=load_per_core, psi=psi, reporting=reporting, stale=stale, target=target, ansible_available=has_capability("ansible.run_playbook"), managed_key_set=bool((ansible_cfg.get("ssh_public_key") or "").strip()), diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html index 0c7d866..33fcd8d 100644 --- a/plugins/host_agent/templates/panel.html +++ b/plugins/host_agent/templates/panel.html @@ -14,24 +14,37 @@
{% if reporting %} - {# ── Reporting: at-a-glance metrics + lifecycle ── #} -
- {% set cpu = hostlvl.get('cpu_pct') %} - {% set mem = hostlvl.get('mem_used_pct') %} - {% set load1 = hostlvl.get('load_1m') %} + {# ── Reporting: at-a-glance metrics (+ sparkline trend) + lifecycle ── #} + {% set cpu = hostlvl.get('cpu_pct') %} + {% set mem = hostlvl.get('mem_used_pct') %} + {% set lbl = "font-size:0.72rem;color:var(--text-muted);text-transform:uppercase;" %} +
-
CPU
-
{{ '%.0f%%'|format(cpu) if cpu is not none else '—' }}
+
CPU
+
{{ '%.0f%%'|format(cpu) if cpu is not none else '—' }}
+ {{ sparks.cpu | safe }}
-
Memory
-
{{ '%.0f%%'|format(mem) if mem is not none else '—' }}
+
Memory
+
{{ '%.0f%%'|format(mem) if mem is not none else '—' }}
+ {{ sparks.mem | safe }}
-
Disk /
-
{{ '%.0f%%'|format(disk_root) if disk_root is not none else '—' }}
-
-
Load 1m
-
{{ '%.2f'|format(load1) if load1 is not none else '—' }}
+
Disk /
+
{{ '%.0f%%'|format(disk_root) if disk_root is not none else '—' }}
+ {{ sparks.disk | safe }}
+
+
Load /core
+
{{ '%d%%'|format(load_per_core) if load_per_core is not none else ('%.2f'|format(load1) if load1 is not none else '—') }}
+ {{ sparks.load | safe }}
+ {% if psi.cpu is not none or psi.mem is not none or psi.io is not none %} +
+ Pressure 10s +  CPU {{ '%.0f%%'|format(psi.cpu) if psi.cpu is not none else '—' }} + · Mem {{ '%.0f%%'|format(psi.mem) if psi.mem is not none else '—' }} + · IO {{ '%.0f%%'|format(psi.io) if psi.io is not none else '—' }} +
+ {% endif %}
Full metrics → From e5f6a11f941d57a7e3c9da148142244795638cac Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 11:25:25 -0400 Subject: [PATCH 063/126] feat(ansible): playbooks self-describe via "# description:" comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Playbooks can ship a human description Steward reads and shows when one is selected. Convention: a `# description: ` magic comment (Ansible rejects unknown play keys, so a comment is the portable place — works for third-party playbooks too); falls back to the first play's name:. sources .discover_playbook_description(). Surfaced at the top of the shared _playbook_vars.html partial, which loads on playbook selection in the host run form, schedules form, and browse run form. All four bundled playbooks (provision/install/update/docker_prune) now carry a description line. Unit tests added. Scribe #900. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ansible/bundled/host_agent/install.yml | 1 + .../ansible/bundled/host_agent/provision.yml | 1 + steward/ansible/bundled/host_agent/update.yml | 1 + .../bundled/maintenance/docker_prune.yml | 1 + steward/ansible/routes.py | 8 +++-- steward/ansible/sources.py | 26 ++++++++++++++++ steward/templates/ansible/_playbook_vars.html | 12 +++++-- tests/test_playbook_variables.py | 31 ++++++++++++++++++- 8 files changed, 75 insertions(+), 6 deletions(-) diff --git a/steward/ansible/bundled/host_agent/install.yml b/steward/ansible/bundled/host_agent/install.yml index ce4acdd..9257e92 100644 --- a/steward/ansible/bundled/host_agent/install.yml +++ b/steward/ansible/bundled/host_agent/install.yml @@ -1,4 +1,5 @@ --- +# description: Install or re-enroll the Steward host agent on an already-provisioned host. # Install (or update) the Steward host agent on a target. Mirrors the # curl-based install.sh. Pass the per-host registration token as extra-vars: # steward_url=https://steward.example steward_token= diff --git a/steward/ansible/bundled/host_agent/provision.yml b/steward/ansible/bundled/host_agent/provision.yml index 49883eb..c09c0e3 100644 --- a/steward/ansible/bundled/host_agent/provision.yml +++ b/steward/ansible/bundled/host_agent/provision.yml @@ -1,4 +1,5 @@ --- +# description: Provision a fresh host — create the steward account + managed key + passwordless sudo, then install the agent. # First-contact provisioning: bring a fresh host under Steward management, then # install the host agent — all in one idempotent run. # diff --git a/steward/ansible/bundled/host_agent/update.yml b/steward/ansible/bundled/host_agent/update.yml index 8dff5ff..b5b6f8c 100644 --- a/steward/ansible/bundled/host_agent/update.yml +++ b/steward/ansible/bundled/host_agent/update.yml @@ -1,4 +1,5 @@ --- +# description: Update the Steward host agent (refresh agent.py + restart) on hosts already running it. # Update an already-installed Steward host agent: refresh agent.py and restart. # # Deliberately does NOT touch the registration token or /etc/steward-agent.conf diff --git a/steward/ansible/bundled/maintenance/docker_prune.yml b/steward/ansible/bundled/maintenance/docker_prune.yml index 1fcb8ec..d4e744e 100644 --- a/steward/ansible/bundled/maintenance/docker_prune.yml +++ b/steward/ansible/bundled/maintenance/docker_prune.yml @@ -1,4 +1,5 @@ --- +# description: Reclaim disk on Docker / Swarm nodes by pruning unused images, containers, networks and build cache. # Reclaim disk on Docker / Docker Swarm nodes by removing unused data. # Safe by default: prunes dangling images, stopped containers, unused networks # and build cache. Set extra-vars to widen scope: diff --git a/steward/ansible/routes.py b/steward/ansible/routes.py index d88dca8..9bcd5c3 100644 --- a/steward/ansible/routes.py +++ b/steward/ansible/routes.py @@ -165,11 +165,14 @@ async def playbook_vars(): playbook_path = (request.args.get("playbook_path", "") or "").strip() source = next((s for s in _get_sources() if s["name"] == source_name), None) variables: list = [] + description = "" if source and playbook_path: contents = src_module.read_playbook(source["path"], playbook_path) if contents is not None: variables = src_module.discover_playbook_variables(contents) - return await render_template("ansible/_playbook_vars.html", variables=variables) + description = src_module.discover_playbook_description(contents) + return await render_template( + "ansible/_playbook_vars.html", variables=variables, description=description) @ansible_bp.get("/run-form//") @@ -186,6 +189,7 @@ async def run_form(source_name: str, playbook_path: str): if contents is None: return "Playbook not found", 404 variables = src_module.discover_playbook_variables(contents) + description = src_module.discover_playbook_description(contents) inventories = src_module.discover_inventories(source["path"]) async with current_app.db_sessionmaker() as db: @@ -197,7 +201,7 @@ async def run_form(source_name: str, playbook_path: str): return await render_template( "ansible/_run_form.html", source_name=source_name, playbook_path=playbook_path, - variables=variables, targets=targets, groups=groups, + variables=variables, description=description, targets=targets, groups=groups, inventories=inventories, ) diff --git a/steward/ansible/sources.py b/steward/ansible/sources.py index 3e26a20..c495ce2 100644 --- a/steward/ansible/sources.py +++ b/steward/ansible/sources.py @@ -17,6 +17,10 @@ _SECRET_VAR_RE = re.compile( r"(password|passwd|secret|token|api[_-]?key|private[_-]?key|credential)", re.I ) +# A playbook self-describes via a `# description:` magic comment (Ansible rejects +# unknown play keys, so a comment is the portable place). First match wins. +_DESCRIPTION_RE = re.compile(r"^\s*#\s*description:\s*(.+?)\s*$", re.I | re.M) + # Name of the always-present, read-only source of first-party playbooks shipped # inside the app (maintenance tasks, host-agent install). Not operator-editable. BUILTIN_SOURCE_NAME = "steward-builtin" @@ -195,6 +199,28 @@ def delete_playbook(source_path: str, relative_path: str) -> tuple[bool, str | N return True, None +def discover_playbook_description(content: str) -> str: + """A human-readable description of what a playbook does. + + Reads a ``# description: ...`` magic comment (first match), so any playbook + can self-describe without touching its YAML structure. Falls back to the + first play's ``name:``. Returns "" if neither is present. + """ + m = _DESCRIPTION_RE.search(content) + if m: + return m.group(1).strip() + import yaml + try: + plays = yaml.safe_load(content) + except yaml.YAMLError: + return "" + if isinstance(plays, list): + for play in plays: + if isinstance(play, dict) and play.get("name"): + return str(play["name"]).strip() + return "" + + def discover_playbook_variables(content: str) -> list[dict]: """Parse a playbook and list the variables an operator can set at run time. diff --git a/steward/templates/ansible/_playbook_vars.html b/steward/templates/ansible/_playbook_vars.html index ecc611c..31a96fd 100644 --- a/steward/templates/ansible/_playbook_vars.html +++ b/steward/templates/ansible/_playbook_vars.html @@ -1,6 +1,12 @@ -{# Discovered playbook variables as fill-in fields. Shared by the browse run - form, the host run form, and the schedule form. Expects `variables` (from - discover_playbook_variables). Rendered standalone by /ansible/playbook-vars. #} +{# Description + discovered variable fields for a selected playbook. Shared by + the browse run form, host run form, and schedule form. Expects `variables` + and `description`. Rendered standalone by /ansible/playbook-vars. #} +{% if description %} +
+ {{ description }} +
+{% endif %} {% if variables %}
Playbook variables diff --git a/tests/test_playbook_variables.py b/tests/test_playbook_variables.py index 560cf03..c8d9d81 100644 --- a/tests/test_playbook_variables.py +++ b/tests/test_playbook_variables.py @@ -2,7 +2,10 @@ import json from steward.ansible.executor import build_extra_vars_file -from steward.ansible.sources import discover_playbook_variables +from steward.ansible.sources import ( + discover_playbook_description, + discover_playbook_variables, +) def _names(variables): @@ -108,3 +111,29 @@ def test_extra_vars_file_roundtrip_and_space_safe(): def test_extra_vars_file_empty_is_noop(): assert build_extra_vars_file({}, "/td") == ([], []) assert build_extra_vars_file(None, "/td") == ([], []) + + +# ── Playbook description discovery ──────────────────────────────────────────── + +def test_description_from_magic_comment(): + content = """--- +# description: Reclaim disk by pruning Docker. +- hosts: all + name: Docker prune + tasks: [] +""" + assert discover_playbook_description(content) == "Reclaim disk by pruning Docker." + + +def test_description_case_insensitive_key(): + assert discover_playbook_description("# DESCRIPTION: trimmed \n- hosts: all") == "trimmed" + + +def test_description_falls_back_to_play_name(): + content = "- hosts: all\n name: Configure web servers\n tasks: []\n" + assert discover_playbook_description(content) == "Configure web servers" + + +def test_description_empty_when_neither_present(): + assert discover_playbook_description("- hosts: all\n tasks: []\n") == "" + assert discover_playbook_description("not: : valid") == "" From b32fce1d7412aff78ba7ba175bc7420e345f0f87 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 11:26:56 -0400 Subject: [PATCH 064/126] docs: Steward playbook-authoring conventions reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A self-contained guide (docs/reference/playbook-authoring.md) to the contract between a playbook and Steward's run UI — the `# description:` comment, vars/vars_prompt → fill-in fields, secret naming, hosts: all targeting, managed credentials, and idempotency. Includes a "what Steward reads" summary table, the metadata/extensibility note (only `# description:` today; reserved `# steward::` namespace for future keys), and an annotated example. Meant to be fed to another session authoring Steward-friendly playbooks. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/reference/playbook-authoring.md | 169 +++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 docs/reference/playbook-authoring.md diff --git a/docs/reference/playbook-authoring.md b/docs/reference/playbook-authoring.md new file mode 100644 index 0000000..02a6bf9 --- /dev/null +++ b/docs/reference/playbook-authoring.md @@ -0,0 +1,169 @@ +# Authoring Steward-friendly Ansible playbooks + +This is the contract between a playbook and Steward's run UI. Follow it and a +playbook drops into Steward with a description, fill-in variable fields, correct +targeting, and credentials supplied automatically — no per-playbook wiring. + +It applies to **any** playbook in a configured source (bundled, the writable +local source, or a git source), including third-party ones. + +--- + +## 1. Describe what it does — `# description:` + +Steward shows a one-line description when a playbook is selected in the run form. + +```yaml +--- +# description: Reclaim disk on Docker/Swarm nodes by pruning unused images and build cache. +- name: Docker system prune + hosts: all + ... +``` + +- Format: a comment line `# description: ` anywhere in the file. **First + match wins.** Case-insensitive on the `description:` key. +- Why a comment and not a key: Ansible rejects unknown *play* keys (you can't add + `description:` to a play), so a comment is the portable place. It survives + `ansible-playbook` untouched. +- Fallback: if there's no `# description:` comment, Steward uses the **first + play's `name:`**. So always give your play a meaningful `name:` even without + the comment. +- Keep it to one readable line. Longer "how to use" notes can go in additional + normal comments — Steward only reads the `description:` line. + +## 2. Declare tunables in `vars:` — they become fill-in fields + +Every **scalar** entry in a play's `vars:` block becomes an editable field in the +run form, with the default shown as the input's placeholder. + +```yaml + vars: + prune_all_images: false # → checkbox-ish text field, placeholder "default: false" + keep_last_days: 7 # → field, placeholder "default: 7" + registry_url: "" # → field, placeholder "no default" +``` + +- **Blank field = use the default.** Steward only sends fields the operator + actually fills, so an untouched field falls through to the playbook/inventory + default rather than overriding it. +- Only scalars (string/int/float/bool) surface as fields. Lists/dicts are + skipped — set those in the playbook or via inventory group/host vars. +- Values are delivered as **extra-vars** (`-e`), which are the **highest** + precedence in Ansible — they override the `vars:` defaults. (This is why the + default can be empty and still be safely overridden at run time.) + +### `vars_prompt:` also works + +Steward reads `vars_prompt` too. Use it when you want an explicit prompt or a +required value: + +```yaml + vars_prompt: + - name: release_tag + prompt: "Which release to deploy?" # shown as the field label + # no default → Steward marks the field REQUIRED + - name: admin_password + prompt: "Admin password" + private: true # → masked field, never stored +``` + +## 3. Secrets — name them so they're masked and not persisted + +A field is treated as **secret** (rendered masked, and its value is **never +written to the DB / run history**) when either: + +- the variable name contains `password`, `passwd`, `secret`, `token`, + `api_key` / `apikey`, `private_key`, or `credential` (case-insensitive), **or** +- it's a `vars_prompt` entry with `private: true` (Ansible's default for + vars_prompt is private). + +So name sensitive variables accordingly (`db_password`, `api_token`, +`vault_secret`) and they're handled safely with no extra config. Non-secret +run-time vars are persisted (so scheduled runs can reuse them); secret ones are +passed to the run only. + +## 4. Target with `hosts: all` + +Steward builds the inventory itself from the **target / group** the operator +picks in the run form (or the single host on a host page). Your play should: + +```yaml + hosts: all # run against whatever Steward scoped to +``` + +- Don't hardcode hostnames or rely on a checked-in inventory for Steward runs + (Steward generates a fresh inventory per run). +- Per-host connection vars (`ansible_host`, plus anything you set on the target + in **Ansible → Inventory**) arrive as inventory host vars. +- The run form's **Limit** / **Tags** map to `--limit` / `--tags`. + +## 5. Privileges & connection — don't put credentials in the playbook + +Steward supplies SSH and become for you: + +- Steady-state runs connect as the managed **`steward`** account using Steward's + managed key; that account has **passwordless sudo**. So just use + `become: true` where you need root. +- First-contact provisioning uses a one-time bootstrap user/password the + operator enters (never stored). +- Never embed SSH keys, passwords, or `ansible_user`/`ansible_ssh_pass` in the + playbook. Connection identity is global (Settings → Ansible) or per-target. + +## 6. Be idempotent + +Steward re-runs playbooks (updates, schedules, retries). Use modules that +converge (state-based) rather than ad-hoc `command:`/`shell:` where possible, so +re-runs are safe. + +--- + +## What Steward reads from a playbook (summary) + +| Source in the playbook | What Steward does with it | +|---|---| +| `# description: ` comment | Description shown on selection (first match) | +| first play `name:` | Description fallback | +| `vars:` scalar entries | Run-time fill-in fields (placeholder = default) | +| `vars_prompt:` entries | Run-time fields (required if no default) | +| secret-looking var name / `private: true` | Field masked + value not persisted | +| `hosts:` | Expected to be `all`; Steward provides the inventory | + +Everything else (SSH user/key, become password, the inventory, `steward_token` +etc. for the agent playbooks) is injected by Steward at run time. + +## Metadata convention & extensibility + +Today the **only comment-based metadata** Steward parses is `# description:`. +Everything else is *structural* (`vars`, `vars_prompt`, `hosts`, the play +`name`). If we add more comment metadata later, it will use a reserved +**`# steward:: `** namespace (e.g. `# steward:category: maintenance`, +`# steward:confirm: true` for a run-confirmation gate) so it can't collide with +ordinary comments. **These extra keys are not implemented yet** — only +`# description:` is. Don't author playbooks that depend on `# steward:*` keys +until this doc says they're live. + +## Minimal annotated example + +```yaml +--- +# description: Restart a systemd service and confirm it came back up. +- name: Restart a service + hosts: all + become: true + gather_facts: false + vars: + service_name: "" # required-ish: operator fills it in the run form + tasks: + - name: Validate input + ansible.builtin.assert: + that: service_name | default('') | length > 0 + fail_msg: "Set service_name." + - name: Restart + ansible.builtin.systemd: + name: "{{ service_name }}" + state: restarted + - name: Confirm active + ansible.builtin.command: "systemctl is-active {{ service_name }}" + changed_when: false +``` From 42f7840c2601edfbab418187a661b495e4ac40eb Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 11:35:35 -0400 Subject: [PATCH 065/126] feat(ansible): steward:category + steward:confirm playbook metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the playbook metadata convention with a namespaced `# steward::` comment block: - steward:category — free-text grouping label, shown as a badge in the browse list and on the run form. - steward:confirm — true/yes/1/on marks a playbook destructive; the run form then requires a confirmation tick (required checkbox in the shared vars fragment) before it can launch. sources.discover_playbook_meta() parses description + category + confirm (first match per key; `# description:` still primary, `# steward:description:` alias). discover_playbook_description() now delegates to it. The browse list reads per-playbook meta to show category badges + descriptions; the run-form and playbook-vars fragments render the badge + confirm gate. Bundled playbooks tagged: docker_prune → category maintenance + confirm true; provision/install/update → category host-agent. Docs: docs/reference/playbook-authoring.md updated (keys now implemented) and a quick reference added next to the code at steward/ansible/PLAYBOOK_CONVENTIONS.md. Tests added for category/confirm/alias parsing. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/reference/playbook-authoring.md | 40 ++++++++--- steward/ansible/PLAYBOOK_CONVENTIONS.md | 39 +++++++++++ .../ansible/bundled/host_agent/install.yml | 1 + .../ansible/bundled/host_agent/provision.yml | 1 + steward/ansible/bundled/host_agent/update.yml | 1 + .../bundled/maintenance/docker_prune.yml | 2 + steward/ansible/routes.py | 23 +++++-- steward/ansible/sources.py | 68 +++++++++++++------ steward/templates/ansible/_playbook_vars.html | 13 +++- steward/templates/ansible/browse.html | 14 ++-- tests/test_playbook_variables.py | 29 ++++++++ 11 files changed, 192 insertions(+), 39 deletions(-) create mode 100644 steward/ansible/PLAYBOOK_CONVENTIONS.md diff --git a/docs/reference/playbook-authoring.md b/docs/reference/playbook-authoring.md index 02a6bf9..8b6c19e 100644 --- a/docs/reference/playbook-authoring.md +++ b/docs/reference/playbook-authoring.md @@ -124,6 +124,8 @@ re-runs are safe. |---|---| | `# description: ` comment | Description shown on selection (first match) | | first play `name:` | Description fallback | +| `# steward:category: ` | Grouping badge in the browse list + run form | +| `# steward:confirm: true` | Requires a confirmation tick before the run launches | | `vars:` scalar entries | Run-time fill-in fields (placeholder = default) | | `vars_prompt:` entries | Run-time fields (required if no default) | | secret-looking var name / `private: true` | Field masked + value not persisted | @@ -132,16 +134,36 @@ re-runs are safe. Everything else (SSH user/key, become password, the inventory, `steward_token` etc. for the agent playbooks) is injected by Steward at run time. -## Metadata convention & extensibility +## Metadata comments -Today the **only comment-based metadata** Steward parses is `# description:`. -Everything else is *structural* (`vars`, `vars_prompt`, `hosts`, the play -`name`). If we add more comment metadata later, it will use a reserved -**`# steward:: `** namespace (e.g. `# steward:category: maintenance`, -`# steward:confirm: true` for a run-confirmation gate) so it can't collide with -ordinary comments. **These extra keys are not implemented yet** — only -`# description:` is. Don't author playbooks that depend on `# steward:*` keys -until this doc says they're live. +Steward reads metadata from magic comments (Ansible rejects unknown play keys, +so comments are the portable place). Two forms: + +- `# description: ` — the description (see §1). +- `# steward:: ` — the namespaced metadata block. First match per + key wins; keys are case-insensitive. + +**Implemented `# steward:` keys:** + +| Key | Example | Effect | +|---|---|---| +| `category` | `# steward:category: maintenance` | Free-text grouping label. Shown as a badge in the browse list and on the run form. Purely organizational. | +| `confirm` | `# steward:confirm: true` | Marks the playbook as significant/destructive. The run form then requires an explicit confirmation tick before it can be launched. Use for data-loss-capable plays (prune with volumes, resets, etc.). Truthy values: `true`/`yes`/`1`/`on`. | +| `description` | `# steward:description: ...` | Alias for `# description:` (the unprefixed form takes precedence if both exist). | + +```yaml +--- +# description: Reclaim disk on Docker/Swarm nodes by pruning unused data. +# steward:category: maintenance +# steward:confirm: true +- name: Docker system prune + hosts: all + ... +``` + +Other `# steward::` keys are simply ignored today — the namespace is +reserved, so it's safe to add ones Steward doesn't yet understand without +breaking anything, but only `category`, `confirm`, and `description` do something. ## Minimal annotated example diff --git a/steward/ansible/PLAYBOOK_CONVENTIONS.md b/steward/ansible/PLAYBOOK_CONVENTIONS.md new file mode 100644 index 0000000..a44c3e3 --- /dev/null +++ b/steward/ansible/PLAYBOOK_CONVENTIONS.md @@ -0,0 +1,39 @@ +# Steward playbook conventions (quick reference) + +What Steward reads from a playbook in this subsystem. Full guide: +`docs/reference/playbook-authoring.md`. Parser: +`steward/ansible/sources.py` — `discover_playbook_meta()` and +`discover_playbook_variables()`. + +## Magic comments (metadata) + +```yaml +--- +# description: One line on what this playbook does. # shown on selection +# steward:category: maintenance # grouping badge +# steward:confirm: true # require confirm to run +- name: … # used as the description if no `# description:` comment + hosts: all # Steward generates the inventory from the chosen target/group +``` + +- `# description:` — first match wins; case-insensitive; falls back to first + play `name:`. +- `# steward:: ` — namespaced metadata. Implemented keys: + - `category` — free-text grouping label (badge in browse + run form). + - `confirm` — `true`/`yes`/`1`/`on` → run form requires a confirmation tick. + - `description` — alias for `# description:` (unprefixed wins if both present). + - Unknown `# steward:*` keys are ignored (namespace reserved for future use). + +## Structural cues (no comments needed) + +- **`vars:` scalars** → editable run-time fields (default shown as placeholder; + blank = keep default; values sent as highest-precedence extra-vars). +- **`vars_prompt:`** → run-time fields (required when no default; `private: true` + → masked). +- **Secret naming** — a var whose name contains `password`/`passwd`/`secret`/ + `token`/`api_key`/`private_key`/`credential` is masked **and not persisted**. +- **`hosts: all`** — target via the run form's scope; don't hardcode hosts. +- **Credentials** — never embed them. Steward connects as the managed `steward` + account (passwordless sudo; use `become: true`) or one-time bootstrap creds. + +Be idempotent — Steward re-runs playbooks (updates, schedules, retries). diff --git a/steward/ansible/bundled/host_agent/install.yml b/steward/ansible/bundled/host_agent/install.yml index 9257e92..a1074d2 100644 --- a/steward/ansible/bundled/host_agent/install.yml +++ b/steward/ansible/bundled/host_agent/install.yml @@ -1,5 +1,6 @@ --- # description: Install or re-enroll the Steward host agent on an already-provisioned host. +# steward:category: host-agent # Install (or update) the Steward host agent on a target. Mirrors the # curl-based install.sh. Pass the per-host registration token as extra-vars: # steward_url=https://steward.example steward_token= diff --git a/steward/ansible/bundled/host_agent/provision.yml b/steward/ansible/bundled/host_agent/provision.yml index c09c0e3..09b50ec 100644 --- a/steward/ansible/bundled/host_agent/provision.yml +++ b/steward/ansible/bundled/host_agent/provision.yml @@ -1,5 +1,6 @@ --- # description: Provision a fresh host — create the steward account + managed key + passwordless sudo, then install the agent. +# steward:category: host-agent # First-contact provisioning: bring a fresh host under Steward management, then # install the host agent — all in one idempotent run. # diff --git a/steward/ansible/bundled/host_agent/update.yml b/steward/ansible/bundled/host_agent/update.yml index b5b6f8c..82f60f9 100644 --- a/steward/ansible/bundled/host_agent/update.yml +++ b/steward/ansible/bundled/host_agent/update.yml @@ -1,5 +1,6 @@ --- # description: Update the Steward host agent (refresh agent.py + restart) on hosts already running it. +# steward:category: host-agent # Update an already-installed Steward host agent: refresh agent.py and restart. # # Deliberately does NOT touch the registration token or /etc/steward-agent.conf diff --git a/steward/ansible/bundled/maintenance/docker_prune.yml b/steward/ansible/bundled/maintenance/docker_prune.yml index d4e744e..51c5bcf 100644 --- a/steward/ansible/bundled/maintenance/docker_prune.yml +++ b/steward/ansible/bundled/maintenance/docker_prune.yml @@ -1,5 +1,7 @@ --- # description: Reclaim disk on Docker / Swarm nodes by pruning unused images, containers, networks and build cache. +# steward:category: maintenance +# steward:confirm: true # Reclaim disk on Docker / Docker Swarm nodes by removing unused data. # Safe by default: prunes dangling images, stopped containers, unused networks # and build cache. Set extra-vars to widen scope: diff --git a/steward/ansible/routes.py b/steward/ansible/routes.py index 9bcd5c3..af09e19 100644 --- a/steward/ansible/routes.py +++ b/steward/ansible/routes.py @@ -50,7 +50,16 @@ async def browse(): all_sources = _get_sources() source_data = [] for source in all_sources: - playbooks = src_module.discover_playbooks(source["path"]) + playbooks = [] + for path in src_module.discover_playbooks(source["path"]): + contents = src_module.read_playbook(source["path"], path) + meta = (src_module.discover_playbook_meta(contents) if contents is not None + else {"description": "", "category": ""}) + playbooks.append({ + "path": path, + "category": meta["category"], + "description": meta["description"], + }) inventories = src_module.discover_inventories(source["path"]) source_data.append({ "source": source, @@ -165,14 +174,15 @@ async def playbook_vars(): playbook_path = (request.args.get("playbook_path", "") or "").strip() source = next((s for s in _get_sources() if s["name"] == source_name), None) variables: list = [] - description = "" + meta: dict = {"description": "", "category": "", "confirm": False} if source and playbook_path: contents = src_module.read_playbook(source["path"], playbook_path) if contents is not None: variables = src_module.discover_playbook_variables(contents) - description = src_module.discover_playbook_description(contents) + meta = src_module.discover_playbook_meta(contents) return await render_template( - "ansible/_playbook_vars.html", variables=variables, description=description) + "ansible/_playbook_vars.html", variables=variables, + description=meta["description"], category=meta["category"], confirm=meta["confirm"]) @ansible_bp.get("/run-form//") @@ -189,7 +199,7 @@ async def run_form(source_name: str, playbook_path: str): if contents is None: return "Playbook not found", 404 variables = src_module.discover_playbook_variables(contents) - description = src_module.discover_playbook_description(contents) + meta = src_module.discover_playbook_meta(contents) inventories = src_module.discover_inventories(source["path"]) async with current_app.db_sessionmaker() as db: @@ -201,7 +211,8 @@ async def run_form(source_name: str, playbook_path: str): return await render_template( "ansible/_run_form.html", source_name=source_name, playbook_path=playbook_path, - variables=variables, description=description, targets=targets, groups=groups, + variables=variables, description=meta["description"], category=meta["category"], + confirm=meta["confirm"], targets=targets, groups=groups, inventories=inventories, ) diff --git a/steward/ansible/sources.py b/steward/ansible/sources.py index c495ce2..9c0cc6e 100644 --- a/steward/ansible/sources.py +++ b/steward/ansible/sources.py @@ -20,6 +20,52 @@ _SECRET_VAR_RE = re.compile( # A playbook self-describes via a `# description:` magic comment (Ansible rejects # unknown play keys, so a comment is the portable place). First match wins. _DESCRIPTION_RE = re.compile(r"^\s*#\s*description:\s*(.+?)\s*$", re.I | re.M) +# Namespaced metadata: `# steward:: ` (category, confirm, …). +_STEWARD_META_RE = re.compile(r"^\s*#\s*steward:(\w+):\s*(.+?)\s*$", re.I | re.M) +_TRUTHY = {"1", "true", "yes", "on"} + + +def _first_play_name(content: str) -> str: + import yaml + try: + plays = yaml.safe_load(content) + except yaml.YAMLError: + return "" + if isinstance(plays, list): + for play in plays: + if isinstance(play, dict) and play.get("name"): + return str(play["name"]).strip() + return "" + + +def discover_playbook_meta(content: str) -> dict: + """Parse Steward's playbook metadata from magic comments. + + Returns {description, category, confirm}. Sources: + - description: ``# description:`` (primary) or ``# steward:description:``; + falls back to the first play's ``name:``. + - category: ``# steward:category: `` (free-text grouping label). + - confirm: ``# steward:confirm: true`` → require an explicit confirmation + in the run form before launching (for destructive playbooks). + First match wins for each key. + """ + steward: dict[str, str] = {} + for km in _STEWARD_META_RE.finditer(content): + steward.setdefault(km.group(1).lower(), km.group(2).strip()) + + m = _DESCRIPTION_RE.search(content) + if m: + description = m.group(1).strip() + elif steward.get("description"): + description = steward["description"] + else: + description = _first_play_name(content) + + return { + "description": description, + "category": steward.get("category", ""), + "confirm": str(steward.get("confirm", "")).strip().lower() in _TRUTHY, + } # Name of the always-present, read-only source of first-party playbooks shipped # inside the app (maintenance tasks, host-agent install). Not operator-editable. @@ -200,25 +246,9 @@ def delete_playbook(source_path: str, relative_path: str) -> tuple[bool, str | N def discover_playbook_description(content: str) -> str: - """A human-readable description of what a playbook does. - - Reads a ``# description: ...`` magic comment (first match), so any playbook - can self-describe without touching its YAML structure. Falls back to the - first play's ``name:``. Returns "" if neither is present. - """ - m = _DESCRIPTION_RE.search(content) - if m: - return m.group(1).strip() - import yaml - try: - plays = yaml.safe_load(content) - except yaml.YAMLError: - return "" - if isinstance(plays, list): - for play in plays: - if isinstance(play, dict) and play.get("name"): - return str(play["name"]).strip() - return "" + """A human-readable description of what a playbook does (see + discover_playbook_meta). Returns "" if none can be determined.""" + return discover_playbook_meta(content)["description"] def discover_playbook_variables(content: str) -> list[dict]: diff --git a/steward/templates/ansible/_playbook_vars.html b/steward/templates/ansible/_playbook_vars.html index 31a96fd..38d2149 100644 --- a/steward/templates/ansible/_playbook_vars.html +++ b/steward/templates/ansible/_playbook_vars.html @@ -1,12 +1,23 @@ {# Description + discovered variable fields for a selected playbook. Shared by the browse run form, host run form, and schedule form. Expects `variables` and `description`. Rendered standalone by /ansible/playbook-vars. #} -{% if description %} +{% if description or category %}
+ {% if category %}{{ category }}{% endif %} {{ description }}
{% endif %} +{% if confirm %} + +{% endif %} {% if variables %}
Playbook variables diff --git a/steward/templates/ansible/browse.html b/steward/templates/ansible/browse.html index 4832eff..ba30058 100644 --- a/steward/templates/ansible/browse.html +++ b/steward/templates/ansible/browse.html @@ -70,13 +70,19 @@ {% for pb in sd.playbooks %} - {{ pb }} + +
+ {% if pb.category %}{{ pb.category }}{% endif %} + {{ pb.path }} +
+ {% if pb.description %}
{{ pb.description }}
{% endif %} + - View + View {% if sd.editable and session.user_role == 'admin' %} - Edit + Edit {% endif %} - Run diff --git a/tests/test_playbook_variables.py b/tests/test_playbook_variables.py index c8d9d81..a5ffa08 100644 --- a/tests/test_playbook_variables.py +++ b/tests/test_playbook_variables.py @@ -4,6 +4,7 @@ import json from steward.ansible.executor import build_extra_vars_file from steward.ansible.sources import ( discover_playbook_description, + discover_playbook_meta, discover_playbook_variables, ) @@ -137,3 +138,31 @@ def test_description_falls_back_to_play_name(): def test_description_empty_when_neither_present(): assert discover_playbook_description("- hosts: all\n tasks: []\n") == "" assert discover_playbook_description("not: : valid") == "" + + +# ── steward:category / steward:confirm metadata ─────────────────────────────── + +def test_meta_category_and_confirm(): + content = """--- +# description: Prune docker. +# steward:category: maintenance +# steward:confirm: true +- hosts: all + tasks: [] +""" + meta = discover_playbook_meta(content) + assert meta["description"] == "Prune docker." + assert meta["category"] == "maintenance" + assert meta["confirm"] is True + + +def test_meta_confirm_falsey_and_absent(): + assert discover_playbook_meta("# steward:confirm: false\n- hosts: all")["confirm"] is False + assert discover_playbook_meta("- hosts: all\n name: x")["confirm"] is False + assert discover_playbook_meta("- hosts: all\n name: x")["category"] == "" + + +def test_meta_steward_description_alias(): + # `# steward:description:` works when there's no plain `# description:`. + meta = discover_playbook_meta("# steward:description: via namespace\n- hosts: all") + assert meta["description"] == "via namespace" From 609bd78af2f0a2021b9ed02108af7eaa430a3bc4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 12:00:12 -0400 Subject: [PATCH 066/126] feat(dashboard): unified Hosts widget + grouped widget picker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring the dashboard in line with the unified host IA + improved displays. - New core "Hosts — Overview" widget (/hosts/overview/widget): one row per host combining monitor status (ping dot + latency, uptime 24h) with the agent glance (CPU / memory / disk root + stale flag), each row linking to the host hub. Reads agent data from the generic PluginMetric table via a core-safe _agent_overview_by_host helper (no host_agent import); freshness vs the plugin's stale window. The granular Ping/DNS/Uptime/Agent widgets stay. - Group the add-widget picker into Core monitors / Monitoring capabilities / Integrations (a `group` field on every WIDGET_REGISTRY entry + section headings in _edit_panels.html), matching the Settings → Plugins taxonomy. - Fix the agent fleet widget rows to link to the host hub (/hosts/) instead of the old /plugins/host_agent// page. Scribe #903. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../host_agent/templates/widget_table.html | 2 +- steward/core/widgets.py | 36 ++++++++ steward/hosts/routes.py | 85 +++++++++++++++++++ steward/templates/dashboard/_edit_panels.html | 9 +- steward/templates/hosts/overview_widget.html | 42 +++++++++ 5 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 steward/templates/hosts/overview_widget.html diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html index 1250740..65bb418 100644 --- a/plugins/host_agent/templates/widget_table.html +++ b/plugins/host_agent/templates/widget_table.html @@ -5,7 +5,7 @@ {% set stale = r.stale %}
- {{ r.host.name }} diff --git a/steward/core/widgets.py b/steward/core/widgets.py index d637f1f..5f92858 100644 --- a/steward/core/widgets.py +++ b/steward/core/widgets.py @@ -32,6 +32,16 @@ WIDGET_REGISTRY: dict[str, dict] = { "poll": True, "params": [], }, + "hosts_overview": { + "key": "hosts_overview", + "label": "Hosts — Overview", + "description": "One row per host: monitor status + uptime and agent CPU/memory/disk, linking to the host hub", + "hx_url": "/hosts/overview/widget", + "detail_url": "/hosts/", + "plugin": None, + "poll": True, + "params": [], + }, "uptime_summary": { "key": "uptime_summary", "label": "Uptime / SLA", @@ -292,8 +302,34 @@ WIDGET_REGISTRY: dict[str, dict] = { } +# Group each widget for the picker (Core monitors / Monitoring capabilities / +# Integrations), mirroring the Settings → Plugins taxonomy. Capability plugins +# are host/monitoring facets; traefik/unifi are discrete vendor integrations. +_CORE_WIDGETS = {"ping", "dns", "status_overview", "uptime_summary", "hosts_overview"} +_INTEGRATION_PLUGINS = {"traefik", "unifi"} +GROUP_LABELS = { + "core": "Core monitors", + "capability": "Monitoring capabilities", + "integration": "Integrations", +} +GROUP_ORDER = ["core", "capability", "integration"] + + +def _group_for(w: dict) -> str: + if w["key"] in _CORE_WIDGETS: + return "core" + if w.get("plugin") in _INTEGRATION_PLUGINS: + return "integration" + return "capability" + + +for _w in WIDGET_REGISTRY.values(): + _w["group"] = _group_for(_w) + + def register_widget(definition: dict) -> None: """Register a widget at runtime (used by external plugins).""" + definition.setdefault("group", _group_for(definition)) WIDGET_REGISTRY[definition["key"]] = definition diff --git a/steward/hosts/routes.py b/steward/hosts/routes.py index e689220..804ffda 100644 --- a/steward/hosts/routes.py +++ b/steward/hosts/routes.py @@ -104,6 +104,91 @@ async def _agent_metrics_by_host(db, host_names: list[str]) -> dict[str, dict]: return out +async def _agent_overview_by_host(db, host_names: list[str]) -> dict[str, dict]: + """Per-host agent glance (cpu/mem/root-disk + freshness) from PluginMetric. + + Core-safe (no host_agent import). Freshness compares the latest host-level + sample against the plugin's stale window (config), defaulting to 180s. + """ + from steward.models.metrics import PluginMetric + if not host_names: + return {} + now = datetime.now(timezone.utc) + stale_after = int( + (current_app.config.get("PLUGINS", {}).get("host_agent", {}) or {}) + .get("stale_after_seconds", 180)) + resources = list(host_names) + [h + ":/" for h in host_names] + subq = ( + select( + PluginMetric.resource_name, PluginMetric.metric_name, + func.max(PluginMetric.recorded_at).label("m"), + ) + .where( + PluginMetric.source_module == "host_agent", + PluginMetric.resource_name.in_(resources), + PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "disk_used_pct")), + ) + .group_by(PluginMetric.resource_name, PluginMetric.metric_name) + ).subquery() + rows = (await db.execute( + select(PluginMetric).join( + subq, + (PluginMetric.resource_name == subq.c.resource_name) + & (PluginMetric.metric_name == subq.c.metric_name) + & (PluginMetric.recorded_at == subq.c.m), + ).where(PluginMetric.source_module == "host_agent") + )).scalars().all() + out: dict[str, dict] = {} + for r in rows: + if r.resource_name.endswith(":/") and r.metric_name == "disk_used_pct": + out.setdefault(r.resource_name[:-2], {})["disk_root"] = r.value + elif r.resource_name in host_names: + d = out.setdefault(r.resource_name, {}) + d[r.metric_name] = r.value + if d.get("_ts") is None or r.recorded_at > d["_ts"]: + d["_ts"] = r.recorded_at + for d in out.values(): + ts = d.pop("_ts", None) + d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after) + return out + + +@hosts_bp.get("/overview/widget") +@require_role(UserRole.viewer) +async def overview_widget(): + """Dashboard widget: unified per-host monitor + agent glance, linking to the hub.""" + async with current_app.db_sessionmaker() as db: + hosts = (await db.execute(select(Host).order_by(Host.name))).scalars().all() + latest_ping_subq = ( + select(PingResult.host_id, func.max(PingResult.probed_at).label("max_at")) + .group_by(PingResult.host_id).subquery() + ) + latest_pings = {r.host_id: r for r in (await db.execute( + select(PingResult).join( + latest_ping_subq, + (PingResult.host_id == latest_ping_subq.c.host_id) + & (PingResult.probed_at == latest_ping_subq.c.max_at), + ))).scalars()} + latest_dns_subq = ( + select(DnsResult.host_id, func.max(DnsResult.resolved_at).label("max_at")) + .group_by(DnsResult.host_id).subquery() + ) + latest_dns = {r.host_id: r for r in (await db.execute( + select(DnsResult).join( + latest_dns_subq, + (DnsResult.host_id == latest_dns_subq.c.host_id) + & (DnsResult.resolved_at == latest_dns_subq.c.max_at), + ))).scalars()} + uptime = await _compute_uptime(db) + agent = await _agent_overview_by_host(db, [h.name for h in hosts]) + + return await render_template( + "hosts/overview_widget.html", + hosts=hosts, latest_pings=latest_pings, latest_dns=latest_dns, + uptime=uptime, agent=agent, + ) + + @hosts_bp.get("/") @require_role(UserRole.viewer) async def list_hosts(): diff --git a/steward/templates/dashboard/_edit_panels.html b/steward/templates/dashboard/_edit_panels.html index a3c09e3..c9d63c9 100644 --- a/steward/templates/dashboard/_edit_panels.html +++ b/steward/templates/dashboard/_edit_panels.html @@ -37,8 +37,13 @@
Available Widgets
{% if addable %} + {% set _groups = [("core", "Core monitors"), ("capability", "Monitoring capabilities"), ("integration", "Integrations")] %} + {% for _gkey, _glabel in _groups %} + {% set _gw = addable | selectattr("group", "equalto", _gkey) | list %} + {% if _gw %} +
{{ _glabel }}
- {% for w in addable %} + {% for w in _gw %}
No plugin widgets available. Enable plugins in Settings → Plugins. diff --git a/steward/templates/hosts/overview_widget.html b/steward/templates/hosts/overview_widget.html new file mode 100644 index 0000000..6cb2323 --- /dev/null +++ b/steward/templates/hosts/overview_widget.html @@ -0,0 +1,42 @@ +{# Unified Hosts widget — monitor status + agent glance per host, links to the hub. #} +{% if hosts %} +
+ {% for host in hosts %} + {% set ping = latest_pings.get(host.id) %} + {% set dns = latest_dns.get(host.id) %} + {% set ut = uptime.get(host.id, {}) %} + {% set a = agent.get(host.name, {}) %} + {% set down = host.ping_enabled and ping and ping.status.value != 'up' %} +
+ + + {{ host.name }} + + + {# Monitors #} + {% if host.ping_enabled and ping and ping.response_time_ms is not none %} + ping {{ "%.0f"|format(ping.response_time_ms) }}ms + {% endif %} + {% if ut.get('24h') is not none %} + 24h {{ "%.1f"|format(ut['24h']) }}% + {% endif %} + {# Agent glance #} + {% if a %} + {% if a.get('cpu_pct') is not none %}cpu {{ "%.0f"|format(a.cpu_pct) }}%{% endif %} + {% if a.get('mem_used_pct') is not none %}mem {{ "%.0f"|format(a.mem_used_pct) }}%{% endif %} + {% if a.get('disk_root') is not none %}disk / {{ "%.0f"|format(a.disk_root) }}%{% endif %} + {% if not a.fresh %}stale{% endif %} + {% else %} + no agent + {% endif %} + +
+ {% endfor %} +
+{% else %} +
+ No hosts yet. Add one. +
+{% endif %} From e0253fba4826555f77f92be14b2d562cc436b654 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 12:46:08 -0400 Subject: [PATCH 067/126] fix(ansible): surface run failure reason; widget audit cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run UX: when start_run throws before/around launch (e.g. ENOSPC creating the temp dir — the box is out of disk), the run was marked "failed" with empty output. Now the exception is broadcast + written to the run output/results so the run view shows e.g. "[run error] OSError: [Errno 28] No space left on device" instead of a blank failure. Widget audit follow-ups (no broken links were found; these are consistency): - host_resource_history widget now charts root (/) disk, consistent with the host panel (was the opaque "disk worst"). - host_resources widget: tooltip on the health dot explaining it warns on the worst mount while the number shows root. - status_overview widget detail_url /status → /status/ (avoid redirect). - Normalize ad-hoc widget empty-states to the shared .empty style (wording, which distinguishes "configured" vs "data yet", preserved). Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/docker/templates/docker/widget.html | 2 +- plugins/host_agent/routes.py | 15 ++++++++++----- plugins/host_agent/templates/widget_history.html | 6 +++--- plugins/host_agent/templates/widget_table.html | 3 ++- plugins/unifi/templates/unifi/widget_clients.html | 2 +- steward/ansible/executor.py | 9 ++++++++- steward/core/widgets.py | 2 +- steward/templates/status/widget.html | 2 +- 8 files changed, 27 insertions(+), 14 deletions(-) diff --git a/plugins/docker/templates/docker/widget.html b/plugins/docker/templates/docker/widget.html index 427018f..061df4b 100644 --- a/plugins/docker/templates/docker/widget.html +++ b/plugins/docker/templates/docker/widget.html @@ -1,6 +1,6 @@ {# docker/widget.html — dashboard widget: container status overview #} {% if not containers and running_count == 0 and stopped_count == 0 %} -
No containers found.
+

No containers found.

{% else %}
diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 199c552..57c92bc 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -398,16 +398,21 @@ async def widget_history(): points = (await session.execute( select(PluginMetric).where( PluginMetric.source_module == SOURCE_MODULE, - PluginMetric.resource_name == host.name, - PluginMetric.metric_name.in_( - ("cpu_pct", "mem_used_pct", "disk_used_pct_worst")), PluginMetric.recorded_at >= cutoff, + or_( + and_(PluginMetric.resource_name == host.name, + PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct"))), + and_(PluginMetric.resource_name == host.name + ":/", + PluginMetric.metric_name == "disk_used_pct"), + ), ).order_by(PluginMetric.recorded_at) )).scalars().all() - series: dict[str, list[dict]] = {"cpu_pct": [], "mem_used_pct": [], "disk_used_pct_worst": []} + # Disk = root (/), consistent with the host panel — not the opaque "worst". + series: dict[str, list[dict]] = {"cpu_pct": [], "mem_used_pct": [], "disk_root": []} for p in points: - series[p.metric_name].append({"t": p.recorded_at.isoformat(), "v": p.value}) + key = "disk_root" if p.resource_name != host.name else p.metric_name + series[key].append({"t": p.recorded_at.isoformat(), "v": p.value}) return await render_template("widget_history.html", host=host, series=series, hours=hours) diff --git a/plugins/host_agent/templates/widget_history.html b/plugins/host_agent/templates/widget_history.html index 35e363f..0d5b567 100644 --- a/plugins/host_agent/templates/widget_history.html +++ b/plugins/host_agent/templates/widget_history.html @@ -9,9 +9,9 @@ type: "line", data: { datasets: [ - { label: "CPU %", data: series.cpu_pct.map(p => ({x: p.t, y: p.v})) }, - { label: "Mem %", data: series.mem_used_pct.map(p => ({x: p.t, y: p.v})) }, - { label: "Disk % worst", data: series.disk_used_pct_worst.map(p => ({x: p.t, y: p.v})) }, + { label: "CPU %", data: series.cpu_pct.map(p => ({x: p.t, y: p.v})) }, + { label: "Mem %", data: series.mem_used_pct.map(p => ({x: p.t, y: p.v})) }, + { label: "Disk / %", data: series.disk_root.map(p => ({x: p.t, y: p.v})) }, ], }, options: { diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html index 65bb418..14a5bbd 100644 --- a/plugins/host_agent/templates/widget_table.html +++ b/plugins/host_agent/templates/widget_table.html @@ -4,7 +4,8 @@ {% for r in rows %} {% set stale = r.stale %}
- + {{ r.host.name }} diff --git a/plugins/unifi/templates/unifi/widget_clients.html b/plugins/unifi/templates/unifi/widget_clients.html index 99a5892..56059a9 100644 --- a/plugins/unifi/templates/unifi/widget_clients.html +++ b/plugins/unifi/templates/unifi/widget_clients.html @@ -1,6 +1,6 @@ {# unifi/widget_clients.html — dashboard widget: top active clients #} {% if not snapshot %} -
No client data yet.
+

No client data yet.

{% else %}
diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py index 3f51975..e061930 100644 --- a/steward/ansible/executor.py +++ b/steward/ansible/executor.py @@ -462,9 +462,16 @@ async def start_run( except _Aborted: final_status = "cancelled" - except Exception: + except Exception as exc: logger.exception("Ansible run %s failed with exception", run_id) final_status = "cancelled" if run_id in _cancelled else "failed" + # Surface WHY in the run output instead of an empty "failed" (e.g. ENOSPC + # when the temp dir can't be created — the run never reaches Ansible). + msg = f"[run error] {type(exc).__name__}: {exc}" + _broadcast(run_id, msg) + if len(failures) < _FAILURE_CAP: + failures.append(msg[:500]) + db_output = (db_output + "\n" + msg) if db_output else msg finally: if logf is not None: logf.close() diff --git a/steward/core/widgets.py b/steward/core/widgets.py index 5f92858..7ea8671 100644 --- a/steward/core/widgets.py +++ b/steward/core/widgets.py @@ -27,7 +27,7 @@ WIDGET_REGISTRY: dict[str, dict] = { "label": "Status — Overview", "description": "Unified up/down/pending summary across ping, DNS, and HTTP monitors", "hx_url": "/status/widget", - "detail_url": "/status", + "detail_url": "/status/", "plugin": None, "poll": True, "params": [], diff --git a/steward/templates/status/widget.html b/steward/templates/status/widget.html index eef40b8..2acc92f 100644 --- a/steward/templates/status/widget.html +++ b/steward/templates/status/widget.html @@ -7,7 +7,7 @@ {% set problems = entries | rejectattr("status", "equalto", "up") | list %} {% if not entries %} -
No monitors configured.
+

No monitors configured.

{% elif not problems %}
✓ All {{ up }} monitors are up.
{% else %} From 2594ca517d0e0b945f774d92390289bd1ddcaadd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 12:52:24 -0400 Subject: [PATCH 068/126] fix(ansible): clear error when the managed SSH key can't be decrypted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When ansible.ssh_private_key can't be decrypted (app secret key changed), decrypt_secret returns the ciphertext unchanged; the executor was writing that enc:v1: blob as the SSH key file → cryptic "Load key ...: error in libcrypto" → Permission denied. Now: - build_credentials skips a still-encrypted key value (never writes ciphertext as a key file). - start_run broadcasts a plain-language run error: "The managed SSH key could not be decrypted (the app secret key changed). Regenerate it in Settings → Ansible and re-provision the host(s)." The run still fails (no usable key), but the reason is now obvious instead of a libcrypto error. Operator remedy: regenerate the managed key + re-provision. Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/ansible/executor.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py index e061930..ac8667e 100644 --- a/steward/ansible/executor.py +++ b/steward/ansible/executor.py @@ -13,6 +13,8 @@ from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING +from steward.core.crypto import is_encrypted as _crypto_is_encrypted + if TYPE_CHECKING: from quart import Quart @@ -133,8 +135,12 @@ def build_credentials(creds: dict | None, tmpdir: str) -> tuple[list[str], list[ files: list[tuple[str, str]] = [] creds = creds or {} + from steward.core.crypto import is_encrypted key = creds.get("ssh_private_key") - if key: + # A still-encrypted value means decryption failed (app secret key changed) — + # writing the ciphertext as a key file just yields a cryptic libcrypto error, + # so skip it; start_run surfaces a clear message instead. + if key and not is_encrypted(key): path = os.path.join(tmpdir, "id_key") files.append((path, key if key.endswith("\n") else key + "\n")) args += ["--private-key", path] @@ -391,6 +397,11 @@ async def start_run( effective_creds = dict(creds) if conn.get("password"): effective_creds.pop("ssh_private_key", None) + elif _crypto_is_encrypted(effective_creds.get("ssh_private_key") or ""): + _broadcast(run_id, + "[run error] The managed SSH key could not be decrypted " + "(the app secret key changed). Regenerate it in " + "Settings → Ansible and re-provision the host(s).") cred_args, cred_files = build_credentials(effective_creds, tmpdir) boot_args, boot_files = build_bootstrap(conn, tmpdir) From 71715c38d8c53f27906b2f02cfe9fb0d492f59e4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 13:05:37 -0400 Subject: [PATCH 069/126] feat(auth): return to the original view after an auth-expiry redirect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a session has expired, require_role now bounces to /login?next= and sends the user back there after re-login. - middleware: safe_next_url() (same-site relative path/query only; rejects off-site, protocol-relative, javascript:, and the auth pages — no open redirect). _login_redirect() builds the next param; for HTMX requests it uses HX-Current-URL (the page, not the fragment) and HX-Redirect so the whole browser navigates instead of swapping the login page into a fragment. - login GET/POST carry `next` (hidden field), validated, used on success for local + LDAP; OIDC stashes it in session across the IdP round-trip. - login.html: hidden next field + next on the SSO link. - tests for safe_next_url. Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/auth/middleware.py | 53 +++++++++++++++++++++++++++++-- steward/auth/routes.py | 15 ++++++--- steward/templates/auth/login.html | 3 +- tests/test_safe_next_url.py | 39 +++++++++++++++++++++++ 4 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 tests/test_safe_next_url.py diff --git a/steward/auth/middleware.py b/steward/auth/middleware.py index 6aca001..e1b06c9 100644 --- a/steward/auth/middleware.py +++ b/steward/auth/middleware.py @@ -1,16 +1,65 @@ from __future__ import annotations import functools -from quart import session, redirect, url_for, abort, g +from urllib.parse import quote, urlsplit +from quart import session, redirect, url_for, abort, g, request, Response from steward.models.users import UserRole _ROLE_ORDER = [UserRole.viewer, UserRole.operator, UserRole.admin] +# Paths we never bounce back to after login (would loop or make no sense). +_NO_RETURN = {"/login", "/logout", "/setup"} + def role_meets(user_role: UserRole, minimum_role: UserRole) -> bool: """True when user_role is at least minimum_role in the viewer= _ROLE_ORDER.index(minimum_role) +def safe_next_url(raw: str | None) -> str | None: + """Return a safe same-site relative path+query from `raw`, else None. + + Accepts a relative path or a full URL (e.g. HTMX's HX-Current-URL) and + reduces it to path[?query]. Rejects off-site/protocol-relative targets and + the auth pages themselves, so it can't be abused as an open redirect. + """ + if not raw: + return None + parts = urlsplit(raw) + # Protocol-relative (//host/…) has a netloc but no scheme — reject it; a full + # same-site URL (HX-Current-URL) has a scheme and we keep only its path. + if parts.netloc and not parts.scheme: + return None + path = parts.path + if parts.query: + path += "?" + parts.query + if not path.startswith("/") or path.startswith("//") or path.startswith("/\\"): + return None + if path.split("?", 1)[0] in _NO_RETURN: + return None + return path + + +def _login_redirect(): + """Redirect an unauthenticated request to /login, preserving where to return. + + For HTMX requests the destination is the page the user is on (HX-Current-URL) + and we use HX-Redirect so the whole browser navigates (not a fragment swap); + for normal requests it's the requested path. + """ + is_htmx = request.headers.get("HX-Request") == "true" + nxt = safe_next_url(request.headers.get("HX-Current-URL")) if is_htmx else None + if nxt is None: + nxt = safe_next_url(request.full_path) + login_url = url_for("auth.login") + if nxt: + login_url = f"{login_url}?next={quote(nxt, safe='/')}" + if is_htmx: + resp = Response("", status=401) + resp.headers["HX-Redirect"] = login_url + return resp + return redirect(login_url) + + def require_role(minimum_role: UserRole): """Decorator: requires authenticated user with at least minimum_role. Also allows access for validated share-token requests (viewer level only). @@ -23,7 +72,7 @@ def require_role(minimum_role: UserRole): return await f(*args, **kwargs) user_id = session.get("user_id") if not user_id: - return redirect(url_for("auth.login")) + return _login_redirect() user_role = UserRole(session.get("user_role", "viewer")) if _ROLE_ORDER.index(user_role) < _ROLE_ORDER.index(minimum_role): abort(403) diff --git a/steward/auth/routes.py b/steward/auth/routes.py index f1cd510..399b0e9 100644 --- a/steward/auth/routes.py +++ b/steward/auth/routes.py @@ -2,7 +2,7 @@ from __future__ import annotations import bcrypt from quart import Blueprint, render_template, request, redirect, url_for, session from sqlalchemy import select, func -from steward.auth.middleware import login_user, logout_user +from steward.auth.middleware import login_user, logout_user, safe_next_url from steward.models.users import User, UserRole auth_bp = Blueprint("auth", __name__) @@ -54,7 +54,8 @@ async def login(): return redirect(url_for("auth.setup")) oidc_cfg = current_app.config.get("OIDC", {}) return await render_template("auth/login.html", - oidc_enabled=oidc_cfg.get("enabled", False)) + oidc_enabled=oidc_cfg.get("enabled", False), + next_url=safe_next_url(request.args.get("next"))) @auth_bp.post("/login") @@ -65,6 +66,7 @@ async def login_post(): username = form.get("username", "").strip() password_str = form.get("password", "") password = password_str.encode() + dest = safe_next_url(form.get("next")) or url_for("dashboard.index") # Try local auth first user = await get_user_by_username(current_app, username) @@ -72,7 +74,7 @@ async def login_post(): login_user(user) await log_audit(current_app, user.id, user.username, "auth.login", detail={"method": "local", "role": user.role.value}) - return redirect(url_for("dashboard.index")) + return redirect(dest) # Try LDAP fallback if enabled ldap_cfg = current_app.config.get("LDAP", {}) @@ -86,13 +88,14 @@ async def login_post(): login_user(user) await log_audit(current_app, user.id, user.username, "auth.login", detail={"method": "ldap", "role": user.role.value}) - return redirect(url_for("dashboard.index")) + return redirect(dest) oidc_cfg = current_app.config.get("OIDC", {}) return await render_template( "auth/login.html", error="Invalid credentials", oidc_enabled=oidc_cfg.get("enabled", False), + next_url=safe_next_url(form.get("next")), ), 400 @@ -112,6 +115,7 @@ async def login_oidc(): nonce = generate_state() session["oidc_state"] = state session["oidc_nonce"] = nonce + session["oidc_next"] = safe_next_url(request.args.get("next")) redirect_uri = url_for("auth.login_oidc_callback", _external=True) url = build_authorize_url( doc, oidc_cfg["client_id"], redirect_uri, @@ -170,7 +174,8 @@ async def login_oidc_callback(): login_user(user) await log_audit(current_app, user.id, user.username, "auth.login", detail={"method": "oidc", "role": role}) - return redirect(url_for("dashboard.index")) + dest = safe_next_url(session.pop("oidc_next", None)) or url_for("dashboard.index") + return redirect(dest) @auth_bp.get("/logout") diff --git a/steward/templates/auth/login.html b/steward/templates/auth/login.html index 19c2745..a94f7e5 100644 --- a/steward/templates/auth/login.html +++ b/steward/templates/auth/login.html @@ -11,7 +11,7 @@
{% endif %} {% if oidc_enabled %} - + Sign in with SSO
@@ -21,6 +21,7 @@
{% endif %}
+ {% if next_url %}{% endif %}
diff --git a/tests/test_safe_next_url.py b/tests/test_safe_next_url.py new file mode 100644 index 0000000..4b6b9cd --- /dev/null +++ b/tests/test_safe_next_url.py @@ -0,0 +1,39 @@ +"""Unit tests for safe_next_url — the auth post-login return-path guard.""" +from steward.auth.middleware import safe_next_url + + +def test_relative_path_passes(): + assert safe_next_url("/hosts/") == "/hosts/" + assert safe_next_url("/hosts/abc?range=24h") == "/hosts/abc?range=24h" + + +def test_full_url_reduced_to_path(): + # HTMX HX-Current-URL is absolute; we keep only the same-site path+query. + assert safe_next_url("http://steward.local/d/1") == "/d/1" + assert safe_next_url("https://steward.local/hosts/x?a=1") == "/hosts/x?a=1" + + +def test_offsite_host_is_dropped_not_redirected(): + # A crafted absolute URL can't become an open redirect — netloc is discarded. + assert safe_next_url("http://evil.com/phish") == "/phish" + + +def test_protocol_relative_rejected(): + assert safe_next_url("//evil.com/x") is None + assert safe_next_url("/\\evil.com") is None + + +def test_scheme_only_rejected(): + assert safe_next_url("javascript:alert(1)") is None + + +def test_auth_pages_not_returned_to(): + assert safe_next_url("/login") is None + assert safe_next_url("/login?next=/x") is None + assert safe_next_url("/logout") is None + assert safe_next_url("/setup") is None + + +def test_empty_and_none(): + assert safe_next_url(None) is None + assert safe_next_url("") is None From 88afad9de46159a2b0dfbb8206e7748c5da7ba90 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 13:08:35 -0400 Subject: [PATCH 070/126] feat(hosts): re-provision action on reporting hosts; Ansible off the edit page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Host agent panel (reporting state) gains a "Re-provision" collapsible (admin + linked target + managed key): bootstrap user/password → provision.yml, which reinstalls the steward account + managed key + agent. This is the missing path after regenerating the managed key — Update alone can't fix a broken key. - Remove the Ansible sections (run-playbook + target link) from the host EDIT page — they were the stale free-text version and the wrong place. They live on the host detail hub (dropdown + discovered variables). Edit page now links to the host page; edit_host route simplified (no ansible fetch). Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/templates/panel.html | 23 +++++++ steward/hosts/routes.py | 28 +------- steward/templates/hosts/form.html | 87 ++----------------------- 3 files changed, 30 insertions(+), 108 deletions(-) diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html index 33fcd8d..9982ca3 100644 --- a/plugins/host_agent/templates/panel.html +++ b/plugins/host_agent/templates/panel.html @@ -71,6 +71,29 @@

{% endif %} + {% if session.user_role == 'admin' and ansible_available and target and managed_key_set %} +
+ Re-provision (reinstall the steward account + managed key, then the agent) +

+ Use after regenerating the managed key, or if SSH auth as steward breaks. + Connects over a one-time bootstrap user + password (not stored). +

+ + +
+ + +
+
+ + +
+ + +
+ {% endif %} + {% else %} {# ── Not reporting: pending (token minted, no check-in) or never installed ── #} {% if reg %} diff --git a/steward/hosts/routes.py b/steward/hosts/routes.py index 804ffda..f0c3a2d 100644 --- a/steward/hosts/routes.py +++ b/steward/hosts/routes.py @@ -304,38 +304,16 @@ async def create_host(): @hosts_bp.get("//edit") @require_role(UserRole.operator) async def edit_host(host_id: str): - from steward.models.ansible_inventory import AnsibleTarget - from sqlalchemy.orm import selectinload - + # Pure host config (name/address/monitors). Agent, Ansible target linking, + # and playbook runs live on the host detail page (the hub), not here. async with current_app.db_sessionmaker() as db: result = await db.execute(select(Host).where(Host.id == host_id)) host = result.scalar_one_or_none() if host is None: return "Not found", 404 - linked_result = await db.execute( - select(AnsibleTarget) - .where(AnsibleTarget.host_id == host_id) - .options(selectinload(AnsibleTarget.groups)) - ) - linked_target = linked_result.scalar_one_or_none() - - # Only show targets not yet linked to any host (plus the currently linked one) - linkable_result = await db.execute( - select(AnsibleTarget) - .where(AnsibleTarget.host_id.is_(None)) - .order_by(AnsibleTarget.name) - ) - linkable_targets = linkable_result.scalars().all() - return await render_template( - "hosts/form.html", - host=host, - probe_types=list(ProbeType), - ansible_sources=_ansible_source_names(), - linked_target=linked_target, - linkable_targets=linkable_targets, - ) + "hosts/form.html", host=host, probe_types=list(ProbeType)) @hosts_bp.post("/") diff --git a/steward/templates/hosts/form.html b/steward/templates/hosts/form.html index e39507c..d6124b7 100644 --- a/steward/templates/hosts/form.html +++ b/steward/templates/hosts/form.html @@ -51,90 +51,11 @@
- - {% if host and ansible_sources %} -
-

Run Ansible playbook against this host

-

- Runs against an ephemeral one-host inventory for - {{ host.name }} ({{ host.address or "no address — resolves by name" }}). - Use a playbook with hosts: all. -

-
-
- - -
-
- - -
-
- - -
-
- - -
-
- -
- -
-
- {% endif %} -
- -{% if host %} -
-

Ansible Target

- {% if linked_target %} -
-
- {{ linked_target.name }} - {{ linked_target.address }} - {% if linked_target.groups %} -
- {% for grp in linked_target.groups %} - {{ grp.name }} - {% endfor %} -
- {% endif %} -
- Edit Target -
- - -
-
- {% else %} -

- No Ansible target linked. Link an existing target or create one from this host. + {% if host %} +

+ Agent, Ansible target, and playbook runs live on the + host page.

-
- {% if linkable_targets %} -
- - - -
- {% endif %} -
- - -
-
{% endif %}
-{% endif %} {% endblock %} From 2ea8c2f9afb57206ba61ef4aa89d30198269b2ab Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 14:15:33 -0400 Subject: [PATCH 071/126] feat(ansible): bundled system_update maintenance playbook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New maintenance/system_update.yml: cross-distro OS package upgrade (apt + dnf/yum) with two run-form flags: - restart_services: restart every service that needs it after the upgrade (Debian via needrestart -r a, installed if missing; RHEL via needs-restarting -s → systemctl try-restart). - reboot_if_required: reboot the host only when a reboot is actually pending (Debian /var/run/reboot-required; RHEL needs-restarting -r). Never reboots otherwise. Tagged steward:category=maintenance and steward:confirm=true (significant / reboot-capable), so it shows in the run UI with the confirm gate and both flags as fill-in fields. No code changes — auto-discovered from the builtin source. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../bundled/maintenance/system_update.yml | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 steward/ansible/bundled/maintenance/system_update.yml diff --git a/steward/ansible/bundled/maintenance/system_update.yml b/steward/ansible/bundled/maintenance/system_update.yml new file mode 100644 index 0000000..f94bebe --- /dev/null +++ b/steward/ansible/bundled/maintenance/system_update.yml @@ -0,0 +1,82 @@ +--- +# description: Update all OS packages; optionally restart affected services and reboot if one is pending. +# steward:category: maintenance +# steward:confirm: true +# +# Cross-distro (apt / dnf-yum). Flags (set in the run form): +# restart_services=true after the upgrade, restart every service that needs +# it (Debian: needrestart -r a; installed if missing). +# reboot_if_required=true reboot the host only if the update flagged a pending +# reboot (Debian: /var/run/reboot-required; +# RHEL: needs-restarting -r). Never reboots otherwise. +- name: System package update + hosts: all + become: true + gather_facts: true + vars: + restart_services: false + reboot_if_required: false + tasks: + # ── Debian / Ubuntu ────────────────────────────────────────────────────── + - name: Upgrade all packages (apt) + ansible.builtin.apt: + update_cache: true + upgrade: dist + autoremove: true + when: ansible_pkg_mgr == "apt" + + - name: Ensure needrestart is present (apt, for service restarts) + ansible.builtin.apt: + name: needrestart + state: present + when: + - restart_services | bool + - ansible_pkg_mgr == "apt" + + - name: Restart services that need it (apt / needrestart) + ansible.builtin.command: needrestart -r a + register: needrestart_run + changed_when: needrestart_run.rc == 0 + failed_when: false + when: + - restart_services | bool + - ansible_pkg_mgr == "apt" + + # ── RHEL / Fedora ───────────────────────────────────────────────────────── + - name: Upgrade all packages (dnf/yum) + ansible.builtin.dnf: + name: "*" + state: latest + when: ansible_pkg_mgr in ("dnf", "yum") + + - name: Restart services that need it (dnf / needs-restarting) + ansible.builtin.shell: needs-restarting -s | xargs -r -n1 systemctl try-restart + register: rhel_restart + changed_when: rhel_restart.rc == 0 + failed_when: false + when: + - restart_services | bool + - ansible_pkg_mgr in ("dnf", "yum") + + # ── Reboot only if one is pending ───────────────────────────────────────── + - name: Check pending reboot (Debian) + ansible.builtin.stat: + path: /var/run/reboot-required + register: deb_reboot + when: ansible_pkg_mgr == "apt" + + - name: Check pending reboot (RHEL) + ansible.builtin.command: needs-restarting -r + register: rhel_reboot + changed_when: false + failed_when: false + when: ansible_pkg_mgr in ("dnf", "yum") + + - name: Reboot if required + ansible.builtin.reboot: + msg: "Reboot triggered by Steward system_update" + reboot_timeout: 900 + when: + - reboot_if_required | bool + - (ansible_pkg_mgr == "apt" and deb_reboot.stat.exists) + or (ansible_pkg_mgr in ("dnf", "yum") and (rhel_reboot.rc | default(0)) != 0) From 5f92340c0c7227ae8f945be6a4801f963248704c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 14:49:49 -0400 Subject: [PATCH 072/126] =?UTF-8?q?feat(dashboard):=20phase=20A=20widget?= =?UTF-8?q?=20clarity=20=E2=80=94=20threshold=20colours,=20host=20links,?= =?UTF-8?q?=20tooltips?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Milestone 72 phase A (clarity wins, no schema change): - Add shared metric_style() macro in _macros.html: colours a numeric metric amber (>=warn) / red (>=crit) on the value ITSELF, not just the status dot. Defaults 80/90 for percentage gauges; load /core uses warn 80 / crit 100. Applied to CPU/mem/disk across host_agent panel + fleet widget + the unified hosts-overview widget. - Link every host reference to the host hub (/hosts/) in ping, dns, hosts-overview, host_agent fleet, and uptime widgets; status widget entries link to their own detail_url. All guarded on session.user_id so the public share view degrades to plain text (the bare href carries no share token). - Fix truncation: title= tooltips on all names that ellipsis, plus a hover underline affordance on the now-clickable ping-name links. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/templates/panel.html | 10 ++++++---- .../host_agent/templates/widget_table.html | 17 ++++++++++------- steward/templates/_macros.html | 13 +++++++++++++ steward/templates/base.html | 1 + steward/templates/dns/rows.html | 8 +++++++- steward/templates/hosts/overview_widget.html | 19 +++++++++++-------- steward/templates/hosts/uptime_widget.html | 7 ++++++- steward/templates/ping/rows.html | 8 +++++++- steward/templates/status/widget.html | 8 +++++++- 9 files changed, 68 insertions(+), 23 deletions(-) diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html index 9982ca3..d85904d 100644 --- a/plugins/host_agent/templates/panel.html +++ b/plugins/host_agent/templates/panel.html @@ -1,4 +1,5 @@ {# Per-host agent panel — embedded into /hosts/ via HTMX. Self-contained. #} +{% from "_macros.html" import metric_style %} {% set scope = "steward:target:" ~ target.id if target else "" %}
@@ -21,19 +22,20 @@
CPU
-
{{ '%.0f%%'|format(cpu) if cpu is not none else '—' }}
+
{{ '%.0f%%'|format(cpu) if cpu is not none else '—' }}
{{ sparks.cpu | safe }}
Memory
-
{{ '%.0f%%'|format(mem) if mem is not none else '—' }}
+
{{ '%.0f%%'|format(mem) if mem is not none else '—' }}
{{ sparks.mem | safe }}
Disk /
-
{{ '%.0f%%'|format(disk_root) if disk_root is not none else '—' }}
+
{{ '%.0f%%'|format(disk_root) if disk_root is not none else '—' }}
{{ sparks.disk | safe }}
+ {# Load /core: 100% = run queue matches CPU capacity, so warn 80 / crit 100. #}
Load /core
-
{{ '%d%%'|format(load_per_core) if load_per_core is not none else ('%.2f'|format(load1) if load1 is not none else '—') }}
+
{{ '%d%%'|format(load_per_core) if load_per_core is not none else ('%.2f'|format(load1) if load1 is not none else '—') }}
{{ sparks.load | safe }}
{% if psi.cpu is not none or psi.mem is not none or psi.io is not none %} diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html index 14a5bbd..df3f489 100644 --- a/plugins/host_agent/templates/widget_table.html +++ b/plugins/host_agent/templates/widget_table.html @@ -1,4 +1,5 @@ {# host_agent widget — fleet glance, flex rows (no header wrap) #} +{% from "_macros.html" import metric_style %} {% if rows %}
{% for r in rows %} @@ -6,19 +7,21 @@
- - {{ r.host.name }} - + {% set name_style = "flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-weight:500;" %} + {% if session.user_id %} + {{ r.host.name }} + {% else %} + {{ r.host.name }} + {% endif %} {% if r.cpu_pct is not none %} - cpu {{ "%.0f"|format(r.cpu_pct) }}% + cpu {{ "%.0f"|format(r.cpu_pct) }}% {% endif %} {% if r.mem_used_pct is not none %} - mem {{ "%.0f"|format(r.mem_used_pct) }}% + mem {{ "%.0f"|format(r.mem_used_pct) }}% {% endif %} {% if r.disk_root is not none %} - disk / {{ "%.0f"|format(r.disk_root) }}% + disk / {{ "%.0f"|format(r.disk_root) }}% {% endif %} {% if r.load_1m is not none %} load {{ "%.2f"|format(r.load_1m) }} diff --git a/steward/templates/_macros.html b/steward/templates/_macros.html index 4cb4b16..2867b70 100644 --- a/steward/templates/_macros.html +++ b/steward/templates/_macros.html @@ -16,3 +16,16 @@ {%- endfor -%} {%- endmacro %} + +{# Threshold emphasis for a numeric metric (CPU/mem/disk %, etc). + Returns an inline-style fragment to drop into a span's style="": amber + + semibold at >= warn, red + bold at >= crit, empty (inherit) when the value + is normal or None. This makes the FAILING METRIC ITSELF stand out — not just + the status dot. Defaults suit percentage gauges (warn 80 / crit 90); pass + explicit warn/crit for other scales. Keep thresholds here so they live in one + place rather than hardcoded per template. #} +{% macro metric_style(value, warn=80, crit=90) -%} +{%- if value is not none and value >= crit -%}color:var(--red);font-weight:700; +{%- elif value is not none and value >= warn -%}color:var(--yellow);font-weight:600; +{%- endif -%} +{%- endmacro %} diff --git a/steward/templates/base.html b/steward/templates/base.html index 49dff1f..84a83f8 100644 --- a/steward/templates/base.html +++ b/steward/templates/base.html @@ -164,6 +164,7 @@ textarea { resize: vertical; } .ping-row:last-child { border-bottom:none; } .ping-meta { min-width:120px; max-width:180px; flex-shrink:1; overflow:hidden; } .ping-name { font-size:0.875rem; font-weight:500; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; display:block; } +a.ping-name:hover { text-decoration:underline; } .ping-addr { display:block; color:var(--text-muted); font-size:0.75rem; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; } .ping-pills { display:flex; gap:2px; flex:1; min-width:0; overflow:hidden; align-items:center; } .pill { display:inline-block; width:7px; height:22px; border-radius:2px; cursor:default; transition:opacity .1s; } diff --git a/steward/templates/dns/rows.html b/steward/templates/dns/rows.html index 1aa4a7c..6df4664 100644 --- a/steward/templates/dns/rows.html +++ b/steward/templates/dns/rows.html @@ -3,7 +3,13 @@ {% set d = latest.get(host.id) %}
- {{ host.name }} + {# Link to the host hub for logged-in users; plain text on the public share view (no auth, no token on the href). #} + {% if session.user_id %} + {{ host.name }} + {% else %} + {{ host.name }} + {% endif %} {{ host.address }}
diff --git a/steward/templates/hosts/overview_widget.html b/steward/templates/hosts/overview_widget.html index 6cb2323..6e4ebf8 100644 --- a/steward/templates/hosts/overview_widget.html +++ b/steward/templates/hosts/overview_widget.html @@ -1,4 +1,5 @@ {# Unified Hosts widget — monitor status + agent glance per host, links to the hub. #} +{% from "_macros.html" import metric_style %} {% if hosts %}
{% for host in hosts %} @@ -10,10 +11,12 @@
- - {{ host.name }} - + {% set name_style = "flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-weight:500;" %} + {% if session.user_id %} + {{ host.name }} + {% else %} + {{ host.name }} + {% endif %} {# Monitors #} {% if host.ping_enabled and ping and ping.response_time_ms is not none %} @@ -22,11 +25,11 @@ {% if ut.get('24h') is not none %} 24h {{ "%.1f"|format(ut['24h']) }}% {% endif %} - {# Agent glance #} + {# Agent glance — failing metric coloured amber/red (≥80/≥90) #} {% if a %} - {% if a.get('cpu_pct') is not none %}cpu {{ "%.0f"|format(a.cpu_pct) }}%{% endif %} - {% if a.get('mem_used_pct') is not none %}mem {{ "%.0f"|format(a.mem_used_pct) }}%{% endif %} - {% if a.get('disk_root') is not none %}disk / {{ "%.0f"|format(a.disk_root) }}%{% endif %} + {% if a.get('cpu_pct') is not none %}cpu {{ "%.0f"|format(a.cpu_pct) }}%{% endif %} + {% if a.get('mem_used_pct') is not none %}mem {{ "%.0f"|format(a.mem_used_pct) }}%{% endif %} + {% if a.get('disk_root') is not none %}disk / {{ "%.0f"|format(a.disk_root) }}%{% endif %} {% if not a.fresh %}stale{% endif %} {% else %} no agent diff --git a/steward/templates/hosts/uptime_widget.html b/steward/templates/hosts/uptime_widget.html index 0f4df37..50b34ed 100644 --- a/steward/templates/hosts/uptime_widget.html +++ b/steward/templates/hosts/uptime_widget.html @@ -15,7 +15,12 @@ {% for host in hosts %} {% set ut = uptime.get(host.id, {}) %}
- {{ host.name }} + {% if session.user_id %} + {{ host.name }} + {% else %} + {{ host.name }} + {% endif %} {% for window in ["24h", "7d", "30d"] %} {% set pct = ut.get(window) %} diff --git a/steward/templates/ping/rows.html b/steward/templates/ping/rows.html index 1adcd88..c20ee06 100644 --- a/steward/templates/ping/rows.html +++ b/steward/templates/ping/rows.html @@ -22,7 +22,13 @@ {% set last = host_pings[-1] if host_pings else none %}
- {{ host.name }} + {# Link to the host hub for logged-in users; plain text on the public share view (no auth, no token on the href). #} + {% if session.user_id %} + {{ host.name }} + {% else %} + {{ host.name }} + {% endif %} {{ host.address }}
diff --git a/steward/templates/status/widget.html b/steward/templates/status/widget.html index 2acc92f..486e104 100644 --- a/steward/templates/status/widget.html +++ b/steward/templates/status/widget.html @@ -15,7 +15,13 @@ {% for e in problems[:8] %}
- {{ e.name }} + {% set nm_style = "flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" %} + {# Link to the monitor's own detail view for logged-in users; plain text on the public share view. #} + {% if session.user_id and e.detail_url %} + {{ e.name }} + {% else %} + {{ e.name }} + {% endif %} {{ e.kind }} {{ e.status }}
From 525f6eedbdcbc71822b69c28b309a89edbda9924 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 15:04:00 -0400 Subject: [PATCH 073/126] feat(dashboard): phase B drag-resize grid (Gridstack) replacing masonry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Milestone 72 phase B — Grafana-style drag-resize grid for dashboard widgets: - DashboardWidget: replace `position` with a 12-col grid placement (grid_x/grid_y/grid_w/grid_h). Migration 0021 backfills the old position order into a 3-up grid (4 cols x 4 cells each) and drops position. - View + share render a static CSS grid from x/y/w/h: fixed cell height (h * 70px) with the body scrolling, so the arranged layout is what's shown; collapses to a single column under 820px. - Edit view: Gridstack.js 12.6.0 (vanilla, CDN, pinned) — drag the title bar to move, drag a corner/edge to resize; every change autosaves to a new /d//edit/layout endpoint. Replaces the SortableJS position reorder. - add_widget appends at the bottom-left; remove no longer renumbers. _get_widgets now orders by grid position (drives DOM + mobile fallback order). Note: Gridstack drag-resize is browser-only, so CI can't exercise it — needs an operator visual check of the edit experience. Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/dashboard/routes.py | 50 +++++++++------ .../versions/0021_dashboard_widget_grid.py | 48 +++++++++++++++ steward/models/dashboard.py | 7 ++- steward/templates/base.html | 12 ++++ steward/templates/dashboard/_edit_panels.html | 43 +++++++------ steward/templates/dashboard/edit.html | 61 +++++++++++-------- steward/templates/dashboard/index.html | 10 +-- steward/templates/dashboard/share_view.html | 19 ++++-- 8 files changed, 177 insertions(+), 73 deletions(-) create mode 100644 steward/migrations/versions/0021_dashboard_widget_grid.py diff --git a/steward/dashboard/routes.py b/steward/dashboard/routes.py index b9a3da1..8802b04 100644 --- a/steward/dashboard/routes.py +++ b/steward/dashboard/routes.py @@ -38,19 +38,16 @@ def _can_edit(dash: Dashboard, user_id: str, user_role: str) -> bool: # ── DB helpers ──────────────────────────────────────────────────────────────── async def _get_widgets(db, dashboard_id: int) -> list[DashboardWidget]: + # Reading order = grid order (top-to-bottom, then left-to-right). This drives + # DOM order, which is what the responsive single-column fallback collapses to. result = await db.execute( select(DashboardWidget) .where(DashboardWidget.dashboard_id == dashboard_id) - .order_by(DashboardWidget.position) + .order_by(DashboardWidget.grid_y, DashboardWidget.grid_x, DashboardWidget.id) ) return list(result.scalars()) -async def _normalise_positions(db, dashboard_id: int) -> None: - for i, w in enumerate(await _get_widgets(db, dashboard_id)): - w.position = i - - async def _accessible_dashboards(db, user_id: str, user_role: str) -> list[Dashboard]: """All dashboards visible to this user, ordered by id.""" result = await db.execute(select(Dashboard).order_by(Dashboard.id)) @@ -97,9 +94,11 @@ async def _get_or_create_system_default(db) -> Dashboard: db.add(dash) await db.flush() plugins_cfg = current_app.config.get("PLUGINS", {}) + # Seed a 3-up grid (each widget 4 cols wide x 4 cells tall on the 12-col grid). for pos, w in enumerate(get_available_widgets(plugins_cfg)): db.add(DashboardWidget( - dashboard_id=dash.id, widget_key=w["key"], position=pos, + dashboard_id=dash.id, widget_key=w["key"], + grid_x=(pos % 3) * 4, grid_y=(pos // 3) * 4, grid_w=4, grid_h=4, )) return dash @@ -408,10 +407,12 @@ async def add_widget(dash_id: int, widget_key: str): dash = result.scalar_one_or_none() if dash is None or not _can_edit(dash, user_id, user_role): abort(403) - widgets = await _get_widgets(db, dash_id) - next_pos = max((w.position for w in widgets), default=-1) + 1 + widgets = await _get_widgets(db, dash_id) + # Append below everything at full-left; the user then drags/resizes. + next_y = max((w.grid_y + w.grid_h for w in widgets), default=0) db.add(DashboardWidget( - dashboard_id=dash_id, widget_key=widget_key, position=next_pos, + dashboard_id=dash_id, widget_key=widget_key, + grid_x=0, grid_y=next_y, grid_w=4, grid_h=4, title=title, config_json=config_json, )) return await _panels_response(dash_id) @@ -434,30 +435,39 @@ async def remove_widget(dash_id: int, widget_id: int): widget = result.scalar_one_or_none() if widget and widget.dashboard_id == dash_id: await db.delete(widget) - await _normalise_positions(db, dash_id) return await _panels_response(dash_id) -@dashboard_bp.post("/d//edit/reorder") +@dashboard_bp.post("/d//edit/layout") @require_role(UserRole.viewer) -async def reorder_widgets(dash_id: int): +async def save_layout(dash_id: int): + """Persist the drag-resize grid (Gridstack) — one {id,x,y,w,h} per widget.""" user_id = current_user_id() user_role = session.get("user_role", "viewer") data = await request.get_json(silent=True) - if not data or not isinstance(data.get("order"), list): + if not data or not isinstance(data.get("items"), list): abort(400) - order: list[int] = data["order"] async with current_app.db_sessionmaker() as db: async with db.begin(): result = await db.execute(select(Dashboard).where(Dashboard.id == dash_id)) dash = result.scalar_one_or_none() if dash is None or not _can_edit(dash, user_id, user_role): abort(403) - widgets = await _get_widgets(db, dash_id) - widget_map = {w.id: w for w in widgets} - for pos, wid in enumerate(order): - if wid in widget_map: - widget_map[wid].position = pos + widget_map = {w.id: w for w in await _get_widgets(db, dash_id)} + for it in data["items"]: + if not isinstance(it, dict): + continue + w = widget_map.get(it.get("id")) + if w is None: + continue + try: + # Clamp to the 12-col grid; widths can't exceed it. + w.grid_x = max(0, min(11, int(it["x"]))) + w.grid_w = max(1, min(12, int(it["w"]))) + w.grid_y = max(0, int(it["y"])) + w.grid_h = max(1, int(it["h"])) + except (KeyError, TypeError, ValueError): + continue return ("", 204) diff --git a/steward/migrations/versions/0021_dashboard_widget_grid.py b/steward/migrations/versions/0021_dashboard_widget_grid.py new file mode 100644 index 0000000..252b256 --- /dev/null +++ b/steward/migrations/versions/0021_dashboard_widget_grid.py @@ -0,0 +1,48 @@ +"""Replace dashboard_widgets.position with a Grafana-style grid (x/y/w/h) + +Revision ID: 0021_dashboard_widget_grid +Revises: 0020_ansible_run_results +Create Date: 2026-06-17 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "0021_dashboard_widget_grid" +down_revision: Union[str, None] = "0020_ansible_run_results" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # New grid columns (server_default so existing rows are populated; the ORM + # supplies these on insert thereafter). 12-col grid, default widget 4x4. + op.add_column("dashboard_widgets", + sa.Column("grid_x", sa.Integer, nullable=False, server_default="0")) + op.add_column("dashboard_widgets", + sa.Column("grid_y", sa.Integer, nullable=False, server_default="0")) + op.add_column("dashboard_widgets", + sa.Column("grid_w", sa.Integer, nullable=False, server_default="4")) + op.add_column("dashboard_widgets", + sa.Column("grid_h", sa.Integer, nullable=False, server_default="4")) + # Backfill from the old `position` order into a 3-up grid, reproducing the + # previous masonry layout: 3 widgets per row, each 4 cols wide x 4 cells tall. + op.execute( + "UPDATE dashboard_widgets " + "SET grid_x = (position % 3) * 4, grid_y = (position / 3) * 4" + ) + op.drop_column("dashboard_widgets", "position") + + +def downgrade() -> None: + op.add_column("dashboard_widgets", + sa.Column("position", sa.Integer, nullable=False, server_default="0")) + # Reconstruct a linear order from the grid (reading order: top-to-bottom, + # then left-to-right within a row). + op.execute( + "UPDATE dashboard_widgets SET position = grid_y * 3 + (grid_x / 4)" + ) + op.drop_column("dashboard_widgets", "grid_h") + op.drop_column("dashboard_widgets", "grid_w") + op.drop_column("dashboard_widgets", "grid_y") + op.drop_column("dashboard_widgets", "grid_x") diff --git a/steward/models/dashboard.py b/steward/models/dashboard.py index 34368c1..389b478 100644 --- a/steward/models/dashboard.py +++ b/steward/models/dashboard.py @@ -50,6 +50,11 @@ class DashboardWidget(Base): Integer, ForeignKey("dashboards.id", ondelete="CASCADE"), nullable=False, ) widget_key: Mapped[str] = mapped_column(String(64), nullable=False) - position: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + # Grid placement (Grafana-style): 12-column grid, row height in cell units. + # x/y are the top-left cell; w/h the span. Replaces the old `position` order. + grid_x: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + grid_y: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + grid_w: Mapped[int] = mapped_column(Integer, nullable=False, default=4) + grid_h: Mapped[int] = mapped_column(Integer, nullable=False, default=4) title: Mapped[str | None] = mapped_column(String(128), nullable=True) config_json: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/steward/templates/base.html b/steward/templates/base.html index 84a83f8..32cfb37 100644 --- a/steward/templates/base.html +++ b/steward/templates/base.html @@ -165,6 +165,18 @@ textarea { resize: vertical; } .ping-meta { min-width:120px; max-width:180px; flex-shrink:1; overflow:hidden; } .ping-name { font-size:0.875rem; font-weight:500; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; display:block; } a.ping-name:hover { text-decoration:underline; } +/* Dashboard widget grid — Grafana-style 12-col layout; cells sized by drag-resize + (Gridstack) in the edit view and rendered statically here from each widget's + x/y/w/h. Cell height is fixed (h * cell unit) with the body scrolling, so the + layout the operator arranged is exactly what they see. */ +.dash-grid { display:grid; grid-template-columns:repeat(12,1fr); grid-auto-rows:70px; gap:1rem; } +.dash-grid > .dash-cell { margin-bottom:0; min-width:0; overflow:hidden; display:flex; flex-direction:column; } +.dash-cell .dash-cell-body { flex:1; min-height:0; overflow:auto; } +@media (max-width:820px) { + .dash-grid { grid-template-columns:1fr; grid-auto-rows:auto; } + .dash-grid > .dash-cell { grid-column:1 / -1 !important; grid-row:auto !important; } + .dash-cell .dash-cell-body { overflow:visible; } +} .ping-addr { display:block; color:var(--text-muted); font-size:0.75rem; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; } .ping-pills { display:flex; gap:2px; flex:1; min-width:0; overflow:hidden; align-items:center; } .pill { display:inline-block; width:7px; height:22px; border-radius:2px; cursor:default; transition:opacity .1s; } diff --git a/steward/templates/dashboard/_edit_panels.html b/steward/templates/dashboard/_edit_panels.html index c9d63c9..fe12d49 100644 --- a/steward/templates/dashboard/_edit_panels.html +++ b/steward/templates/dashboard/_edit_panels.html @@ -1,34 +1,37 @@ {# dashboard/_edit_panels.html — returned by all mutation routes and included by edit.html #}
- {# ── Current layout ───────────────────────────────────────────────────── #} -
-
Current Layout
+ {# ── Current layout (drag to move, drag the corner/edge to resize) ──────── #} +
+
+
Current Layout
+ {% if widgets %} + Drag to move · drag a corner to resize · saved automatically + {% endif %} +
{% if widgets %} -
+
{% for item in widgets %} -
- -
⠿
- -
-
{{ item.title }}
-
{{ item.defn.description }}
+
+
+
+ ⠿ + {{ item.title }} + +
+
{{ item.defn.description }}
- - -
{% endfor %}
{% else %}
- No widgets yet. Draw one from the right to begin. + No widgets yet. Add one from the right to begin.
{% endif %}
diff --git a/steward/templates/dashboard/edit.html b/steward/templates/dashboard/edit.html index 9b68f2a..ad371c9 100644 --- a/steward/templates/dashboard/edit.html +++ b/steward/templates/dashboard/edit.html @@ -2,6 +2,9 @@ {% from "_macros.html" import crumbs %} {% block title %}Edit: {{ dashboard.name }} — Steward{% endblock %} {% block breadcrumb %}{{ crumbs([("Dashboard", "/"), ("Dashboards", "/dashboards/"), ("Edit " ~ dashboard.name, "")]) }}{% endblock %} +{% block head %} + +{% endblock %} {% block content %}

Edit: {{ dashboard.name }}

@@ -10,36 +13,46 @@ {% include "dashboard/_edit_panels.html" %} {% endblock %} {% block extra_scripts %} - + {% endblock %} diff --git a/steward/templates/dashboard/index.html b/steward/templates/dashboard/index.html index f03a073..317bb23 100644 --- a/steward/templates/dashboard/index.html +++ b/steward/templates/dashboard/index.html @@ -62,14 +62,16 @@ {# ── Widget grid ───────────────────────────────────────────────────────────── #} {% if widgets %} -
+
{% for item in widgets %} -
+ {% set gw = item.db %} +
- {{ item.title }} + {{ item.title }} Details →
-
diff --git a/steward/templates/dashboard/share_view.html b/steward/templates/dashboard/share_view.html index 4f1d37e..67d11c9 100644 --- a/steward/templates/dashboard/share_view.html +++ b/steward/templates/dashboard/share_view.html @@ -55,6 +55,15 @@ .ping-cur-nd { color:var(--text-dim); } .widget-header { display:flex; justify-content:space-between; align-items:center; margin-bottom:0.75rem; } .widget-title { font-family:var(--font-serif); font-size:0.8rem; font-weight:600; color:var(--text-muted); text-transform:uppercase; letter-spacing:0.06em; } + /* Same Grafana-style grid as the authenticated view, rendered from x/y/w/h. */ + .dash-grid { display:grid; grid-template-columns:repeat(12,1fr); grid-auto-rows:70px; gap:1rem; } + .dash-grid > .dash-cell { margin-bottom:0; min-width:0; overflow:hidden; display:flex; flex-direction:column; } + .dash-cell .dash-cell-body { flex:1; min-height:0; overflow:auto; } + @media (max-width:820px) { + .dash-grid { grid-template-columns:1fr; grid-auto-rows:auto; } + .dash-grid > .dash-cell { grid-column:1 / -1 !important; grid-row:auto !important; } + .dash-cell .dash-cell-body { overflow:visible; } + } .stat-val { font-size: 2rem; font-weight: 700; line-height: 1; } .stat-label { font-size: 0.8rem; color: var(--text-muted); margin-left: 0.3rem; } header { background: var(--bg-card); padding: 0 1.5rem; display: flex; align-items: center; justify-content: space-between; border-bottom: 1px solid var(--border-mid); height: 48px; font-family: var(--font-serif); } @@ -72,14 +81,16 @@
{% if widgets %} -
+
{% for item in widgets %} -
+ {% set gw = item.db %} +
- {{ item.title }} + {{ item.title }}
{# Pass share token as &s= param so require_role allows through #} -
From 988c13d51f243ebf7df6fbc3e8b2027532e7d858 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 15:18:50 -0400 Subject: [PATCH 074/126] =?UTF-8?q?feat(dashboard):=20phase=20C=20richer?= =?UTF-8?q?=20panels=20=E2=80=94=20host=20time-series=20graph=20+=20cpu=20?= =?UTF-8?q?sparklines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Milestone 72 phase C — bring the host-view graphs onto the dashboard: - host_resource_history widget reworked into a real host-view chart: epoch-ms linear axis (no Chart.js date adapter), themed like the host-detail charts, maintainAspectRatio:false so it fills the resized panel, unique canvas per widget instance (wid), and empty states ("pick a host" / "no metrics yet"). Was previously unusable — it had a broken time axis and no way to choose a host. - Add a "host" param type: the edit form renders a live dropdown of hosts (dashboard routes now pass the host list to the editor); the chosen host_id is stored in config and fed to the widget. - Hosts-overview widget gains a per-row CPU sparkline (last hour) via the shared sparkline_svg helper — the host-view at-a-glance trend, on the main widget. Charts/sparklines render only in the browser, so CI can't exercise them — needs an operator visual check. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 60 ++++++++------- .../host_agent/templates/widget_history.html | 75 +++++++++++++------ steward/core/widgets.py | 13 +++- steward/dashboard/routes.py | 14 ++-- steward/hosts/routes.py | 36 ++++++++- steward/templates/dashboard/_edit_panels.html | 10 +++ steward/templates/hosts/overview_widget.html | 2 + 7 files changed, 151 insertions(+), 59 deletions(-) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 57c92bc..8134ca9 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -387,34 +387,44 @@ async def widget_table(): @host_agent_bp.get("/widget/history") async def widget_history(): host_id = request.args.get("host_id", "") - hours = int(request.args.get("hours", "6")) + try: + hours = max(1, min(168, int(request.args.get("hours", "6")))) + except ValueError: + hours = 6 + # wid (the dashboard widget row id) keeps the id unique when several + # history widgets share a page; fall back to the host id. + wid = request.args.get("wid", "") or host_id cutoff = datetime.now(timezone.utc) - timedelta(hours=hours) - async with current_app.db_sessionmaker() as session: - host = (await session.execute( - select(Host).where(Host.id == host_id))).scalar_one_or_none() - if host is None: - return _error(404, "not_found") - points = (await session.execute( - select(PluginMetric).where( - PluginMetric.source_module == SOURCE_MODULE, - PluginMetric.recorded_at >= cutoff, - or_( - and_(PluginMetric.resource_name == host.name, - PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct"))), - and_(PluginMetric.resource_name == host.name + ":/", - PluginMetric.metric_name == "disk_used_pct"), - ), - ).order_by(PluginMetric.recorded_at) - )).scalars().all() + series: dict[str, list[list]] = {"cpu_pct": [], "mem_used_pct": [], "disk_root": []} + host = None + if host_id: + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is not None: + points = (await session.execute( + select(PluginMetric).where( + PluginMetric.source_module == SOURCE_MODULE, + PluginMetric.recorded_at >= cutoff, + or_( + and_(PluginMetric.resource_name == host.name, + PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct"))), + and_(PluginMetric.resource_name == host.name + ":/", + PluginMetric.metric_name == "disk_used_pct"), + ), + ).order_by(PluginMetric.recorded_at) + )).scalars().all() + # Disk = root (/), consistent with the host panel — not "worst". + # Epoch-ms x values let the chart use a plain linear axis (no + # Chart.js date adapter needed), matching the host-detail charts. + for p in points: + key = "disk_root" if p.resource_name != host.name else p.metric_name + series[key].append([int(p.recorded_at.timestamp() * 1000), round(p.value, 2)]) - # Disk = root (/), consistent with the host panel — not the opaque "worst". - series: dict[str, list[dict]] = {"cpu_pct": [], "mem_used_pct": [], "disk_root": []} - for p in points: - key = "disk_root" if p.resource_name != host.name else p.metric_name - series[key].append({"t": p.recorded_at.isoformat(), "v": p.value}) - - return await render_template("widget_history.html", host=host, series=series, hours=hours) + return await render_template( + "widget_history.html", host=host, series=series, hours=hours, wid=wid, + ) # Host-level metrics charted on the detail page (sub-resources are shown as diff --git a/plugins/host_agent/templates/widget_history.html b/plugins/host_agent/templates/widget_history.html index 0d5b567..cf9f492 100644 --- a/plugins/host_agent/templates/widget_history.html +++ b/plugins/host_agent/templates/widget_history.html @@ -1,24 +1,53 @@ -
-

{{ host.name }} — last {{ hours }}h

- - +{# host_agent history-graph widget — the host-view utilization chart, embedded + on a dashboard. Fills the (resizable) panel; epoch-ms linear axis so no + Chart.js date adapter is needed. #} +{% set has_data = host and (series.cpu_pct or series.mem_used_pct or series.disk_root) %} +{% if not host %} +
+ No host selected. Edit this dashboard and pick a host for this graph.
+{% elif not has_data %} +
+ No agent metrics for {{ host.name }} in the last {{ hours }}h yet. +
+{% else %} +
+ +
+ +{% endif %} diff --git a/steward/core/widgets.py b/steward/core/widgets.py index 7ea8671..a4f0349 100644 --- a/steward/core/widgets.py +++ b/steward/core/widgets.py @@ -279,13 +279,20 @@ WIDGET_REGISTRY: dict[str, dict] = { }, "host_resource_history": { "key": "host_resource_history", - "label": "Host Agent — History", - "description": "CPU/memory/disk history for one host", + "label": "Host Agent — History Graph", + "description": "CPU / memory / disk time-series graph for one host (the host-view chart, on your dashboard)", "hx_url": "/plugins/host_agent/widget/history", - "detail_url": "/plugins/host_agent/fleet/", + "detail_url": "/hosts/", "plugin": "host_agent", "poll": True, "params": [ + # type "host" → the edit form renders a live dropdown of hosts. + { + "key": "host_id", + "label": "Host", + "type": "host", + "default": "", + }, { "key": "hours", "label": "Time range", diff --git a/steward/dashboard/routes.py b/steward/dashboard/routes.py index 8802b04..57a82d4 100644 --- a/steward/dashboard/routes.py +++ b/steward/dashboard/routes.py @@ -173,7 +173,7 @@ async def _get_panels_data(db, dashboard_id: int) -> tuple: result = await db.execute(select(Dashboard).where(Dashboard.id == dashboard_id)) dash = result.scalar_one_or_none() if dash is None: - return None, [], [] + return None, [], [], [] widgets = await _get_widgets(db, dashboard_id) plugins_cfg = current_app.config.get("PLUGINS", {}) available = get_available_widgets(plugins_cfg) @@ -182,15 +182,17 @@ async def _get_panels_data(db, dashboard_id: int) -> tuple: {"db": w, "defn": WIDGET_REGISTRY[w.widget_key], "title": w.title or WIDGET_REGISTRY[w.widget_key]["label"]} for w in widgets if w.widget_key in WIDGET_REGISTRY ] - return dash, resolved, available + # Hosts feed the "host" param dropdown (e.g. the history-graph widget). + hosts = list((await db.execute(select(Host).order_by(Host.name))).scalars()) + return dash, resolved, available, hosts async def _panels_response(dashboard_id: int): async with current_app.db_sessionmaker() as db: - dash, resolved, addable = await _get_panels_data(db, dashboard_id) + dash, resolved, addable, hosts = await _get_panels_data(db, dashboard_id) return await render_template( "dashboard/_edit_panels.html", - dashboard=dash, widgets=resolved, addable=addable, + dashboard=dash, widgets=resolved, addable=addable, hosts=hosts, ) @@ -369,12 +371,12 @@ async def edit(dash_id: int): user_id = current_user_id() user_role = session.get("user_role", "viewer") async with current_app.db_sessionmaker() as db: - dash, resolved, addable = await _get_panels_data(db, dash_id) + dash, resolved, addable, hosts = await _get_panels_data(db, dash_id) if dash is None or not _can_edit(dash, user_id, user_role): abort(403) return await render_template( "dashboard/edit.html", - dashboard=dash, widgets=resolved, addable=addable, + dashboard=dash, widgets=resolved, addable=addable, hosts=hosts, ) diff --git a/steward/hosts/routes.py b/steward/hosts/routes.py index f0c3a2d..6917ccb 100644 --- a/steward/hosts/routes.py +++ b/steward/hosts/routes.py @@ -153,6 +153,36 @@ async def _agent_overview_by_host(db, host_names: list[str]) -> dict[str, dict]: return out +async def _agent_cpu_sparks_by_host(db, host_names: list[str], hours: int = 1) -> dict[str, str]: + """{host_name: inline-SVG cpu sparkline} over the last `hours` (core-safe). + + A tiny at-a-glance trend in each row — the host-view sparkline, on the widget. + """ + from steward.models.metrics import PluginMetric + from steward.core.status import sparkline_svg + if not host_names: + return {} + cutoff = datetime.now(timezone.utc) - timedelta(hours=hours) + rows = (await db.execute( + select(PluginMetric.resource_name, PluginMetric.value, PluginMetric.recorded_at) + .where( + PluginMetric.source_module == "host_agent", + PluginMetric.metric_name == "cpu_pct", + PluginMetric.resource_name.in_(host_names), + PluginMetric.recorded_at >= cutoff, + ) + .order_by(PluginMetric.recorded_at) + )).all() + by_host: dict[str, list[float]] = {} + for r in rows: + by_host.setdefault(r.resource_name, []).append(r.value) + # Amber line to read as "CPU"; only draw when there are a couple of points. + return { + name: sparkline_svg(vals[-40:], width=64, height=18, stroke="#c8a840") + for name, vals in by_host.items() if len(vals) >= 2 + } + + @hosts_bp.get("/overview/widget") @require_role(UserRole.viewer) async def overview_widget(): @@ -180,12 +210,14 @@ async def overview_widget(): & (DnsResult.resolved_at == latest_dns_subq.c.max_at), ))).scalars()} uptime = await _compute_uptime(db) - agent = await _agent_overview_by_host(db, [h.name for h in hosts]) + host_names = [h.name for h in hosts] + agent = await _agent_overview_by_host(db, host_names) + cpu_sparks = await _agent_cpu_sparks_by_host(db, host_names) return await render_template( "hosts/overview_widget.html", hosts=hosts, latest_pings=latest_pings, latest_dns=latest_dns, - uptime=uptime, agent=agent, + uptime=uptime, agent=agent, cpu_sparks=cpu_sparks, ) diff --git a/steward/templates/dashboard/_edit_panels.html b/steward/templates/dashboard/_edit_panels.html index fe12d49..2a410e6 100644 --- a/steward/templates/dashboard/_edit_panels.html +++ b/steward/templates/dashboard/_edit_panels.html @@ -92,6 +92,16 @@ {% endfor %} + {% elif p.type == "host" %} + {% endif %}
{% endfor %} diff --git a/steward/templates/hosts/overview_widget.html b/steward/templates/hosts/overview_widget.html index 6e4ebf8..8c2052b 100644 --- a/steward/templates/hosts/overview_widget.html +++ b/steward/templates/hosts/overview_widget.html @@ -17,6 +17,8 @@ {% else %} {{ host.name }} {% endif %} + {% set spark = cpu_sparks.get(host.name) %} + {% if spark %}{{ spark | safe }}{% endif %} {# Monitors #} {% if host.ping_enabled and ping and ping.response_time_ms is not none %} From 88dca32d3c81f14ffaa1211239e16e7bff066b96 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 15:42:10 -0400 Subject: [PATCH 075/126] feat(dashboard): edit in place over the live dashboard with a bottom widget drawer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Milestone 72 phase D — the dashboard view IS the edit surface (operator ask): - /d/ renders the grid via Gridstack in STATIC mode — positioned exactly like before, live HTMX widget bodies keep polling. An "Edit" button flips the same grid interactive (grid.setStatic(false)) in place, so you drag/resize over real data. "Done" flips it back. Layout autosaves on change. - The widget picker is now a bottom drawer (position:fixed overlay) revealed by body.dash-editing — so the dashboard width is identical entering/leaving edit. - Add: POST returns a single grid item; JS inserts it + grid.makeWidget + htmx.process so it loads live data. Remove: POST 204 + grid.removeWidget. Per-panel drag handle + remove ✕ are in the DOM for editors, shown only while editing. - Gridstack loads for everyone (viewers get the static grid); edit wiring is gated on can_edit. Mobile collapses to one column. - Removed the separate /d//edit route + edit.html + _edit_panels.html (rule 22). Dashboard-list Edit and new-dashboard create now deep-link /d/?edit=1 which opens edit mode on load. Browser-only behaviour — CI can't exercise it; needs an operator visual check. Co-Authored-By: Claude Opus 4.8 (1M context) --- steward/dashboard/routes.py | 81 +++++------ steward/templates/base.html | 32 +++-- steward/templates/dashboard/_edit_panels.html | 127 ------------------ steward/templates/dashboard/_grid_item.html | 25 ++++ .../templates/dashboard/_widget_drawer.html | 78 +++++++++++ steward/templates/dashboard/edit.html | 58 -------- steward/templates/dashboard/index.html | 122 +++++++++++++---- steward/templates/dashboard/list.html | 4 +- 8 files changed, 257 insertions(+), 270 deletions(-) delete mode 100644 steward/templates/dashboard/_edit_panels.html create mode 100644 steward/templates/dashboard/_grid_item.html create mode 100644 steward/templates/dashboard/_widget_drawer.html delete mode 100644 steward/templates/dashboard/edit.html diff --git a/steward/dashboard/routes.py b/steward/dashboard/routes.py index 57a82d4..ddf32a3 100644 --- a/steward/dashboard/routes.py +++ b/steward/dashboard/routes.py @@ -169,33 +169,6 @@ def _resolve_widgets(widgets: list[DashboardWidget]) -> list[dict]: # ── Edit panel helpers ──────────────────────────────────────────────────────── -async def _get_panels_data(db, dashboard_id: int) -> tuple: - result = await db.execute(select(Dashboard).where(Dashboard.id == dashboard_id)) - dash = result.scalar_one_or_none() - if dash is None: - return None, [], [], [] - widgets = await _get_widgets(db, dashboard_id) - plugins_cfg = current_app.config.get("PLUGINS", {}) - available = get_available_widgets(plugins_cfg) - # Multiple instances of the same widget type are allowed — show all available - resolved = [ - {"db": w, "defn": WIDGET_REGISTRY[w.widget_key], "title": w.title or WIDGET_REGISTRY[w.widget_key]["label"]} - for w in widgets if w.widget_key in WIDGET_REGISTRY - ] - # Hosts feed the "host" param dropdown (e.g. the history-graph widget). - hosts = list((await db.execute(select(Host).order_by(Host.name))).scalars()) - return dash, resolved, available, hosts - - -async def _panels_response(dashboard_id: int): - async with current_app.db_sessionmaker() as db: - dash, resolved, addable, hosts = await _get_panels_data(db, dashboard_id) - return await render_template( - "dashboard/_edit_panels.html", - dashboard=dash, widgets=resolved, addable=addable, hosts=hosts, - ) - - # ── Root redirect ───────────────────────────────────────────────────────────── @dashboard_bp.get("/") @@ -229,10 +202,18 @@ async def view(dash_id: int): stats = await _get_summary_stats(db) widgets = await _get_widgets(db, dash_id) accessible = await _accessible_dashboards(db, user_id, user_role) + can_edit = _can_edit(dash, user_id, user_role) + # Editors get the bottom-drawer picker (available widgets + host list); + # viewers/shared-readers don't, so skip the extra queries for them. + if can_edit: + plugins_cfg = current_app.config.get("PLUGINS", {}) + addable = get_available_widgets(plugins_cfg) + hosts = list((await db.execute(select(Host).order_by(Host.name))).scalars()) + else: + addable, hosts = [], [] resolved = _resolve_widgets(widgets) poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) - can_edit = _can_edit(dash, user_id, user_role) return await render_template( "dashboard/index.html", @@ -241,6 +222,8 @@ async def view(dash_id: int): widgets=resolved, poll_interval=poll_interval, can_edit=can_edit, + addable=addable, + hosts=hosts, **stats, ) @@ -282,7 +265,8 @@ async def create_dashboard(): db.add(dash) await db.flush() dash_id = dash.id - return redirect(f"/d/{dash_id}/edit") + # Open the new (empty) dashboard straight into edit mode to add widgets. + return redirect(f"/d/{dash_id}?edit=1") @dashboard_bp.post("/dashboards//rename") @@ -363,26 +347,12 @@ async def delete_dashboard(dash_id: int): return redirect("/dashboards/") -# ── Dashboard edit ──────────────────────────────────────────────────────────── - -@dashboard_bp.get("/d//edit") -@require_role(UserRole.viewer) -async def edit(dash_id: int): - user_id = current_user_id() - user_role = session.get("user_role", "viewer") - async with current_app.db_sessionmaker() as db: - dash, resolved, addable, hosts = await _get_panels_data(db, dash_id) - if dash is None or not _can_edit(dash, user_id, user_role): - abort(403) - return await render_template( - "dashboard/edit.html", - dashboard=dash, widgets=resolved, addable=addable, hosts=hosts, - ) - +# ── Dashboard edit (in-place over the live dashboard) ───────────────────────── @dashboard_bp.post("/d//edit/add/") @require_role(UserRole.viewer) async def add_widget(dash_id: int, widget_key: str): + """Add a widget and return its single grid-item HTML for in-place insertion.""" user_id = current_user_id() user_role = session.get("user_role", "viewer") defn = WIDGET_REGISTRY.get(widget_key) @@ -412,12 +382,24 @@ async def add_widget(dash_id: int, widget_key: str): widgets = await _get_widgets(db, dash_id) # Append below everything at full-left; the user then drags/resizes. next_y = max((w.grid_y + w.grid_h for w in widgets), default=0) - db.add(DashboardWidget( + new = DashboardWidget( dashboard_id=dash_id, widget_key=widget_key, grid_x=0, grid_y=next_y, grid_w=4, grid_h=4, title=title, config_json=config_json, - )) - return await _panels_response(dash_id) + ) + db.add(new) + await db.flush() + new_id = new.id + new = (await db.execute( + select(DashboardWidget).where(DashboardWidget.id == new_id))).scalar_one() + resolved = _resolve_widgets([new]) + if not resolved: + return ("", 204) + poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) + return await render_template( + "dashboard/_grid_item.html", + item=resolved[0], poll_interval=poll_interval, can_edit=True, + ) @dashboard_bp.post("/d//edit/remove/") @@ -437,7 +419,8 @@ async def remove_widget(dash_id: int, widget_id: int): widget = result.scalar_one_or_none() if widget and widget.dashboard_id == dash_id: await db.delete(widget) - return await _panels_response(dash_id) + # Client removes the DOM/grid item itself; nothing to render back. + return ("", 204) @dashboard_bp.post("/d//edit/layout") diff --git a/steward/templates/base.html b/steward/templates/base.html index 32cfb37..1652153 100644 --- a/steward/templates/base.html +++ b/steward/templates/base.html @@ -165,18 +165,28 @@ textarea { resize: vertical; } .ping-meta { min-width:120px; max-width:180px; flex-shrink:1; overflow:hidden; } .ping-name { font-size:0.875rem; font-weight:500; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; display:block; } a.ping-name:hover { text-decoration:underline; } -/* Dashboard widget grid — Grafana-style 12-col layout; cells sized by drag-resize - (Gridstack) in the edit view and rendered statically here from each widget's - x/y/w/h. Cell height is fixed (h * cell unit) with the body scrolling, so the - layout the operator arranged is exactly what they see. */ -.dash-grid { display:grid; grid-template-columns:repeat(12,1fr); grid-auto-rows:70px; gap:1rem; } -.dash-grid > .dash-cell { margin-bottom:0; min-width:0; overflow:hidden; display:flex; flex-direction:column; } -.dash-cell .dash-cell-body { flex:1; min-height:0; overflow:auto; } -@media (max-width:820px) { - .dash-grid { grid-template-columns:1fr; grid-auto-rows:auto; } - .dash-grid > .dash-cell { grid-column:1 / -1 !important; grid-row:auto !important; } - .dash-cell .dash-cell-body { overflow:visible; } +/* Dashboard widget grid — Gridstack-rendered. Static (positioned but inert) in + view mode; the dashboard itself becomes the edit surface when Edit is toggled, + so drag-resize happens over live widget data. The bottom drawer is a fixed + overlay, so the dashboard width is identical entering/leaving edit mode. */ +.grid-stack-item-content { + background:var(--bg-card); border:1px solid var(--border-mid); border-radius:8px; + padding:0.8rem 1rem; display:flex; flex-direction:column; overflow:hidden; } +.grid-stack-item-content .dash-cell-body { flex:1; min-height:0; overflow:auto; } +.panel-chrome { display:none; } +.panel-drag { cursor:grab; color:var(--text-dim); user-select:none; } +.panel-drag:active { cursor:grabbing; } +body.dash-editing .panel-chrome { display:inline-flex; align-items:center; } +body.dash-editing .grid-stack-item-content { outline:1px dashed var(--border-mid); outline-offset:-1px; } +body.dash-editing main { padding-bottom:46vh; } +.widget-drawer { + position:fixed; left:0; right:0; bottom:0; transform:translateY(100%); + transition:transform .25s ease; max-height:44vh; overflow-y:auto; + background:var(--bg-card); border-top:1px solid var(--border-mid); + box-shadow:0 -10px 30px rgba(0,0,0,0.45); z-index:60; padding:1rem 1.5rem 1.5rem; +} +body.dash-editing .widget-drawer { transform:translateY(0); } .ping-addr { display:block; color:var(--text-muted); font-size:0.75rem; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; } .ping-pills { display:flex; gap:2px; flex:1; min-width:0; overflow:hidden; align-items:center; } .pill { display:inline-block; width:7px; height:22px; border-radius:2px; cursor:default; transition:opacity .1s; } diff --git a/steward/templates/dashboard/_edit_panels.html b/steward/templates/dashboard/_edit_panels.html deleted file mode 100644 index 2a410e6..0000000 --- a/steward/templates/dashboard/_edit_panels.html +++ /dev/null @@ -1,127 +0,0 @@ -{# dashboard/_edit_panels.html — returned by all mutation routes and included by edit.html #} -
- - {# ── Current layout (drag to move, drag the corner/edge to resize) ──────── #} -
-
-
Current Layout
- {% if widgets %} - Drag to move · drag a corner to resize · saved automatically - {% endif %} -
- {% if widgets %} -
- {% for item in widgets %} -
-
-
- ⠿ - {{ item.title }} - -
-
{{ item.defn.description }}
-
-
- {% endfor %} -
- {% else %} -
- No widgets yet. Add one from the right to begin. -
- {% endif %} -
- - {# ── Available widgets ────────────────────────────────────────────────── #} -
-
Available Widgets
- {% if addable %} - {% set _groups = [("core", "Core monitors"), ("capability", "Monitoring capabilities"), ("integration", "Integrations")] %} - {% for _gkey, _glabel in _groups %} - {% set _gw = addable | selectattr("group", "equalto", _gkey) | list %} - {% if _gw %} -
{{ _glabel }}
-
- {% for w in _gw %} -
-
- -
-
{{ w.label }}
-
{{ w.description }}
-
- Add ▾ - - -
-
- -
- - -
- - {% for p in w.params %} -
- - {% if p.type == "select" %} - - {% elif p.type == "host" %} - - {% endif %} -
- {% endfor %} - - -
-
-
-
- {% endfor %} -
- {% endif %} - {% endfor %} - {% else %} -
- No plugin widgets available. Enable plugins in Settings → Plugins. -
- {% endif %} -
- -
diff --git a/steward/templates/dashboard/_grid_item.html b/steward/templates/dashboard/_grid_item.html new file mode 100644 index 0000000..e5562b3 --- /dev/null +++ b/steward/templates/dashboard/_grid_item.html @@ -0,0 +1,25 @@ +{# One dashboard widget as a Gridstack item — shared by index.html (initial render) + and the add-widget endpoint (single-item insert). Edit chrome (drag handle + + remove) is in the DOM only for editors and shown via body.dash-editing CSS. #} +
+
+
+ + {% if can_edit %}⠿{% endif %} + {{ item.title }} + + + Details → + {% if can_edit %}{% endif %} + +
+
+
+
diff --git a/steward/templates/dashboard/_widget_drawer.html b/steward/templates/dashboard/_widget_drawer.html new file mode 100644 index 0000000..f5a9ecf --- /dev/null +++ b/steward/templates/dashboard/_widget_drawer.html @@ -0,0 +1,78 @@ +{# Bottom drawer — the widget picker, revealed by body.dash-editing. Fixed-position + overlay so it never changes the dashboard's width. Add forms are submitted via + JS (see index.html) which inserts the returned grid item in place. #} +
+
+
Add a widget
+ +
+ {% if addable %} + {% set _groups = [("core", "Core monitors"), ("capability", "Monitoring capabilities"), ("integration", "Integrations")] %} + {% for _gkey, _glabel in _groups %} + {% set _gw = addable | selectattr("group", "equalto", _gkey) | list %} + {% if _gw %} +
{{ _glabel }}
+
+ {% for w in _gw %} +
+
+ +
+
{{ w.label }}
+
{{ w.description }}
+
+ Add ▾ + +
+
+
+ + +
+ {% for p in w.params %} +
+ + {% if p.type == "select" %} + + {% elif p.type == "host" %} + + {% endif %} +
+ {% endfor %} + +
+
+
+
+ {% endfor %} +
+ {% endif %} + {% endfor %} + {% else %} +
+ No plugin widgets available. Enable plugins in Settings → Plugins. +
+ {% endif %} +
diff --git a/steward/templates/dashboard/edit.html b/steward/templates/dashboard/edit.html deleted file mode 100644 index ad371c9..0000000 --- a/steward/templates/dashboard/edit.html +++ /dev/null @@ -1,58 +0,0 @@ -{% extends "base.html" %} -{% from "_macros.html" import crumbs %} -{% block title %}Edit: {{ dashboard.name }} — Steward{% endblock %} -{% block breadcrumb %}{{ crumbs([("Dashboard", "/"), ("Dashboards", "/dashboards/"), ("Edit " ~ dashboard.name, "")]) }}{% endblock %} -{% block head %} - -{% endblock %} -{% block content %} -
-

Edit: {{ dashboard.name }}

- Done -
-{% include "dashboard/_edit_panels.html" %} -{% endblock %} -{% block extra_scripts %} - - - -{% endblock %} diff --git a/steward/templates/dashboard/index.html b/steward/templates/dashboard/index.html index 317bb23..6feadac 100644 --- a/steward/templates/dashboard/index.html +++ b/steward/templates/dashboard/index.html @@ -1,5 +1,10 @@ {% extends "base.html" %} {% block title %}{{ dashboard.name }} — Steward{% endblock %} +{% block head %} +{# Gridstack renders the grid for everyone (static for viewers); editors also get + the in-place drag-resize + bottom drawer. #} + +{% endblock %} {% block content %} {# ── Header ────────────────────────────────────────────────────────────────── #} @@ -21,7 +26,7 @@ {% endif %} {% if can_edit %} - Edit + Share {% endif %} Dashboards @@ -60,31 +65,102 @@
-{# ── Widget grid ───────────────────────────────────────────────────────────── #} -{% if widgets %} -
- {% for item in widgets %} - {% set gw = item.db %} -
-
- {{ item.title }} - Details → -
-
-
-
- {% endfor %} -
-{% else %} -
+{# ── Widget grid (Gridstack; static until Edit is toggled) ──────────────────── #} +{% if not widgets %} +
No widgets on this dashboard yet.
{% if can_edit %} - Add Widgets + {% endif %}
{% endif %} +
+ {% for item in widgets %} + {% include "dashboard/_grid_item.html" %} + {% endfor %} +
+ +{% if can_edit %}{% include "dashboard/_widget_drawer.html" %}{% endif %} +{% endblock %} + +{% block extra_scripts %} + + {% endblock %} diff --git a/steward/templates/dashboard/list.html b/steward/templates/dashboard/list.html index 213e1ef..a13aec7 100644 --- a/steward/templates/dashboard/list.html +++ b/steward/templates/dashboard/list.html @@ -32,7 +32,7 @@
- Edit + Edit
Rename @@ -99,7 +99,7 @@ {# Admins can also edit system dashboards #} {% if user_role == 'admin' %} - Edit + Edit {% endif %}
From cb47b5e977bf65fb0cff50ae4c482d5e5a48c80c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 16:06:05 -0400 Subject: [PATCH 076/126] =?UTF-8?q?feat(dashboard):=20per-metric=20sparkli?= =?UTF-8?q?nes=20in=20the=20Host=20Agent=20=E2=80=94=20Resources=20widget?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator ask: show the graphs next to their fields in the fleet widget, like the host page's AGENT panel. Each row now renders cpu/mem/disk/load as a value with a trend sparkline beneath it (1h window), instead of bare numbers. - _fleet_rows fetches a per-host recent series (cpu/mem/load host-level, disk from the root mount) in one 1h query and attaches a sparkline per metric to each row. - widget_table.html lays out a metric cell (label + threshold-coloured value + sparkline) per field, mirroring panel.html. Threshold colour is computed in the loop and passed into the cell macro (keeps Jinja macro scope clean). Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 39 +++++++++++++++++++ .../host_agent/templates/widget_table.html | 36 +++++++++-------- steward/core/widgets.py | 2 +- 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 8134ca9..b73237d 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -349,6 +349,39 @@ async def _fleet_rows(session) -> list[dict]: continue latest.setdefault(row.resource_name, {})[row.metric_name] = row.value + # Per-host recent series for the inline sparklines (cpu/mem/load host-level, + # disk from the root mount), so each metric shows its trend beside the value — + # the host-panel at-a-glance, on the fleet widget. 1h window keeps it cheap. + host_names = [h.name for h in hosts.values()] + spark_series: dict[str, dict[str, list]] = { + n: {"cpu": [], "mem": [], "disk": [], "load": []} for n in host_names + } + if host_names: + since = datetime.now(timezone.utc) - timedelta(hours=1) + sp_rows = (await session.execute( + select(PluginMetric).where( + PluginMetric.source_module == SOURCE_MODULE, + PluginMetric.recorded_at >= since, + or_( + and_(PluginMetric.resource_name.in_(host_names), + PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))), + and_(PluginMetric.resource_name.in_([n + ":/" for n in host_names]), + PluginMetric.metric_name == "disk_used_pct"), + ), + ).order_by(PluginMetric.recorded_at) + )).scalars().all() + for r in sp_rows: + if r.resource_name.endswith(":/"): + name = r.resource_name[:-2] + if name in spark_series: + spark_series[name]["disk"].append(r.value) + else: + key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name) + if key and r.resource_name in spark_series: + spark_series[r.resource_name][key].append(r.value) + + from steward.core.status import sparkline_svg + stale_after = _stale_after_seconds() now = datetime.now(timezone.utc) rows = [] @@ -359,6 +392,11 @@ async def _fleet_rows(session) -> list[dict]: m = latest.get(host.name, {}) ls = reg.last_seen_at stale = ls is None or (now - ls).total_seconds() > stale_after + ser = spark_series.get(host.name, {}) + sparks = { + k: (sparkline_svg(v[-60:], width=72, height=16) if len(v) >= 2 else "") + for k, v in ser.items() + } rows.append({ "host": host, "reg": reg, @@ -371,6 +409,7 @@ async def _fleet_rows(session) -> list[dict]: "temp_max": m.get("temp_c_max"), "net_rx_bps": m.get("net_rx_bps"), "net_tx_bps": m.get("net_tx_bps"), + "sparks": sparks, }) rows.sort(key=lambda r: r["host"].name.lower()) return rows diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html index df3f489..aa5ffc7 100644 --- a/plugins/host_agent/templates/widget_table.html +++ b/plugins/host_agent/templates/widget_table.html @@ -1,10 +1,20 @@ -{# host_agent widget — fleet glance, flex rows (no header wrap) #} +{# host_agent widget — fleet glance: per-host row with a sparkline beside each + metric (cpu/mem/disk/load), mirroring the host-page AGENT panel. #} {% from "_macros.html" import metric_style %} {% if rows %} +{% macro metric_cell(label, value, fmt, svg, style="", title="") %} +
+
{{ label }}
+
+ {% if value is not none %}{{ fmt|format(value) }}{% else %}—{% endif %} +
+
{{ svg | safe }}
+
+{% endmacro %}
{% for r in rows %} {% set stale = r.stale %} -
+
{% set name_style = "flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-weight:500;" %} @@ -13,21 +23,13 @@ {% else %} {{ r.host.name }} {% endif %} - - {% if r.cpu_pct is not none %} - cpu {{ "%.0f"|format(r.cpu_pct) }}% - {% endif %} - {% if r.mem_used_pct is not none %} - mem {{ "%.0f"|format(r.mem_used_pct) }}% - {% endif %} - {% if r.disk_root is not none %} - disk / {{ "%.0f"|format(r.disk_root) }}% - {% endif %} - {% if r.load_1m is not none %} - load {{ "%.2f"|format(r.load_1m) }} - {% endif %} - - +
+ {{ metric_cell("cpu", r.cpu_pct, "%.0f%%", r.sparks.cpu, metric_style(r.cpu_pct), "Average CPU utilization") }} + {{ metric_cell("mem", r.mem_used_pct, "%.0f%%", r.sparks.mem, metric_style(r.mem_used_pct), "Memory in use") }} + {{ metric_cell("disk /", r.disk_root, "%.0f%%", r.sparks.disk, metric_style(r.disk_root), "Root filesystem (/) usage") }} + {{ metric_cell("load", r.load_1m, "%.2f", r.sparks.load, "", "Load average (1m)") }} +
+ {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }}
diff --git a/steward/core/widgets.py b/steward/core/widgets.py index a4f0349..401db37 100644 --- a/steward/core/widgets.py +++ b/steward/core/widgets.py @@ -270,7 +270,7 @@ WIDGET_REGISTRY: dict[str, dict] = { "host_resources": { "key": "host_resources", "label": "Host Agent — Resources", - "description": "Fleet view: CPU, memory, disk, and load per monitored host", + "description": "Fleet view: CPU, memory, disk, and load per host — each with a trend sparkline", "hx_url": "/plugins/host_agent/widget", "detail_url": "/hosts/", "plugin": "host_agent", From e58c86cf010f690228622a0dc8cae7b921900448 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 16:14:59 -0400 Subject: [PATCH 077/126] =?UTF-8?q?feat(dashboard):=20widget=20readability?= =?UTF-8?q?=20=E2=80=94=20name-per-line,=20fill=20graphs,=20container=20br?= =?UTF-8?q?eakpoints?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address graphical issues raised from the dashboard screenshot: - No more truncated host names. Hosts-Overview and Host-Agent-Resources put the host name on its own line (full, wraps if needed) with its data grouped beneath it; ping/dns (.ping-name) and uptime widget names wrap instead of ellipsis. Prefer vertical overflow over cramming/truncating a row. - History graph fills the panel: drop the fixed 0–100 y-axis ceiling (beginAtZero + 8% grace) so the lines use the vertical space instead of hugging the bottom; axis labels still show real %. - Container-query breakpoints: the widget body is now a query container, so fragments restyle to their OWN panel width. Host-list widgets flow into 2 cols ≥520px and 3 cols ≥900px (.host-blocks) — use the width, remove vertical deadspace — and collapse to one column when narrow. Mirrored into the share view. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../host_agent/templates/widget_history.html | 5 ++- .../host_agent/templates/widget_table.html | 39 +++++++++++-------- steward/templates/base.html | 10 ++++- steward/templates/dashboard/share_view.html | 6 ++- steward/templates/hosts/overview_widget.html | 39 +++++++++++-------- steward/templates/hosts/uptime_widget.html | 4 +- 6 files changed, 64 insertions(+), 39 deletions(-) diff --git a/plugins/host_agent/templates/widget_history.html b/plugins/host_agent/templates/widget_history.html index cf9f492..5d1a464 100644 --- a/plugins/host_agent/templates/widget_history.html +++ b/plugins/host_agent/templates/widget_history.html @@ -43,7 +43,10 @@ elements: { point: { radius: 0 } }, scales: { x: { type: "linear", ticks: { callback: function (v) { return fmtTime(v); }, maxTicksLimit: 6, color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } }, - y: { beginAtZero: true, max: 100, ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } }, + // Auto-scale to the data (with a little headroom) so the lines fill the + // panel instead of hugging the bottom of a fixed 0–100 axis. Axis labels + // still show the real %, so the values stay honest. + y: { beginAtZero: true, grace: "8%", ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } }, }, plugins: { legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } } }, }, diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html index aa5ffc7..a403a0c 100644 --- a/plugins/host_agent/templates/widget_table.html +++ b/plugins/host_agent/templates/widget_table.html @@ -1,9 +1,10 @@ -{# host_agent widget — fleet glance: per-host row with a sparkline beside each - metric (cpu/mem/disk/load), mirroring the host-page AGENT panel. #} +{# host_agent widget — fleet glance. Per host: name on its own line (never + truncated), then a row of metric cells (cpu/mem/disk/load), each value with a + trend sparkline beneath it — the host-page AGENT panel, on the dashboard. #} {% from "_macros.html" import metric_style %} {% if rows %} {% macro metric_cell(label, value, fmt, svg, style="", title="") %} -
+
{{ label }}
{% if value is not none %}{{ fmt|format(value) }}{% else %}—{% endif %} @@ -11,27 +12,31 @@
{{ svg | safe }}
{% endmacro %} -
+
{% for r in rows %} {% set stale = r.stale %} -
- - {% set name_style = "flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-weight:500;" %} - {% if session.user_id %} - {{ r.host.name }} - {% else %} - {{ r.host.name }} - {% endif %} -
+
+ {# Line 1 — health dot + full host name (own line) + last seen #} +
+ + {% if session.user_id %} + {{ r.host.name }} + {% else %} + {{ r.host.name }} + {% endif %} + + {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }} + +
+ {# Line 2 — metric cells with value + trend; wraps instead of truncating #} +
{{ metric_cell("cpu", r.cpu_pct, "%.0f%%", r.sparks.cpu, metric_style(r.cpu_pct), "Average CPU utilization") }} {{ metric_cell("mem", r.mem_used_pct, "%.0f%%", r.sparks.mem, metric_style(r.mem_used_pct), "Memory in use") }} {{ metric_cell("disk /", r.disk_root, "%.0f%%", r.sparks.disk, metric_style(r.disk_root), "Root filesystem (/) usage") }} {{ metric_cell("load", r.load_1m, "%.2f", r.sparks.load, "", "Load average (1m)") }}
- - {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }} -
{% endfor %}
diff --git a/steward/templates/base.html b/steward/templates/base.html index 1652153..b6c8012 100644 --- a/steward/templates/base.html +++ b/steward/templates/base.html @@ -163,7 +163,7 @@ textarea { resize: vertical; } .ping-row { display:flex; align-items:center; gap:1rem; padding:0.45rem 0; border-bottom:1px solid var(--border); } .ping-row:last-child { border-bottom:none; } .ping-meta { min-width:120px; max-width:180px; flex-shrink:1; overflow:hidden; } -.ping-name { font-size:0.875rem; font-weight:500; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; display:block; } +.ping-name { font-size:0.875rem; font-weight:500; display:block; overflow-wrap:anywhere; } a.ping-name:hover { text-decoration:underline; } /* Dashboard widget grid — Gridstack-rendered. Static (positioned but inert) in view mode; the dashboard itself becomes the edit surface when Edit is toggled, @@ -174,6 +174,14 @@ a.ping-name:hover { text-decoration:underline; } padding:0.8rem 1rem; display:flex; flex-direction:column; overflow:hidden; } .grid-stack-item-content .dash-cell-body { flex:1; min-height:0; overflow:auto; } +/* The widget body is a query container, so fragments can restyle to their OWN + panel width (not the viewport) — widgets adapt as you resize them. */ +.dash-cell-body { container-type:inline-size; } +/* Host-list widgets flow into columns as the panel widens: use the width, drop + the vertical deadspace; stay single-column (and untruncated) when narrow. */ +.host-blocks { display:grid; grid-template-columns:1fr; gap:0 1.5rem; } +@container (min-width:520px) { .host-blocks { grid-template-columns:1fr 1fr; } } +@container (min-width:900px) { .host-blocks { grid-template-columns:1fr 1fr 1fr; } } .panel-chrome { display:none; } .panel-drag { cursor:grab; color:var(--text-dim); user-select:none; } .panel-drag:active { cursor:grabbing; } diff --git a/steward/templates/dashboard/share_view.html b/steward/templates/dashboard/share_view.html index 67d11c9..d0ed81f 100644 --- a/steward/templates/dashboard/share_view.html +++ b/steward/templates/dashboard/share_view.html @@ -44,7 +44,7 @@ .ping-row { display:flex; align-items:center; gap:1rem; padding:0.45rem 0; border-bottom:1px solid var(--border); } .ping-row:last-child { border-bottom:none; } .ping-meta { min-width:120px; max-width:180px; flex-shrink:1; overflow:hidden; } - .ping-name { font-size:0.875rem; font-weight:500; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; display:block; } + .ping-name { font-size:0.875rem; font-weight:500; display:block; overflow-wrap:anywhere; } .ping-addr { display:block; color:var(--text-muted); font-size:0.75rem; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; } .ping-pills { display:flex; gap:2px; flex:1; min-width:0; overflow:hidden; align-items:center; } .pill { display:inline-block; width:7px; height:22px; border-radius:2px; } @@ -59,6 +59,10 @@ .dash-grid { display:grid; grid-template-columns:repeat(12,1fr); grid-auto-rows:70px; gap:1rem; } .dash-grid > .dash-cell { margin-bottom:0; min-width:0; overflow:hidden; display:flex; flex-direction:column; } .dash-cell .dash-cell-body { flex:1; min-height:0; overflow:auto; } + .dash-cell-body { container-type:inline-size; } + .host-blocks { display:grid; grid-template-columns:1fr; gap:0 1.5rem; } + @container (min-width:520px) { .host-blocks { grid-template-columns:1fr 1fr; } } + @container (min-width:900px) { .host-blocks { grid-template-columns:1fr 1fr 1fr; } } @media (max-width:820px) { .dash-grid { grid-template-columns:1fr; grid-auto-rows:auto; } .dash-grid > .dash-cell { grid-column:1 / -1 !important; grid-row:auto !important; } diff --git a/steward/templates/hosts/overview_widget.html b/steward/templates/hosts/overview_widget.html index 8c2052b..16776e9 100644 --- a/steward/templates/hosts/overview_widget.html +++ b/steward/templates/hosts/overview_widget.html @@ -1,33 +1,38 @@ -{# Unified Hosts widget — monitor status + agent glance per host, links to the hub. #} +{# Unified Hosts widget — monitor status + agent glance per host, links to the hub. + Name sits on its own line (never truncated); its data wraps beneath it, grouped + under the host. Prefer vertical growth over cramming a row. #} {% from "_macros.html" import metric_style %} {% if hosts %} -
+
{% for host in hosts %} {% set ping = latest_pings.get(host.id) %} {% set dns = latest_dns.get(host.id) %} {% set ut = uptime.get(host.id, {}) %} {% set a = agent.get(host.name, {}) %} {% set down = host.ping_enabled and ping and ping.status.value != 'up' %} -
- - {% set name_style = "flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-weight:500;" %} - {% if session.user_id %} - {{ host.name }} - {% else %} - {{ host.name }} - {% endif %} - {% set spark = cpu_sparks.get(host.name) %} - {% if spark %}{{ spark | safe }}{% endif %} - - {# Monitors #} + {% set spark = cpu_sparks.get(host.name) %} +
+ {# Line 1 — status dot + full host name (own line) + cpu trend #} +
+ + {% if session.user_id %} + {{ host.name }} + {% else %} + {{ host.name }} + {% endif %} + {% if spark %}{{ spark | safe }}{% endif %} +
+ {# Line 2 — monitors + agent metrics; wraps instead of truncating #} +
{% if host.ping_enabled and ping and ping.response_time_ms is not none %} ping {{ "%.0f"|format(ping.response_time_ms) }}ms {% endif %} {% if ut.get('24h') is not none %} 24h {{ "%.1f"|format(ut['24h']) }}% {% endif %} - {# Agent glance — failing metric coloured amber/red (≥80/≥90) #} {% if a %} {% if a.get('cpu_pct') is not none %}cpu {{ "%.0f"|format(a.cpu_pct) }}%{% endif %} {% if a.get('mem_used_pct') is not none %}mem {{ "%.0f"|format(a.mem_used_pct) }}%{% endif %} @@ -36,7 +41,7 @@ {% else %} no agent {% endif %} - +
{% endfor %}
diff --git a/steward/templates/hosts/uptime_widget.html b/steward/templates/hosts/uptime_widget.html index 50b34ed..52675e4 100644 --- a/steward/templates/hosts/uptime_widget.html +++ b/steward/templates/hosts/uptime_widget.html @@ -17,9 +17,9 @@
{% if session.user_id %} {{ host.name }} + style="overflow-wrap:anywhere;color:inherit;text-decoration:none;">{{ host.name }} {% else %} - {{ host.name }} + {{ host.name }} {% endif %} {% for window in ["24h", "7d", "30d"] %} {% set pct = ut.get(window) %} From fed9973899d1fce27a6730bb3bc29532a67a8fae Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 16:18:50 -0400 Subject: [PATCH 078/126] fix(dashboard): downsample history graphs + readable tooltip time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent reports every few seconds, so multi-hour history series were hundreds– thousands of points — a dense, noisy line (esp. CPU). Bucket-average server-side to ~120 points (keeps the shape, drops the noise) for both the history-graph widget and the host-detail charts. Also fix the chart tooltip title showing the raw epoch-ms (e.g. 1,781,720,471,459) — format it as HH:MM. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 28 ++++++++++++++++++- plugins/host_agent/templates/host_detail.html | 5 +++- .../host_agent/templates/widget_history.html | 5 +++- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index b73237d..848c05a 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -461,11 +461,35 @@ async def widget_history(): key = "disk_root" if p.resource_name != host.name else p.metric_name series[key].append([int(p.recorded_at.timestamp() * 1000), round(p.value, 2)]) + # Downsample: the agent reports every few seconds, so a multi-hour window is + # hundreds–thousands of points — a noisy, unreadable line. Bucket-average to a + # readable point count, keeping the shape. + series = {k: _downsample(v) for k, v in series.items()} return await render_template( "widget_history.html", host=host, series=series, hours=hours, wid=wid, ) +def _downsample(series: list[list], target: int = 120) -> list[list]: + """Bucket-average a time series [[epoch_ms, value], …] down to ~target points. + + Consecutive samples are grouped into equal buckets and averaged (x = bucket + midpoint). Cheap, order-preserving, and smooths the high-frequency noise that + makes dense agent series hard to read. + """ + n = len(series) + if n <= target: + return series + bucket = (n + target - 1) // target + out: list[list] = [] + for i in range(0, n, bucket): + chunk = series[i:i + bucket] + mid = chunk[len(chunk) // 2][0] + avg = sum(p[1] for p in chunk) / len(chunk) + out.append([mid, round(avg, 2)]) + return out + + # Host-level metrics charted on the detail page (sub-resources are shown as # current-value lists, not time series, to keep the page readable). HISTORY_METRICS = ( @@ -520,7 +544,9 @@ async def _history_for_host(session, host_name: str, since) -> dict[str, list]: series: dict[str, list] = {m: [] for m in HISTORY_METRICS} for p in rows: series[p.metric_name].append([int(p.recorded_at.timestamp() * 1000), round(p.value, 2)]) - return series + # Downsample to a readable point count (see _downsample) — raw agent cadence + # is too dense to read over a multi-hour window. + return {m: _downsample(v) for m, v in series.items()} @host_agent_bp.get("//") diff --git a/plugins/host_agent/templates/host_detail.html b/plugins/host_agent/templates/host_detail.html index c34fda9..42df49e 100644 --- a/plugins/host_agent/templates/host_detail.html +++ b/plugins/host_agent/templates/host_detail.html @@ -216,7 +216,10 @@ x: { type: "linear", ticks: { callback: (v) => fmtTime(v), maxTicksLimit: 8, color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } }, y: { beginAtZero: true, max: ymax, ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } }, }, - plugins: { legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } } }, + plugins: { + legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } }, + tooltip: { callbacks: { title: function (items) { return items.length ? fmtTime(items[0].parsed.x) : ""; } } }, + }, }); const mk = (id, datasets, ymax) => { const el = document.getElementById(id); diff --git a/plugins/host_agent/templates/widget_history.html b/plugins/host_agent/templates/widget_history.html index 5d1a464..1414ad5 100644 --- a/plugins/host_agent/templates/widget_history.html +++ b/plugins/host_agent/templates/widget_history.html @@ -48,7 +48,10 @@ // still show the real %, so the values stay honest. y: { beginAtZero: true, grace: "8%", ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } }, }, - plugins: { legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } } }, + plugins: { + legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } }, + tooltip: { callbacks: { title: function (items) { return items.length ? fmtTime(items[0].parsed.x) : ""; } } }, + }, }, }); })(); From ebc67723d2ff7a228d6ed9b45528278321f4b78b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 16:31:27 -0400 Subject: [PATCH 079/126] feat(host_agent): horizontal zebra-striped fleet widget rows Restore the host_agent fleet-glance widget to one horizontal row per host (name left, metric cells + sparklines right) instead of the multi-column block grid, and zebra-stripe odd rows so adjacent hosts read as distinct. Names still wrap rather than hard-truncate. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../host_agent/templates/widget_table.html | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html index a403a0c..30fac53 100644 --- a/plugins/host_agent/templates/widget_table.html +++ b/plugins/host_agent/templates/widget_table.html @@ -1,37 +1,41 @@ -{# host_agent widget — fleet glance. Per host: name on its own line (never - truncated), then a row of metric cells (cpu/mem/disk/load), each value with a - trend sparkline beneath it — the host-page AGENT panel, on the dashboard. #} +{# host_agent widget — fleet glance. One horizontal row per host, zebra-striped + so adjacent hosts read as distinct groupings: health dot + host name on the + left, metric cells (cpu/mem/disk/load) each with a trend sparkline laid out + horizontally on the right. The name wraps (never hard-truncated) but the row + stays horizontal. #} {% from "_macros.html" import metric_style %} {% if rows %} {% macro metric_cell(label, value, fmt, svg, style="", title="") %} -
-
{{ label }}
+
+
{{ label }}
{% if value is not none %}{{ fmt|format(value) }}{% else %}—{% endif %}
{{ svg | safe }}
{% endmacro %} -
+
{% for r in rows %} {% set stale = r.stale %} -
- {# Line 1 — health dot + full host name (own line) + last seen #} -
+
+ {# Left — health dot + full host name (wraps, never truncated) + last seen #} +
- {% if session.user_id %} - {{ r.host.name }} - {% else %} - {{ r.host.name }} - {% endif %} - - {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }} + + {% if session.user_id %} + {{ r.host.name }} + {% else %} + {{ r.host.name }} + {% endif %} + + {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }} +
- {# Line 2 — metric cells with value + trend; wraps instead of truncating #} -
+ {# Right — metric cells with value + trend, laid out horizontally #} +
{{ metric_cell("cpu", r.cpu_pct, "%.0f%%", r.sparks.cpu, metric_style(r.cpu_pct), "Average CPU utilization") }} {{ metric_cell("mem", r.mem_used_pct, "%.0f%%", r.sparks.mem, metric_style(r.mem_used_pct), "Memory in use") }} {{ metric_cell("disk /", r.disk_root, "%.0f%%", r.sparks.disk, metric_style(r.disk_root), "Root filesystem (/) usage") }} From 10808c1c5de5cd51799b5773cb6d98d18c360395 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 16:44:31 -0400 Subject: [PATCH 080/126] fix(dashboard): inline cpu sparkline, history graph host label + stable y-axis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Hosts-Overview: move the CPU sparkline inline next to the cpu % value it represents (was floated far right on the name line, reading as unrelated). - Host Agent history widget: caption the chart with the host it represents (the panel title is generic) — links to the host hub. - History widget: snap the y-axis to a stable 0..next-10%-band ceiling instead of auto-scaling to the exact peak, so the 'zoom' no longer jumps each poll. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/host_agent/routes.py | 8 +++++++ .../host_agent/templates/widget_history.html | 23 ++++++++++++++----- steward/templates/hosts/overview_widget.html | 3 +-- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 848c05a..644dbee 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -465,8 +465,16 @@ async def widget_history(): # hundreds–thousands of points — a noisy, unreadable line. Bucket-average to a # readable point count, keeping the shape. series = {k: _downsample(v) for k, v in series.items()} + + # Stable y-axis ceiling. All three series are percentages, so snap the top up + # to the next 10% band (floor 20, cap 100). Auto-scaling to the exact peak made + # the chart "zoom" jump on every poll as the peak drifted a few %; snapping to + # a band keeps the axis steady across refreshes while still filling the panel. + peak = max((p[1] for s in series.values() for p in s), default=0.0) + y_max = min(100, max(20, ((int(peak) // 10) + 1) * 10)) return await render_template( "widget_history.html", host=host, series=series, hours=hours, wid=wid, + y_max=y_max, ) diff --git a/plugins/host_agent/templates/widget_history.html b/plugins/host_agent/templates/widget_history.html index 1414ad5..1bd73a8 100644 --- a/plugins/host_agent/templates/widget_history.html +++ b/plugins/host_agent/templates/widget_history.html @@ -11,8 +11,19 @@ No agent metrics for {{ host.name }} in the last {{ hours }}h yet.
{% else %} -
- +
+ {# Name the host the graph represents — the panel title is generic. #} +
+ {% if session.user_id %} + {{ host.name }} + {% else %} + {{ host.name }} + {% endif %} + · last {{ hours }}h +
+
+ +
{% endblock %} diff --git a/steward/templates/hosts/form.html b/steward/templates/hosts/form.html index d6124b7..a6dc34e 100644 --- a/steward/templates/hosts/form.html +++ b/steward/templates/hosts/form.html @@ -15,47 +15,16 @@
-
- - -
-
- - -
-
- -
-
- -
-
- - -
- Cancel + Cancel
- {% if host %}

- Agent, Ansible target, and playbook runs live on the - host page. + {% if host %}Monitors, agent, Ansible target, and playbook runs live on the + host page.{% else %} + After adding, attach ping / DNS / HTTP monitors from the host page.{% endif %}

- {% endif %}
{% endblock %} diff --git a/steward/templates/hosts/list.html b/steward/templates/hosts/list.html index dde6693..c7d72ba 100644 --- a/steward/templates/hosts/list.html +++ b/steward/templates/hosts/list.html @@ -18,10 +18,9 @@ Name Address - Probe - Ping + Status Latency - DNS + Monitors 24h 7d 30d @@ -31,65 +30,39 @@ {% for host in hosts %} - {% set ping = latest_pings.get(host.id) %} - {% set dns = latest_dns.get(host.id) %} + {% set hs = host_status.get(host.id) %} {% set ut = uptime.get(host.id, {}) %} {{ host.name }} {{ host.address }} - - {{ host.probe_type.value | upper }}{% if host.probe_type.value == 'tcp' %}:{{ host.probe_port }}{% endif %} - - {% if not host.ping_enabled %} - off - {% elif ping is none %} - — - {% elif ping.status.value == 'up' %} + {% if not hs %} + — + {% elif hs.state == 'up' %} - {% else %} + {% elif hs.state == 'down' %} + {% else %} + {% endif %} - {% if ping and ping.response_time_ms is not none %} - {% if ping.response_time_ms < 10 %} - {{ ping.response_time_ms | round(1) }} ms - {% elif ping.response_time_ms < 100 %} - {{ ping.response_time_ms | round(1) }} ms - {% else %} - {{ ping.response_time_ms | round(1) }} ms - {% endif %} + {% if hs and hs.latency_ms is not none %} + {{ hs.latency_ms | round(1) }} ms {% else %} — {% endif %} - - {% if not host.dns_enabled %} - off - {% elif dns is none %} - — - {% elif dns.status.value == 'resolved' %} - - {% else %} - - {% endif %} + + {{ hs.count if hs else 0 }} {% for window in ["24h", "7d", "30d"] %} {% set pct = ut.get(window) %} - {% if not host.ping_enabled %} + {% if pct is none %} — - {% elif pct is none %} - — - {% elif pct >= 99.9 %} - {{ "%.1f"|format(pct) }}% - {% elif pct >= 99 %} - {{ "%.1f"|format(pct) }}% - {% elif pct >= 95 %} - {{ "%.1f"|format(pct) }}% {% else %} - {{ "%.1f"|format(pct) }}% + {{ "%.1f"|format(pct) }}% {% endif %} {% endfor %} diff --git a/steward/templates/hosts/overview_widget.html b/steward/templates/hosts/overview_widget.html index e92a125..267f8c4 100644 --- a/steward/templates/hosts/overview_widget.html +++ b/steward/templates/hosts/overview_widget.html @@ -4,18 +4,17 @@ {% if hosts %}
{% for host in hosts %} - {% set ping = latest_pings.get(host.id) %} - {% set dns = latest_dns.get(host.id) %} + {% set hs = host_status.get(host.id) %} {% set ut = uptime.get(host.id, {}) %} {% set a = agent.get(host.name, {}) %} - {% set down = host.ping_enabled and ping and ping.status.value != 'up' %} + {% set down = hs and hs.state == 'down' %} {% set spark = cpu_sparks.get(host.name) %}
{# Line 1 — status dot + full host name (own line) + cpu trend #}
- + title="{% if not hs %}no monitors{% else %}{{ hs.state }}{% endif %}"> {% if session.user_id %} {{ host.name }} {% else %} @@ -25,8 +24,8 @@ {# Line 2 — monitors + agent metrics; wraps instead of truncating #}
- {% if host.ping_enabled and ping and ping.response_time_ms is not none %} - ping {{ "%.0f"|format(ping.response_time_ms) }}ms + {% if hs and hs.latency_ms is not none %} + ping {{ "%.0f"|format(hs.latency_ms) }}ms {% endif %} {% if ut.get('24h') is not none %} 24h {{ "%.1f"|format(ut['24h']) }}% diff --git a/steward/templates/hosts/uptime.html b/steward/templates/hosts/uptime.html index 3a408d1..7a7230e 100644 --- a/steward/templates/hosts/uptime.html +++ b/steward/templates/hosts/uptime.html @@ -7,13 +7,10 @@
{# ── Summary pills ──────────────────────────────────────────────────────────── #} -{% set ping_hosts = hosts | selectattr("ping_enabled") | list %} -{% set total = ping_hosts | length %} +{% set total = hosts | length %} {% if total %} -{% set full_30d = ping_hosts | selectattr("id", "in", uptime.keys()) - | selectattr("id", "defined") | list %} {% set healthy = namespace(count=0) %} -{% for h in ping_hosts %} +{% for h in hosts %} {% set pct = uptime.get(h.id, {}).get("30d") %} {% if pct is not none and pct >= 99 %}{% set healthy.count = healthy.count + 1 %}{% endif %} {% endfor %} @@ -50,15 +47,13 @@ {% for host in hosts %} {% set ut = uptime.get(host.id, {}) %} - {{ host.name }} + {{ host.name }} {{ host.address }} {% for window in ["24h", "7d", "30d"] %} {% set pct = ut.get(window) %} - {% if not host.ping_enabled %} - — - {% elif pct is none %} + {% if pct is none %} — {% elif pct >= 99.9 %} {{ "%.2f"|format(pct) }}% @@ -75,9 +70,7 @@ {# 30-day visual bar #} {% set pct30 = ut.get("30d") %} - {% if not host.ping_enabled %} - ping off - {% elif pct30 is none %} + {% if pct30 is none %} no data {% else %}
@@ -99,6 +92,6 @@ {% endif %}

- Uptime is computed from ping probe results only. Hosts with ping disabled show —. + Uptime aggregates every monitor attached to a host. Hosts with no monitors show —.

{% endblock %} diff --git a/steward/templates/hosts/uptime_widget.html b/steward/templates/hosts/uptime_widget.html index fe31dc2..a9a79c4 100644 --- a/steward/templates/hosts/uptime_widget.html +++ b/steward/templates/hosts/uptime_widget.html @@ -1,7 +1,7 @@ {# hosts/uptime_widget.html — dashboard widget: SLA uptime summary #} {% if not hosts %}
- No ping-enabled hosts configured. + No hosts configured.
{% else %} diff --git a/steward/templates/monitors/_fields.html b/steward/templates/monitors/_fields.html new file mode 100644 index 0000000..d98c6d5 --- /dev/null +++ b/steward/templates/monitors/_fields.html @@ -0,0 +1,64 @@ +{# Shared type-specific monitor form fields. Every field is rendered; a tiny + script (mtoggle) shows only the ones relevant to the selected type. #} +{% macro type_fields(config={}) %} +{% set inp = "width:100%;padding:0.4rem 0.65rem;background:var(--bg);border:1px solid var(--border-mid);border-radius:4px;color:var(--text);font-size:0.88rem;" %} +{% set lbl = "font-size:0.75rem;color:var(--text-muted);display:block;margin-bottom:0.2rem;" %} + +{# tcp #} +
+ + +
+ +{# dns #} +
+ + +
+ +{# http #} +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+{% endmacro %} + +{# Show only the fields whose data-types matches the active monitor type. #} +{% macro toggle_script() %} + +{% endmacro %} diff --git a/steward/templates/monitors/edit.html b/steward/templates/monitors/edit.html new file mode 100644 index 0000000..ac383b8 --- /dev/null +++ b/steward/templates/monitors/edit.html @@ -0,0 +1,48 @@ +{% extends "base.html" %} +{% from "monitors/_fields.html" import type_fields, toggle_script %} +{% block title %}Edit Monitor — Steward{% endblock %} +{% block content %} +
+ ← Monitors +

Edit Monitor

+
+ +{% set inp = "width:100%;padding:0.4rem 0.65rem;background:var(--bg);border:1px solid var(--border-mid);border-radius:4px;color:var(--text);font-size:0.88rem;" %} +{% set lbl = "font-size:0.75rem;color:var(--text-muted);display:block;margin-bottom:0.2rem;" %} + +
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ {{ type_fields(config) }} +
+ {% if monitor.host_id %} +

+ Linked to its host. +

+ {% endif %} +
+ + Cancel +
+
+
+ +{{ toggle_script() }} + +{% endblock %} diff --git a/steward/templates/monitors/index.html b/steward/templates/monitors/index.html new file mode 100644 index 0000000..75708cb --- /dev/null +++ b/steward/templates/monitors/index.html @@ -0,0 +1,60 @@ +{% extends "base.html" %} +{% from "monitors/_fields.html" import type_fields, toggle_script %} +{% block title %}Monitors — Steward{% endblock %} +{% block content %} +
+
+

Monitors

+ polling every {{ poll_interval }}s +
+ {% include "_time_range.html" %} +
+ +{% set inp = "width:100%;padding:0.4rem 0.65rem;background:var(--bg);border:1px solid var(--border-mid);border-radius:4px;color:var(--text);font-size:0.88rem;" %} +{% set lbl = "font-size:0.75rem;color:var(--text-muted);display:block;margin-bottom:0.2rem;" %} + +{# ── Add monitor ──────────────────────────────────────────────────────────── #} +
+
Add Monitor
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ {{ type_fields() }} +
+
+
+
+ +{# ── Live monitor rows ────────────────────────────────────────────────────── #} +
+
+
Loading…
+
+
+ +{{ toggle_script() }} + +{% endblock %} diff --git a/steward/templates/monitors/rows.html b/steward/templates/monitors/rows.html new file mode 100644 index 0000000..5091180 --- /dev/null +++ b/steward/templates/monitors/rows.html @@ -0,0 +1,74 @@ +{# HTMX fragment — monitor rows (list page). #} +{% if monitor_data %} +{% for d in monitor_data %} +{% set m = d.monitor %} +{% set latest = d.latest %} +
+
+ {{ m.type | upper }} + {% if m.host_id %} + {{ m.name }} + {% else %} + {{ m.name }} + {% endif %} + {{ m.target }} +
+ +
+ {% for _ in range([30 - d.heartbeat|length, 0]|max) %} + + {% endfor %} + {% for h in d.heartbeat %} + + {% endfor %} +
+ +
+ {% if latest is none %} + — + {% elif not latest.is_up %} + DOWN + {% elif latest.response_ms is not none %} + + {{ "%.0f"|format(latest.response_ms) }} ms + {% elif m.type == 'dns' and latest.resolved_ip %} + {{ latest.resolved_ip }} + {% else %} + UP + {% endif %} +
+ +
+ {% if d.uptime_pct is not none %}{{ "%.1f"|format(d.uptime_pct) }}%{% else %}—{% endif %} + {{ range_key }} +
+ + {% if d.tls_days is not none %} +
+ cert {{ d.tls_days|round|int }}d +
+ {% endif %} + + {% if session.user_role in ['operator', 'admin'] %} +
+ Edit +
+ +
+
+ +
+
+ {% endif %} +
+{% endfor %} +{% else %} +

+ No monitors yet. Add one above, or attach checks to a host from the + Hosts page. +

+{% endif %} diff --git a/steward/templates/monitors/widget.html b/steward/templates/monitors/widget.html new file mode 100644 index 0000000..aea1b06 --- /dev/null +++ b/steward/templates/monitors/widget.html @@ -0,0 +1,44 @@ +{# monitors/widget.html — dashboard widget: unified monitor summary #} +{% if not monitor_data and up == 0 and down == 0 and pending == 0 %} +
+ No monitors configured. Add monitors → +
+{% else %} + +
+ {% if up %} +
{{ up }} + up
+ {% endif %} + {% if down %} +
{{ down }} + down
+ {% endif %} + {% if pending %} +
{{ pending }} + pending
+ {% endif %} +
+ +
+ {% for d in monitor_data %} + {% set latest = d.latest %} +
+ {% if not latest %} + {% elif latest.is_up %} + {% else %}{% endif %} + {% if d.monitor.host_id %} + {{ d.monitor.name }} + {% else %} + {{ d.monitor.name }} + {% endif %} + {% if latest and latest.response_ms is not none %} + {{ "%.0f"|format(latest.response_ms) }}ms + {% elif latest and not latest.is_up %} + {{ latest.status_code or "down" }} + {% endif %} +
+ {% endfor %} +
+ +{% endif %} diff --git a/steward/templates/ping/index.html b/steward/templates/ping/index.html deleted file mode 100644 index 61776a3..0000000 --- a/steward/templates/ping/index.html +++ /dev/null @@ -1,47 +0,0 @@ -{% extends "base.html" %} -{% block title %}Ping Monitor — Steward{% endblock %} -{% block content %} -
-
-

Ping Monitor

- polling every {{ poll_interval }}s -
- {% include "_time_range.html" %} -
- -{# ── Threshold settings ─────────────────────────────────────────────────── #} -
-

Latency Thresholds

-
-
- - -
-
- - -
- - - Above warn = ■ orange  |  No response = ■ red - -
-
- -{# ── Live host rows ─────────────────────────────────────────────────────── #} -
-
-
-
-{% endblock %} diff --git a/steward/templates/ping/rows.html b/steward/templates/ping/rows.html deleted file mode 100644 index 468fe2e..0000000 --- a/steward/templates/ping/rows.html +++ /dev/null @@ -1,67 +0,0 @@ -{# HTMX fragment — included in both /ping/ page and dashboard widget #} -{% macro pill_bg(p) %} -{%- if p is none or p.status.value == 'down' or p.response_time_ms is none -%} - #6a1515 -{%- elif p.response_time_ms <= good_ms -%} - #1a6632 -{%- elif p.response_time_ms <= warn_ms -%} - #6b5c00 -{%- else -%} - #7a3800 -{%- endif -%} -{% endmacro %} -{% macro pill_title(p) %} -{%- if p is none -%}No data -{%- elif p.status.value == 'down' -%}Down — {{ p.probed_at.strftime('%H:%M:%S') }} UTC -{%- else -%}{{ "%.1f"|format(p.response_time_ms) }} ms — {{ p.probed_at.strftime('%H:%M:%S') }} UTC -{%- endif -%} -{% endmacro %} -{% if hosts %} -{% for host in hosts %} -{% set host_pings = pings_by_host.get(host.id, []) %} -{% set last = host_pings[-1] if host_pings else none %} -
-
- {# Link to the host hub for logged-in users; plain text on the public share view (no auth, no token on the href). #} - {% if session.user_id %} - {{ host.name }} - {% else %} - {{ host.name }} - {% endif %} - {{ host.address }} -
-
- {% for _ in range([30 - host_pings|length, 0]|max) %} - - {% endfor %} - {% for p in host_pings %} - - {% endfor %} -
-
- {% if last is none %} - — - {% elif last.status.value == 'down' or last.response_time_ms is none %} - DOWN - {% elif last.response_time_ms <= good_ms %} - {{ "%.1f"|format(last.response_time_ms) }} ms - {% elif last.response_time_ms <= warn_ms %} - {{ "%.1f"|format(last.response_time_ms) }} ms - {% else %} - {{ "%.1f"|format(last.response_time_ms) }} ms - {% endif %} -
- {% if uptime_pcts is defined %} - {% set pct = uptime_pcts.get(host.id) %} - {% set us = threshold_style(pct, 'uptime') if pct is not none else 'color:var(--text-dim);' %} -
- {% if pct is not none %}{{ "%.1f"|format(pct) }}%{% else %}—{% endif %} - {{ range_key }} -
- {% endif %} -
-{% endfor %} -{% else %} -

No ping-enabled hosts. Add hosts →

-{% endif %} diff --git a/tests/core/test_monitors.py b/tests/core/test_monitors.py new file mode 100644 index 0000000..71e10d3 --- /dev/null +++ b/tests/core/test_monitors.py @@ -0,0 +1,110 @@ +"""Unit tests for the unified Monitor: dispatch, config plumbing, form parsing. + +No DB / no network — the per-type probes are monkeypatched so we exercise the +run_monitor dispatcher and the route helpers in isolation. The DB-backed status +source + migration are covered in the integration lane. +""" +import asyncio + +from steward.models.monitors import Monitor, MONITOR_TYPE_VALUES +from steward.monitors import runner as runner_mod +from steward.monitors.routes import _build_config, _normalise_target + + +def _monitor(mtype, target="example.com", cfg=None): + m = Monitor(id="m1", name="t", type=mtype, target=target) + m.set_config(cfg or {}) + return m + + +def test_monitor_type_values(): + assert MONITOR_TYPE_VALUES == ("icmp", "tcp", "dns", "http") + + +def test_config_json_roundtrip(): + m = _monitor("tcp", cfg={"port": 8443}) + assert m.config == {"port": 8443} + m.set_config({"port": 22}) + assert m.config["port"] == 22 + + +def test_config_handles_bad_json(): + m = Monitor(id="x", name="t", type="tcp", target="h", config_json="not json") + assert m.config == {} + + +def test_run_monitor_unknown_type_is_safe(): + m = _monitor("bogus") + res = asyncio.run(runner_mod.run_monitor(m)) + assert res["is_up"] is False + assert "bogus" in res["error_msg"] + + +def test_run_monitor_tcp_uses_config_port(monkeypatch): + seen = {} + + async def fake_tcp(address, port): + seen["address"], seen["port"] = address, port + return {"is_up": True, "response_ms": 12.0} + + monkeypatch.setattr(runner_mod, "tcp_check", fake_tcp) + res = asyncio.run(runner_mod.run_monitor(_monitor("tcp", "db.local", {"port": 5432}))) + assert seen == {"address": "db.local", "port": 5432} + assert res["is_up"] is True and res["response_ms"] == 12.0 + # Untouched fields stay None (full MonitorResult superset is always present). + assert res["status_code"] is None and res["resolved_ip"] is None + + +def test_run_monitor_http_passes_config(monkeypatch): + seen = {} + + async def fake_http(**kwargs): + seen.update(kwargs) + return {"is_up": True, "status_code": 200, "response_ms": 5.0} + + monkeypatch.setattr(runner_mod, "http_check", fake_http) + cfg = {"method": "HEAD", "expected_status": 204, "content_match": "ok", + "headers": {"X": "1"}, "timeout_seconds": 3, + "follow_redirects": False, "verify_ssl": False} + res = asyncio.run(runner_mod.run_monitor(_monitor("http", "https://x.test", cfg))) + assert seen["method"] == "HEAD" and seen["expected_status"] == 204 + assert seen["headers"] == {"X": "1"} and seen["verify_ssl"] is False + assert res["status_code"] == 200 + + +def test_run_monitor_dns_passes_expected_ip(monkeypatch): + seen = {} + + async def fake_dns(address, expected_ip=None): + seen["address"], seen["expected_ip"] = address, expected_ip + return {"is_up": True, "resolved_ip": "192.0.2.1"} + + monkeypatch.setattr(runner_mod, "dns_check", fake_dns) + res = asyncio.run(runner_mod.run_monitor(_monitor("dns", "host.test", {"expected_ip": "192.0.2.1"}))) + assert seen == {"address": "host.test", "expected_ip": "192.0.2.1"} + assert res["resolved_ip"] == "192.0.2.1" + + +# ── route form helpers ──────────────────────────────────────────────────────── + +def test_normalise_target_adds_https_for_bare_http_host(): + assert _normalise_target("http", "example.com") == "https://example.com" + assert _normalise_target("http", "http://x.com") == "http://x.com" + assert _normalise_target("tcp", "example.com") == "example.com" + + +class _Form(dict): + """Minimal stand-in for a Quart form (dict + 'in' membership for checkboxes).""" + + +def test_build_config_per_type(): + assert _build_config(_Form(port="8080"), "tcp") == {"port": 8080} + assert _build_config(_Form(), "tcp") == {"port": 80} + assert _build_config(_Form(expected_ip="10.0.0.1"), "dns") == {"expected_ip": "10.0.0.1"} + assert _build_config(_Form(), "dns") == {"expected_ip": None} + assert _build_config(_Form(), "icmp") == {} + + http = _build_config(_Form(method="get", expected_status="201", follow_redirects="on"), "http") + assert http["method"] == "GET" and http["expected_status"] == 201 + assert http["follow_redirects"] is True # present in form → checked + assert http["verify_ssl"] is False # absent from form → unchecked diff --git a/tests/integration/test_monitors.py b/tests/integration/test_monitors.py new file mode 100644 index 0000000..0e0ebd7 --- /dev/null +++ b/tests/integration/test_monitors.py @@ -0,0 +1,85 @@ +"""Integration: unified Monitor schema + status source against live Postgres. + +Validates the 0022 unification migration applied (monitors/monitor_results +exist, the old per-type tables are gone) and that monitor_status_source rolls +up MonitorResult history into a StatusEntry. Requires STEWARD_DATABASE_URL. +""" +from __future__ import annotations + +import asyncio +import os +import uuid +from datetime import datetime, timezone, timedelta + +import pytest + +pytestmark = pytest.mark.integration + +_NEEDS_DB = pytest.mark.skipif( + not os.environ.get("STEWARD_DATABASE_URL"), + reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)", +) + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +@_NEEDS_DB +def test_old_monitor_tables_dropped(app): + from sqlalchemy import text + + async def _go(): + async with app.db_sessionmaker() as s: + # New tables exist… + await s.execute(text("SELECT COUNT(*) FROM monitors")) + await s.execute(text("SELECT COUNT(*) FROM monitor_results")) + # …old ones are gone (regclass of a missing table is NULL). + for tbl in ("ping_results", "dns_results", "http_monitors", "http_results"): + missing = (await s.execute( + text("SELECT to_regclass(:t)"), {"t": tbl})).scalar() + assert missing is None, f"{tbl} should have been dropped" + + asyncio.run(_go()) + + +@_NEEDS_DB +def test_status_source_rolls_up_results(app): + from sqlalchemy import text + from steward.models.monitors import Monitor, MonitorResult + from steward.core.status import monitor_status_source + + mid = str(uuid.uuid4()) + now = datetime.now(timezone.utc) + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + # Clean slate for a deterministic assertion. + await s.execute(text("DELETE FROM monitor_results")) + await s.execute(text("DELETE FROM monitors")) + s.add(Monitor(id=mid, name="custom-http", type="http", + target="https://example.test", host_id=None, + config_json="{}", enabled=True)) + async with s.begin(): + for i, up in enumerate([True, True, False, True]): + s.add(MonitorResult( + id=str(uuid.uuid4()), monitor_id=mid, + checked_at=now - timedelta(minutes=4 - i), + is_up=up, response_ms=10.0 if up else None, + )) + entries = await monitor_status_source(s) + return entries + + entries = asyncio.run(_go()) + assert len(entries) == 1 + e = entries[0] + assert e.kind == "http" and e.name == "custom-http" + assert e.target == "https://example.test" + assert e.status == "up" # latest result is up + assert len(e.heartbeat) == 4 + assert e.uptime["24h"] == 75.0 # 3 of 4 up From 7b80552a7de876c85106b843cc53095f8d857708 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 17:54:36 -0400 Subject: [PATCH 087/126] feat(docker): per-host collection via the host agent; drop central scrape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker collection moves off the central single-socket scrape onto the host agent, giving Docker a real per-host dimension. The Steward host now reports its own containers like any other host, and same-named containers on different hosts no longer collide. - agent: stdlib UDS Docker client (AF_UNIX HTTP/1.1, Connection: close, chunked-aware), collect_docker() ports the cpu%/mem math; sample["docker"] added best-effort (silent-skip on absent/unreadable socket). AGENT_VERSION 1.2.0 → 1.3.0; optional docker_socket config key. - ingest: host_agent ingest hands per-host container snapshots to the docker plugin via a new "docker.persist_host_samples" capability (no hard import, no-op when docker disabled), inside a SAVEPOINT so a docker failure never sinks the host metrics. Resource names are host-scoped ("/"). - schema: docker_containers re-keyed (host_id, name); docker_metrics gains host_id; docker_002 migration DROP+recreates (dev-only, rule 122). - ui: Docker page + widgets grouped by host with host links; new per-host Docker panel embedded on the Hosts hub (gated on docker enabled via a new enabled_plugins template context). Replaces the SQLite-only strftime bucketing with DB-agnostic Python bucketing. - provisioning: install/provision playbooks add steward-agent to the docker group (best-effort) so the agent can read the socket. - removed central scrape: docker scheduler.py + scraper.py deleted; plugin.yaml socket_path/scrape_interval_seconds/include_stopped dropped (plugin 2.0.0). - tests: agent docker collector units (math, chunked decode, silent-skip, sample shape, config) + integration (host-scoped schema + persistence). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- plugins/docker/__init__.py | 20 +- plugins/docker/ingest.py | 89 ++++++++ .../versions/docker_002_host_scoped.py | 94 +++++++++ plugins/docker/models.py | 27 ++- plugins/docker/plugin.yaml | 11 +- plugins/docker/routes.py | 192 ++++++++++++----- plugins/docker/scheduler.py | 93 -------- plugins/docker/scraper.py | 139 ------------ .../docker/templates/docker/host_panel.html | 22 ++ plugins/docker/templates/docker/rows.html | 32 ++- plugins/docker/templates/docker/widget.html | 13 +- .../templates/docker/widget_resources.html | 9 +- plugins/host_agent/agent.py | 199 +++++++++++++++++- plugins/host_agent/routes.py | 24 +++ plugins/index.yaml | 6 +- .../ansible/bundled/host_agent/install.yml | 17 ++ .../ansible/bundled/host_agent/provision.yml | 17 ++ steward/app.py | 12 +- steward/templates/hosts/detail.html | 5 + tests/integration/test_docker.py | 109 ++++++++++ tests/plugins/host_agent/test_agent_docker.py | 157 ++++++++++++++ 21 files changed, 964 insertions(+), 323 deletions(-) create mode 100644 plugins/docker/ingest.py create mode 100644 plugins/docker/migrations/versions/docker_002_host_scoped.py delete mode 100644 plugins/docker/scheduler.py delete mode 100644 plugins/docker/scraper.py create mode 100644 plugins/docker/templates/docker/host_panel.html create mode 100644 tests/integration/test_docker.py create mode 100644 tests/plugins/host_agent/test_agent_docker.py diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py index d569556..4272f51 100644 --- a/plugins/docker/__init__.py +++ b/plugins/docker/__init__.py @@ -13,10 +13,26 @@ def setup(app: "Quart") -> None: _app = app from .models import DockerContainer, DockerMetric # noqa: F401 — register with Base + # Publish the per-host persist hook so the host agent can hand off the + # `docker` array from each sample without importing our models (opportunistic + # synergy — host_agent gates on has_capability and no-ops if we're disabled). + # required_role=viewer: this is a trusted server-side data-plane write driven + # by the authenticated agent ingest, not a user-facing privileged action. + from steward.core.capabilities import register_capability + from steward.models.users import UserRole + from .ingest import persist_host_docker + register_capability( + "docker.persist_host_samples", persist_host_docker, + label="Persist host Docker samples", + description="Store per-host container state + metrics pushed by the host agent.", + required_role=UserRole.viewer, + ) + def get_scheduled_tasks() -> list: - from .scheduler import make_task - return [make_task(_app)] + # Collection is now agent-driven (pushed to the host_agent ingest); the + # central socket scrape was removed. No periodic task to register. + return [] def get_blueprint(): diff --git a/plugins/docker/ingest.py b/plugins/docker/ingest.py new file mode 100644 index 0000000..1b715d6 --- /dev/null +++ b/plugins/docker/ingest.py @@ -0,0 +1,89 @@ +# plugins/docker/ingest.py +"""Persist host-scoped Docker samples pushed by the host agent. + +Published as the "docker.persist_host_samples" capability (see __init__.setup), +so the host_agent plugin can hand off the `docker` array from a sample WITHOUT +importing the docker models — the coupling is opportunistic and degrades to a +no-op when the docker plugin is disabled. Runs inside the caller's open +transaction (the ingest handler's session); never opens or commits its own. +""" +from __future__ import annotations + +import json +from datetime import datetime + + +def _parse_started_at(value) -> datetime | None: + """Parse the agent's ISO-8601 started_at string back to a datetime.""" + if not isinstance(value, str) or not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +async def persist_host_docker(session, host, snapshots) -> None: + """Upsert containers + append time-series for one host's docker snapshots. + + `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) — + one entry per ingested sample carrying docker data (usually just one; more + when the agent flushes a backlog). Every snapshot contributes time-series + DockerMetric rows; the newest snapshot drives current container state and + the alert pipeline (record_metric writes "now", so feeding it stale buffered + snapshots would be misleading). + """ + from steward.core.alerts import record_metric + from .models import DockerContainer, DockerMetric + + if not snapshots: + return + ordered = sorted(snapshots, key=lambda s: s[0]) + latest_at, latest_containers = ordered[-1] + + # Time-series points for every snapshot (running containers with a CPU read). + for recorded_at, containers in ordered: + for c in containers: + if c.get("status") == "running" and c.get("cpu_pct") is not None: + session.add(DockerMetric( + host_id=host.id, + container_name=c["name"], + scraped_at=recorded_at, + cpu_pct=c["cpu_pct"], + mem_pct=c.get("mem_pct") or 0.0, + mem_usage_bytes=c.get("mem_usage_bytes") or 0, + )) + + # Current state + alerts from the newest snapshot only. + for c in latest_containers: + name = c.get("name") + if not name: + continue + existing = await session.get(DockerContainer, (host.id, name)) + if existing is None: + existing = DockerContainer(host_id=host.id, name=name) + session.add(existing) + existing.container_id = c.get("container_id", "") or "" + existing.image = c.get("image", "") or "" + existing.status = c.get("status", "unknown") or "unknown" + existing.cpu_pct = c.get("cpu_pct") + existing.mem_usage_bytes = c.get("mem_usage_bytes") + existing.mem_limit_bytes = c.get("mem_limit_bytes") + existing.mem_pct = c.get("mem_pct") + existing.ports_json = json.dumps(c.get("ports") or []) + existing.started_at = _parse_started_at(c.get("started_at")) + existing.scraped_at = latest_at + + # Alert pipeline — resource is host-scoped so containers of the same name + # on different hosts don't collide in the metric/alert namespace. + if c.get("status") == "running" and c.get("cpu_pct") is not None: + resource = f"{host.name}/{name}" + await record_metric( + session=session, source_module="docker", + resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"], + ) + if c.get("mem_pct") is not None: + await record_metric( + session=session, source_module="docker", + resource_name=resource, metric_name="mem_pct", value=c["mem_pct"], + ) diff --git a/plugins/docker/migrations/versions/docker_002_host_scoped.py b/plugins/docker/migrations/versions/docker_002_host_scoped.py new file mode 100644 index 0000000..aaefa83 --- /dev/null +++ b/plugins/docker/migrations/versions/docker_002_host_scoped.py @@ -0,0 +1,94 @@ +"""Docker collection goes per-host: add host_id, re-key by (host_id, name) + +Collection moved from the central single-socket scrape to the host agent, so +containers are now scoped to the host that reported them. docker_containers is +re-keyed (host_id, name) and docker_metrics gains host_id. + +Dev-only posture (rule 122): the old tables only ever held the Steward box's +own containers (a single global namespace), which are disposable — so this +DROP+recreates rather than backfilling a host_id onto orphan rows. + +Revision ID: docker_002_host_scoped +Revises: docker_001_initial +Create Date: 2026-06-18 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_002_host_scoped" +down_revision: Union[str, None] = "docker_001_initial" +branch_labels: Union[str, Sequence[str], None] = None +# FK targets hosts.id (created in 0002_core_monitors) — make the edge explicit. +depends_on: Union[str, Sequence[str], None] = ("0002_core_monitors",) + + +def upgrade() -> None: + op.drop_table("docker_metrics") + op.drop_table("docker_containers") + + op.create_table( + "docker_containers", + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True), + sa.Column("name", sa.String(255), primary_key=True), + sa.Column("container_id", sa.String(64), nullable=False, server_default=""), + sa.Column("image", sa.String(512), nullable=False, server_default=""), + sa.Column("status", sa.String(32), nullable=False, server_default="unknown"), + sa.Column("cpu_pct", sa.Float, nullable=True), + sa.Column("mem_usage_bytes", sa.Integer, nullable=True), + sa.Column("mem_limit_bytes", sa.Integer, nullable=True), + sa.Column("mem_pct", sa.Float, nullable=True), + sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"), + sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "docker_metrics", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False), + sa.Column("container_name", sa.String(255), nullable=False), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"), + ) + op.create_index("ix_docker_metrics_host_id", "docker_metrics", ["host_id"]) + op.create_index("ix_docker_metrics_container_name", "docker_metrics", + ["container_name"]) + op.create_index("ix_docker_metrics_scraped_at", "docker_metrics", ["scraped_at"]) + op.create_index("ix_docker_metrics_host_container_time", "docker_metrics", + ["host_id", "container_name", "scraped_at"]) + + +def downgrade() -> None: + op.drop_table("docker_metrics") + op.drop_table("docker_containers") + + op.create_table( + "docker_containers", + sa.Column("name", sa.String(255), primary_key=True), + sa.Column("container_id", sa.String(64), nullable=False, server_default=""), + sa.Column("image", sa.String(512), nullable=False, server_default=""), + sa.Column("status", sa.String(32), nullable=False, server_default="unknown"), + sa.Column("cpu_pct", sa.Float, nullable=True), + sa.Column("mem_usage_bytes", sa.Integer, nullable=True), + sa.Column("mem_limit_bytes", sa.Integer, nullable=True), + sa.Column("mem_pct", sa.Float, nullable=True), + sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"), + sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_table( + "docker_metrics", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("container_name", sa.String(255), nullable=False, index=True), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True), + sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"), + sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"), + ) diff --git a/plugins/docker/models.py b/plugins/docker/models.py index bec356b..b966455 100644 --- a/plugins/docker/models.py +++ b/plugins/docker/models.py @@ -2,16 +2,24 @@ from __future__ import annotations import uuid from datetime import datetime, timezone -from sqlalchemy import DateTime, Float, Integer, String, Text +from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column from steward.models.base import Base class DockerContainer(Base): - """Latest known state per container — upserted by name on each scrape.""" + """Latest known state per container, scoped to the host that reported it. + + Collection is per-host via the host agent, so container names are only + unique within a host — the natural key is (host_id, name). host_id is NOT + NULL: every container arrives through a host_agent ingest that resolves a + Host first. Deleting a host cascades its containers away. + """ __tablename__ = "docker_containers" - # Container name is the stable natural key; container_id changes on recreation + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True + ) name: Mapped[str] = mapped_column(String(255), primary_key=True) container_id: Mapped[str] = mapped_column(String(64), nullable=False, default="") image: Mapped[str] = mapped_column(String(512), nullable=False, default="") @@ -32,12 +40,16 @@ class DockerContainer(Base): class DockerMetric(Base): - """Time-series CPU/memory per container — one row per scrape per running container.""" + """Time-series CPU/memory per container — one row per sample per running + container, scoped to the reporting host.""" __tablename__ = "docker_metrics" id: Mapped[str] = mapped_column( String(36), primary_key=True, default=lambda: str(uuid.uuid4()) ) + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True + ) container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True) scraped_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, @@ -47,3 +59,10 @@ class DockerMetric(Base): cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) mem_usage_bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + # Per-container history lookups filter on (host_id, container_name) then sort + # by time — a composite index keeps the rows() sparkline queries cheap. + __table_args__ = ( + Index("ix_docker_metrics_host_container_time", + "host_id", "container_name", "scraped_at"), + ) diff --git a/plugins/docker/plugin.yaml b/plugins/docker/plugin.yaml index 9d38f24..1237a62 100644 --- a/plugins/docker/plugin.yaml +++ b/plugins/docker/plugin.yaml @@ -1,6 +1,6 @@ name: docker -version: "1.0.0" -description: "Docker container status, resource usage, restart tracking via Docker socket" +version: "2.0.0" +description: "Per-host Docker container status + resource usage, collected by the host agent" author: "Steward" license: "MIT" min_app_version: "0.1.0" @@ -11,7 +11,6 @@ tags: - docker - infrastructure -config: - socket_path: /var/run/docker.sock - scrape_interval_seconds: 60 - include_stopped: false +# No config: collection is agent-driven (the host agent reads each host's local +# socket and pushes containers to the host_agent ingest). This plugin is pure +# presentation + storage, so there's no socket path or scrape interval to tune. diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index 41b123a..5eaae65 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -2,11 +2,12 @@ from __future__ import annotations import json from quart import Blueprint, current_app, render_template, request -from sqlalchemy import Integer, cast, func, select +from sqlalchemy import select from steward.auth.middleware import require_role from steward.models.users import UserRole -from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds +from steward.models.hosts import Host +from steward.core.time_range import parse_range, DEFAULT_RANGE from .models import DockerContainer, DockerMetric docker_bp = Blueprint("docker", __name__, template_folder="templates") @@ -33,6 +34,38 @@ def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: ) +def _bucket(values: list[float], target: int = 40) -> list[float]: + """Bucket-average a series down to ~target points (DB-agnostic, in Python). + + Replaces the old SQL strftime() bucketing, which was SQLite-only and broke + on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of + rows — averaging into a readable point count keeps the sparkline's shape. + """ + n = len(values) + if n <= target: + return values + size = (n + target - 1) // target + out: list[float] = [] + for i in range(0, n, size): + chunk = values[i:i + size] + out.append(sum(chunk) / len(chunk)) + return out + + +async def _container_history(db, host_id: str, name: str, since) -> tuple[list, list]: + """Return (cpu_series, mem_series) sparkline-ready for one host's container.""" + rows = (await db.execute( + select(DockerMetric.cpu_pct, DockerMetric.mem_pct) + .where(DockerMetric.host_id == host_id) + .where(DockerMetric.container_name == name) + .where(DockerMetric.scraped_at >= since) + .order_by(DockerMetric.scraped_at) + )).all() + cpu = _bucket([r.cpu_pct or 0.0 for r in rows]) + mem = _bucket([r.mem_pct or 0.0 for r in rows]) + return cpu, mem + + @docker_bp.get("/") @require_role(UserRole.viewer) async def index(): @@ -45,64 +78,64 @@ async def index(): ) +async def _host_map(db, host_ids: set[str]) -> dict[str, Host]: + if not host_ids: + return {} + return { + h.id: h for h in (await db.execute( + select(Host).where(Host.id.in_(host_ids)))).scalars().all() + } + + @docker_bp.get("/rows") @require_role(UserRole.viewer) async def rows(): - """HTMX fragment: container list with status and resource sparklines.""" + """HTMX fragment: containers grouped by host, with resource sparklines.""" since, range_key = parse_range(request.args.get("range")) - b_secs = bucket_seconds(since) - - bucket_col = ( - cast(func.strftime('%s', DockerMetric.scraped_at), Integer) / b_secs - ).label("bucket") async with current_app.db_sessionmaker() as db: - # All known containers ordered by running first, then name - result = await db.execute( - select(DockerContainer) - .order_by( - # running first + containers = list((await db.execute( + select(DockerContainer).order_by( (DockerContainer.status == "running").desc(), DockerContainer.name, ) - ) - containers = list(result.scalars()) + )).scalars()) + hosts = await _host_map(db, {c.host_id for c in containers}) - # Build per-container sparkline histories - histories: dict[str, list] = {} + # Group by host so each container is clearly attributed to the box it + # runs on (names are only unique within a host now). + groups: dict[str, dict] = {} for c in containers: - result = await db.execute( - select( - func.avg(DockerMetric.cpu_pct).label("cpu_pct"), - func.avg(DockerMetric.mem_pct).label("mem_pct"), - bucket_col, - ) - .where(DockerMetric.container_name == c.name) - .where(DockerMetric.scraped_at >= since) - .group_by(bucket_col) - .order_by(bucket_col) - ) - histories[c.name] = result.all() - - running = sum(1 for c in containers if c.status == "running") - stopped = len(containers) - running - - container_data = [] - for c in containers: - hist = histories.get(c.name, []) - ports = json.loads(c.ports_json) if c.ports_json else [] - container_data.append({ - "container": c, - "ports": ports, - "sparkline_cpu": _sparkline([r.cpu_pct or 0 for r in hist]), - "sparkline_mem": _sparkline([r.mem_pct or 0 for r in hist]), - }) + g = groups.get(c.host_id) + if g is None: + host = hosts.get(c.host_id) + g = groups[c.host_id] = { + "host": host, + "host_id": c.host_id, + "host_name": host.name if host else c.host_id, + "containers": [], "running": 0, "stopped": 0, + } + cpu_hist, mem_hist = await _container_history(db, c.host_id, c.name, since) + g["containers"].append({ + "container": c, + "ports": json.loads(c.ports_json) if c.ports_json else [], + "sparkline_cpu": _sparkline(cpu_hist), + "sparkline_mem": _sparkline(mem_hist), + }) + if c.status == "running": + g["running"] += 1 + else: + g["stopped"] += 1 + host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower()) + running = sum(g["running"] for g in host_groups) + total = len(containers) return await render_template( "docker/rows.html", - container_data=container_data, + host_groups=host_groups, running=running, - stopped=stopped, + stopped=total - running, + total=total, range_key=range_key, ) @@ -110,27 +143,38 @@ async def rows(): @docker_bp.get("/widget") @require_role(UserRole.viewer) async def widget(): - """HTMX dashboard widget: container status overview.""" + """HTMX dashboard widget: container status overview, grouped by host.""" show_stopped = request.args.get("show_stopped", "no") == "yes" widget_id = request.args.get("wid", "0") async with current_app.db_sessionmaker() as db: - result = await db.execute( - select(DockerContainer) - .order_by( + all_containers = list((await db.execute( + select(DockerContainer).order_by( (DockerContainer.status == "running").desc(), DockerContainer.name, ) - ) - all_containers = list(result.scalars()) + )).scalars()) + hosts = await _host_map(db, {c.host_id for c in all_containers}) running = [c for c in all_containers if c.status == "running"] stopped = [c for c in all_containers if c.status != "running"] display = all_containers if show_stopped else running + # Group for display so multi-host fleets read clearly; single-host stays flat. + groups: dict[str, dict] = {} + for c in display: + g = groups.setdefault(c.host_id, { + "host_id": c.host_id, + "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id, + "containers": [], + }) + g["containers"].append(c) + host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower()) + return await render_template( "docker/widget.html", - containers=display, + host_groups=host_groups, + multi_host=len(host_groups) > 1, running_count=len(running), stopped_count=len(stopped), show_stopped=show_stopped, @@ -141,20 +185,54 @@ async def widget(): @docker_bp.get("/widget/resources") @require_role(UserRole.viewer) async def widget_resources(): - """HTMX dashboard widget: CPU + memory usage for running containers.""" + """HTMX dashboard widget: CPU + memory usage for the busiest containers.""" limit = max(1, min(20, int(request.args.get("limit", 10) or 10))) widget_id = request.args.get("wid", "0") async with current_app.db_sessionmaker() as db: - result = await db.execute( + containers = list((await db.execute( select(DockerContainer) .where(DockerContainer.status == "running") .order_by(DockerContainer.cpu_pct.desc().nullslast()) - ) - containers = list(result.scalars())[:limit] + )).scalars())[:limit] + hosts = await _host_map(db, {c.host_id for c in containers}) + rows_data = [ + {"c": c, "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id} + for c in containers + ] return await render_template( "docker/widget_resources.html", - containers=containers, + rows=rows_data, + multi_host=len({c.host_id for c in containers}) > 1, widget_id=widget_id, ) + + +@docker_bp.get("/host/") +@require_role(UserRole.viewer) +async def host_panel(host_id: str): + """Per-host Docker fragment embedded on the core Hosts hub via HTMX. + + Renders nothing when the host reports no containers, so hosts without Docker + don't carry an empty card. + """ + async with current_app.db_sessionmaker() as db: + containers = list((await db.execute( + select(DockerContainer) + .where(DockerContainer.host_id == host_id) + .order_by( + (DockerContainer.status == "running").desc(), + DockerContainer.name, + ) + )).scalars()) + + if not containers: + return "" + running = sum(1 for c in containers if c.status == "running") + return await render_template( + "docker/host_panel.html", + containers=containers, + running=running, + stopped=len(containers) - running, + ) diff --git a/plugins/docker/scheduler.py b/plugins/docker/scheduler.py deleted file mode 100644 index 827cf5f..0000000 --- a/plugins/docker/scheduler.py +++ /dev/null @@ -1,93 +0,0 @@ -# plugins/docker/scheduler.py -from __future__ import annotations -import json -import logging -from datetime import datetime, timezone - -from steward.core.scheduler import ScheduledTask -from steward.core.alerts import record_metric - -logger = logging.getLogger(__name__) - - -def make_task(app) -> ScheduledTask: - interval = int( - app.config["PLUGINS"]["docker"].get("scrape_interval_seconds", 60) - ) - - async def scrape(): - await _do_scrape(app) - - return ScheduledTask( - name="docker_scrape", - coro_factory=scrape, - interval_seconds=interval, - run_on_startup=True, - ) - - -async def _do_scrape(app) -> None: - from .scraper import scrape_docker - from .models import DockerContainer, DockerMetric - - cfg = app.config["PLUGINS"]["docker"] - socket_path = cfg.get("socket_path", "/var/run/docker.sock") - include_stopped = bool(cfg.get("include_stopped", False)) - - try: - containers = await scrape_docker(socket_path, include_stopped) - except ConnectionError as exc: - logger.error("Docker scrape failed: %s", exc) - return - except Exception: - logger.exception("Docker scrape error") - return - - now = datetime.now(timezone.utc) - - async with app.db_sessionmaker() as session: - async with session.begin(): - for c in containers: - # Upsert container state - existing = await session.get(DockerContainer, c["name"]) - if existing is None: - existing = DockerContainer(name=c["name"]) - session.add(existing) - existing.container_id = c["container_id"] - existing.image = c["image"] - existing.status = c["status"] - existing.cpu_pct = c["cpu_pct"] - existing.mem_usage_bytes = c["mem_usage_bytes"] - existing.mem_limit_bytes = c["mem_limit_bytes"] - existing.mem_pct = c["mem_pct"] - existing.restart_count = c["restart_count"] - existing.ports_json = json.dumps(c["ports"]) - existing.started_at = c["started_at"] - existing.scraped_at = now - - # Time-series metric (running containers only) - if c["status"] == "running" and c["cpu_pct"] is not None: - session.add(DockerMetric( - container_name=c["name"], - scraped_at=now, - cpu_pct=c["cpu_pct"], - mem_pct=c["mem_pct"] or 0.0, - mem_usage_bytes=c["mem_usage_bytes"] or 0, - )) - - # Feed alert pipeline - await record_metric( - session=session, - source_module="docker", - resource_name=c["name"], - metric_name="cpu_pct", - value=c["cpu_pct"], - ) - if c["mem_pct"] is not None: - await record_metric( - session=session, - source_module="docker", - resource_name=c["name"], - metric_name="mem_pct", - value=c["mem_pct"], - ) diff --git a/plugins/docker/scraper.py b/plugins/docker/scraper.py deleted file mode 100644 index f4133ed..0000000 --- a/plugins/docker/scraper.py +++ /dev/null @@ -1,139 +0,0 @@ -# plugins/docker/scraper.py -"""Docker API client via Unix socket.""" -from __future__ import annotations -import asyncio -import logging -from datetime import datetime, timezone - -import httpx - -logger = logging.getLogger(__name__) - - -def _short_id(container_id: str) -> str: - return container_id[:12] if container_id else "" - - -def _calc_cpu_pct(stats: dict) -> float: - """Calculate CPU % from a Docker stats snapshot (one-shot).""" - try: - cpu = stats.get("cpu_stats", {}) - precpu = stats.get("precpu_stats", {}) - cpu_delta = ( - cpu["cpu_usage"]["total_usage"] - precpu["cpu_usage"]["total_usage"] - ) - sys_delta = cpu.get("system_cpu_usage", 0) - precpu.get("system_cpu_usage", 0) - num_cpus = cpu.get("online_cpus") or len( - cpu["cpu_usage"].get("percpu_usage") or [None] - ) - if sys_delta <= 0 or cpu_delta < 0: - return 0.0 - return round((cpu_delta / sys_delta) * num_cpus * 100.0, 2) - except (KeyError, TypeError, ZeroDivisionError): - return 0.0 - - -def _calc_mem(stats: dict) -> tuple[int, int, float]: - """Return (usage_bytes, limit_bytes, mem_pct) from Docker stats.""" - try: - mem = stats.get("memory_stats", {}) - usage = mem.get("usage", 0) - limit = mem.get("limit", 0) - # Subtract page cache for working set (cgroup v1: "cache", v2: "inactive_file") - cache = mem.get("stats", {}).get("cache", 0) or \ - mem.get("stats", {}).get("inactive_file", 0) - actual = max(0, usage - cache) - pct = round(actual / limit * 100.0, 2) if limit > 0 else 0.0 - return actual, limit, pct - except (KeyError, TypeError): - return 0, 0, 0.0 - - -def _parse_ports(ports: list) -> list[dict]: - """Normalise Docker port bindings to a compact list.""" - result = [] - for p in (ports or []): - if p.get("PublicPort"): - result.append({ - "host_port": p["PublicPort"], - "container_port": p["PrivatePort"], - "protocol": p.get("Type", "tcp"), - }) - return result - - -async def scrape_docker(socket_path: str, include_stopped: bool) -> list[dict]: - """ - Fetch all containers + resource stats from the Docker daemon. - Returns a list of normalised dicts ready for the scheduler to persist. - Raises ConnectionError if the socket is unreachable. - """ - transport = httpx.AsyncHTTPTransport(uds=socket_path) - try: - async with httpx.AsyncClient( - transport=transport, - base_url="http://docker", - timeout=10.0, - ) as client: - resp = await client.get( - "/containers/json", - params={"all": "true" if include_stopped else "false"}, - ) - resp.raise_for_status() - containers = resp.json() - - async def _get_stats(c: dict) -> tuple[dict, dict | None]: - if c.get("State") != "running": - return c, None - try: - r = await client.get( - f"/containers/{c['Id']}/stats", - params={"stream": "false", "one-shot": "true"}, - timeout=5.0, - ) - return c, r.json() - except Exception as exc: - logger.debug( - "Stats unavailable for %s: %s", _short_id(c.get("Id", "")), exc - ) - return c, None - - pairs = await asyncio.gather(*[_get_stats(c) for c in containers]) - - except httpx.ConnectError as exc: - raise ConnectionError( - f"Cannot connect to Docker socket at {socket_path}: {exc}" - ) from exc - - results = [] - for container, stats in pairs: - names = container.get("Names") or [] - name = names[0].lstrip("/") if names else _short_id(container.get("Id", "")) - - cpu_pct = mem_usage = mem_limit = mem_pct = None - if stats is not None: - cpu_pct = _calc_cpu_pct(stats) - mem_usage, mem_limit, mem_pct = _calc_mem(stats) - - # Created is a Unix epoch int from /containers/json - created_ts = container.get("Created") - started_at = ( - datetime.fromtimestamp(created_ts, tz=timezone.utc) - if isinstance(created_ts, (int, float)) else None - ) - - results.append({ - "name": name, - "container_id": _short_id(container.get("Id", "")), - "image": container.get("Image", ""), - "status": container.get("State", "unknown"), - "cpu_pct": cpu_pct, - "mem_usage_bytes": mem_usage, - "mem_limit_bytes": mem_limit, - "mem_pct": mem_pct, - "restart_count": 0, # would require /inspect; left as 0 for now - "ports": _parse_ports(container.get("Ports", [])), - "started_at": started_at, - }) - - return results diff --git a/plugins/docker/templates/docker/host_panel.html b/plugins/docker/templates/docker/host_panel.html new file mode 100644 index 0000000..1666174 --- /dev/null +++ b/plugins/docker/templates/docker/host_panel.html @@ -0,0 +1,22 @@ +{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub #} +
+
+

Docker

+ + {{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %} + All → + +
+
+ {% for c in containers %} +
+ + {{ c.name }} + + {% if c.cpu_pct is not none %}{{ "%.1f" | format(c.cpu_pct) }}%{% endif %} + {% if c.mem_pct is not none %} · {{ "%.1f" | format(c.mem_pct) }}%{% endif %} + +
+ {% endfor %} +
+
diff --git a/plugins/docker/templates/docker/rows.html b/plugins/docker/templates/docker/rows.html index 3b6c84e..686609e 100644 --- a/plugins/docker/templates/docker/rows.html +++ b/plugins/docker/templates/docker/rows.html @@ -1,4 +1,4 @@ -{# docker/rows.html — HTMX fragment for the Docker main page #} +{# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #} {# ── Summary strip ─────────────────────────────────────────────────────────── #}
@@ -12,13 +12,28 @@
Total
- {{ container_data | length }} + {{ total }} +
+
+
Hosts
+ {{ host_groups | length }}
-{# ── Container table ─────────────────────────────────────────────────────── #} -{% if container_data %} -
+{# ── Container tables, one per host ───────────────────────────────────────── #} +{% if host_groups %} +{% for g in host_groups %} +
+
+ {% if g.host %} + {{ g.host.name }} + {% else %} + {{ g.host_name }} + {% endif %} + + {{ g.running }} running{% if g.stopped %} · {{ g.stopped }} stopped{% endif %} + +
@@ -32,7 +47,7 @@ - {% for item in container_data %} + {% for item in g.containers %} {% set c = item.container %}
@@ -77,10 +92,13 @@
+{% endfor %} {% else %}
- No containers found. Make sure the Docker socket is accessible and the plugin is configured correctly. + No containers reported yet. Containers are collected by the Steward host agent — + deploy the agent to a host running Docker (Hosts → the host → agent panel) and + its containers will appear here under that host.
{% endif %} diff --git a/plugins/docker/templates/docker/widget.html b/plugins/docker/templates/docker/widget.html index 061df4b..9a8a5a4 100644 --- a/plugins/docker/templates/docker/widget.html +++ b/plugins/docker/templates/docker/widget.html @@ -1,6 +1,6 @@ -{# docker/widget.html — dashboard widget: container status overview #} -{% if not containers and running_count == 0 and stopped_count == 0 %} -

No containers found.

+{# docker/widget.html — dashboard widget: container status overview, by host #} +{% if running_count == 0 and stopped_count == 0 %} +

No containers reported yet.

{% else %}
@@ -16,8 +16,12 @@ {% endif %}
+{% for g in host_groups %} +{% if multi_host %} +
{{ g.host_name }}
+{% endif %}
- {% for c in containers %} + {% for c in g.containers %}
@@ -31,5 +35,6 @@
{% endfor %}
+{% endfor %} {% endif %} diff --git a/plugins/docker/templates/docker/widget_resources.html b/plugins/docker/templates/docker/widget_resources.html index 39c90dc..3358bc3 100644 --- a/plugins/docker/templates/docker/widget_resources.html +++ b/plugins/docker/templates/docker/widget_resources.html @@ -1,13 +1,14 @@ -{# docker/widget_resources.html — dashboard widget: CPU + memory bars for running containers #} -{% if not containers %} +{# docker/widget_resources.html — dashboard widget: CPU + memory bars, busiest containers #} +{% if not rows %}
No running containers.
{% else %}
- {% for c in containers %} + {% for r in rows %} + {% set c = r.c %}
- {{ c.name }} + {{ c.name }}{% if multi_host %} · {{ r.host_name }}{% endif %} {% if c.cpu_pct is not none %}CPU {{ "%.1f" | format(c.cpu_pct) }}%{% endif %} diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index f78cef0..3e8356d 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -19,7 +19,12 @@ import urllib.request from collections import deque from datetime import datetime, timezone -AGENT_VERSION = "1.2.0" +AGENT_VERSION = "1.3.0" + +# Default path to the local Docker Engine socket. Overridable via the +# `docker_socket` config key; collection is silently skipped if it's absent or +# unreadable, so this never needs setting on a Docker-less host (zero-config). +DEFAULT_DOCKER_SOCKET = "/var/run/docker.sock" class ConfigError(Exception): @@ -62,6 +67,7 @@ def read_config(path: str) -> dict: raise ConfigError(f"{path}: missing required key(s): {', '.join(missing)}") cfg.setdefault("interval_seconds", 30) + cfg.setdefault("docker_socket", DEFAULT_DOCKER_SOCKET) return cfg @@ -386,6 +392,180 @@ def _rates(cur: dict, prev: dict, dt: float) -> dict: return out +# ─── docker ────────────────────────────────────────────────────────────────── +# +# Per-host container collection over the local Docker socket. stdlib-only: a +# minimal HTTP/1.1 client over an AF_UNIX socket. We send `Connection: close` +# so the daemon closes the socket at end of response and we can read to EOF; +# Docker still frames the body with Transfer-Encoding: chunked, so we de-chunk +# when that header is present rather than assume Content-Length. + +DOCKER_API_TIMEOUT = 5.0 + + +def _dechunk(body: bytes) -> bytes: + """Decode an HTTP/1.1 chunked-transfer body into the raw payload.""" + out = bytearray() + while body: + line, sep, rest = body.partition(b"\r\n") + if not sep: + break + try: + size = int(line.split(b";", 1)[0], 16) # ignore chunk extensions + except ValueError: + break + if size == 0: + break + out += rest[:size] + body = rest[size + 2:] # skip the chunk data and its trailing CRLF + return bytes(out) + + +def _docker_request(socket_path: str, path: str, timeout: float = DOCKER_API_TIMEOUT): + """GET `path` from the Docker Engine API over a Unix socket; return JSON. + + Raises OSError on any socket/transport problem (absent socket, permission + denied, non-200) and ValueError on a non-JSON body — both are caught by the + caller and treated as "no docker here", so collection degrades silently. + """ + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.settimeout(timeout) + try: + sock.connect(socket_path) + req = ( + "GET " + path + " HTTP/1.1\r\n" + "Host: docker\r\n" + "Accept: application/json\r\n" + "Connection: close\r\n" + "\r\n" + ) + sock.sendall(req.encode("ascii")) + chunks = [] + while True: + buf = sock.recv(65536) + if not buf: + break + chunks.append(buf) + finally: + sock.close() + + raw = b"".join(chunks) + head, _, body = raw.partition(b"\r\n\r\n") + header_text = head.decode("latin-1") + status_line = header_text.split("\r\n", 1)[0] + parts = status_line.split(None, 2) # "HTTP/1.1 200 OK" + status = int(parts[1]) if len(parts) >= 2 and parts[1].isdigit() else 0 + if status != 200: + raise OSError(f"docker API {path} returned {status}") + if "transfer-encoding: chunked" in header_text.lower(): + body = _dechunk(body) + return json.loads(body.decode("utf-8")) + + +def _docker_cpu_pct(stats: dict) -> float: + """CPU % from a Docker stats snapshot (delta of container vs system CPU). + + Ported verbatim from the old central scraper's math; correct as long as both + cpu_stats and precpu_stats are populated (we request without `one-shot` so + the daemon fills precpu over one cycle). + """ + try: + cpu = stats.get("cpu_stats", {}) + precpu = stats.get("precpu_stats", {}) + cpu_delta = ( + cpu["cpu_usage"]["total_usage"] - precpu["cpu_usage"]["total_usage"] + ) + sys_delta = cpu.get("system_cpu_usage", 0) - precpu.get("system_cpu_usage", 0) + num_cpus = cpu.get("online_cpus") or len( + cpu["cpu_usage"].get("percpu_usage") or [None] + ) + if sys_delta <= 0 or cpu_delta < 0: + return 0.0 + return round((cpu_delta / sys_delta) * num_cpus * 100.0, 2) + except (KeyError, TypeError, ZeroDivisionError): + return 0.0 + + +def _docker_mem(stats: dict): + """Return (usage_bytes, limit_bytes, mem_pct); working set excludes cache.""" + try: + mem = stats.get("memory_stats", {}) + usage = mem.get("usage", 0) + limit = mem.get("limit", 0) + # Subtract page cache for working set (cgroup v1 "cache", v2 "inactive_file"). + cache = mem.get("stats", {}).get("cache", 0) or \ + mem.get("stats", {}).get("inactive_file", 0) + actual = max(0, usage - cache) + pct = round(actual / limit * 100.0, 2) if limit > 0 else 0.0 + return actual, limit, pct + except (KeyError, TypeError): + return 0, 0, 0.0 + + +def _docker_ports(ports: list) -> list: + """Normalise Docker port bindings to a compact published-ports list.""" + out = [] + for p in (ports or []): + if p.get("PublicPort"): + out.append({ + "host_port": p["PublicPort"], + "container_port": p.get("PrivatePort"), + "protocol": p.get("Type", "tcp"), + }) + return out + + +def collect_docker(socket_path: str) -> list: + """Per-container state from the local Docker socket, or [] if unavailable. + + Absent socket / permission denied / non-Docker endpoint all yield [] — the + agent silently reports no containers rather than erroring, so a Docker-less + host needs no configuration. + """ + try: + containers = _docker_request(socket_path, "/containers/json?all=true") + except (OSError, ValueError): + return [] + if not isinstance(containers, list): + return [] + + out = [] + for c in containers: + cid = c.get("Id", "") or "" + names = c.get("Names") or [] + name = names[0].lstrip("/") if names else cid[:12] + state = c.get("State", "unknown") + + cpu_pct = mem_usage = mem_limit = mem_pct = None + if state == "running": + try: + stats = _docker_request( + socket_path, f"/containers/{cid}/stats?stream=false") + cpu_pct = _docker_cpu_pct(stats) + mem_usage, mem_limit, mem_pct = _docker_mem(stats) + except (OSError, ValueError): + pass # stats unavailable for this container — leave gauges None + + created = c.get("Created") + started_at = ( + datetime.fromtimestamp(created, tz=timezone.utc).isoformat() + if isinstance(created, (int, float)) else None + ) + out.append({ + "name": name, + "container_id": cid[:12], + "image": c.get("Image", ""), + "status": state, + "cpu_pct": cpu_pct, + "mem_usage_bytes": mem_usage, + "mem_limit_bytes": mem_limit, + "mem_pct": mem_pct, + "ports": _docker_ports(c.get("Ports", [])), + "started_at": started_at, + }) + return out + + # ─── ring buffer ───────────────────────────────────────────────────────────── @@ -408,11 +588,14 @@ class RingBuffer: # ─── payload ───────────────────────────────────────────────────────────────── -def build_sample(mounts: list[str], state: dict) -> dict: +def build_sample(mounts: list[str], state: dict, + docker_socket: str = DEFAULT_DOCKER_SOCKET) -> dict: """Collect one full sample. Partial samples allowed if a collector fails. `state` carries the previous network/disk counters + monotonic timestamp so throughput rates can be derived from deltas; it is mutated in place. + `docker_socket` is probed best-effort — the `docker` key is omitted entirely + when no containers are found, so non-Docker hosts add nothing to the payload. """ sample: dict = {"ts": datetime.now(timezone.utc).isoformat()} try: @@ -466,6 +649,14 @@ def build_sample(mounts: list[str], state: dict) -> dict: for n, (rd, wr) in _rates(disk_raw, state.get("disk", {}), dt).items() ] state["net"], state["disk"], state["mono"] = net_raw, disk_raw, now_mono + + try: + docker = collect_docker(docker_socket) + except Exception: + docker = [] + if docker: + sample["docker"] = docker + return sample @@ -547,6 +738,7 @@ def main_loop(conf_path: str) -> int: metadata = collect_metadata() hostname = cfg.get("hostname") or socket.gethostname() mounts = cfg.get("mounts") or default_mounts() + docker_socket = cfg.get("docker_socket") or DEFAULT_DOCKER_SOCKET buffer = RingBuffer(maxlen=20) backoff = 0 # Carries previous net/disk counters + monotonic ts for rate computation. @@ -560,13 +752,14 @@ def main_loop(conf_path: str) -> int: try: cfg = read_config(conf_path) mounts = cfg.get("mounts") or default_mounts() + docker_socket = cfg.get("docker_socket") or DEFAULT_DOCKER_SOCKET metadata = collect_metadata() # refresh host_ip/distro on reload _log("INFO", "config reloaded") except ConfigError as e: _log("ERROR", f"reload failed: {e}") _reload_requested = False - sample = build_sample(mounts, rate_state) + sample = build_sample(mounts, rate_state, docker_socket) buffered = buffer.drain() payload = build_payload( samples=buffered + [sample], diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index d8c66b1..9670569 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -196,6 +196,7 @@ async def ingest(): accepted = 0 latest_ts: datetime | None = None + docker_snapshots: list[tuple[datetime, list]] = [] for sample in samples: try: recorded_at = _parse_ts(sample["ts"]) @@ -204,6 +205,9 @@ async def ingest(): metrics = _expand_sample_to_metrics(sample, host.name, recorded_at) for m in metrics: session.add(m) + docker = sample.get("docker") + if isinstance(docker, list) and docker: + docker_snapshots.append((recorded_at, docker)) accepted += 1 if latest_ts is None or recorded_at > latest_ts: latest_ts = recorded_at @@ -212,6 +216,26 @@ async def ingest(): await session.rollback() return _error(400, "malformed_payload", "no valid samples") + # Hand host-scoped container data to the docker plugin if it's enabled + # (opportunistic synergy via the capability registry — no hard import, + # no-op when docker is disabled). A failure here must never sink the + # whole ingest, so the metrics above still land. + if docker_snapshots: + from steward.core.capabilities import has_capability, invoke_capability + if has_capability("docker.persist_host_samples"): + try: + # SAVEPOINT so a docker-side failure rolls back only the + # docker writes, leaving the host metrics intact to commit. + async with session.begin_nested(): + await invoke_capability( + "docker.persist_host_samples", UserRole.admin, + session, host, docker_snapshots, + ) + except Exception: + current_app.logger.exception( + "host_agent ingest: docker persist failed for host=%s", + host.name) + md = payload.get("metadata") or {} changed = False if latest_ts and (reg.last_seen_at is None or latest_ts > reg.last_seen_at): diff --git a/plugins/index.yaml b/plugins/index.yaml index 0198bc3..3d5fe8e 100644 --- a/plugins/index.yaml +++ b/plugins/index.yaml @@ -14,14 +14,14 @@ updated: "2026-03-22" plugins: - name: docker - version: "1.0.0" - description: "Docker container status, resource usage, and restart tracking via Docker socket" + version: "2.0.0" + description: "Per-host Docker container status + resource usage, collected by the Steward host agent" author: "Steward" license: "MIT" min_app_version: "0.1.0" repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins" homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker" - download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v1.0.0/docker.zip" + download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v2.0.0/docker.zip" checksum_sha256: "" tags: - containers diff --git a/steward/ansible/bundled/host_agent/install.yml b/steward/ansible/bundled/host_agent/install.yml index a1074d2..ce50939 100644 --- a/steward/ansible/bundled/host_agent/install.yml +++ b/steward/ansible/bundled/host_agent/install.yml @@ -29,6 +29,23 @@ shell: /usr/sbin/nologin create_home: false + # Let the agent read the local Docker socket so it can report this host's + # containers (the per-host docker collection path). Best-effort: skipped on + # hosts without Docker. Restart picks up the new supplementary group. + - name: Detect docker group + ansible.builtin.command: getent group docker + register: _docker_grp + changed_when: false + failed_when: false + + - name: Add steward-agent to the docker group + ansible.builtin.user: + name: steward-agent + groups: docker + append: true + when: _docker_grp.rc == 0 + notify: restart steward-agent + - name: Create agent directory ansible.builtin.file: path: /usr/local/lib/steward-agent diff --git a/steward/ansible/bundled/host_agent/provision.yml b/steward/ansible/bundled/host_agent/provision.yml index 09b50ec..21b62ac 100644 --- a/steward/ansible/bundled/host_agent/provision.yml +++ b/steward/ansible/bundled/host_agent/provision.yml @@ -85,6 +85,23 @@ shell: /usr/sbin/nologin create_home: false + # Let the agent read the local Docker socket so it can report this host's + # containers (the per-host docker collection path). Best-effort: skipped on + # hosts without Docker. Restart picks up the new supplementary group. + - name: Detect docker group + ansible.builtin.command: getent group docker + register: _docker_grp + changed_when: false + failed_when: false + + - name: Add steward-agent to the docker group + ansible.builtin.user: + name: steward-agent + groups: docker + append: true + when: _docker_grp.rc == 0 + notify: restart steward-agent + - name: Create agent directory ansible.builtin.file: path: /usr/local/lib/steward-agent diff --git a/steward/app.py b/steward/app.py index 7410bbb..29b26f4 100644 --- a/steward/app.py +++ b/steward/app.py @@ -167,7 +167,17 @@ def create_app( @app.context_processor def _inject_plugin_failures(): from .core.plugin_manager import get_plugin_failures - return {"plugin_failures": get_plugin_failures()} + # enabled_plugins lets templates gate cross-plugin embeds (e.g. the + # Hosts hub only fetches the docker fragment when docker is enabled), + # avoiding a 404 to a route whose blueprint isn't registered. + enabled_plugins = { + name for name, cfg in (app.config.get("PLUGINS") or {}).items() + if isinstance(cfg, dict) and cfg.get("enabled") + } + return { + "plugin_failures": get_plugin_failures(), + "enabled_plugins": enabled_plugins, + } # ── 11. Share-token middleware ───────────────────────────────────────────── @app.before_request diff --git a/steward/templates/hosts/detail.html b/steward/templates/hosts/detail.html index 9651b43..81fb23e 100644 --- a/steward/templates/hosts/detail.html +++ b/steward/templates/hosts/detail.html @@ -122,6 +122,11 @@
Loading agent…
+{# ── Docker (docker plugin fragment; renders nothing if the host has none) ──── #} +{% if "docker" in enabled_plugins %} +
+{% endif %} + {# ── Ansible ──────────────────────────────────────────────────────────────── #}

Ansible

diff --git a/tests/integration/test_docker.py b/tests/integration/test_docker.py new file mode 100644 index 0000000..9271e3c --- /dev/null +++ b/tests/integration/test_docker.py @@ -0,0 +1,109 @@ +"""Integration: per-host Docker schema + host-scoped persistence. + +Validates the docker_002 migration (host_id columns + composite key) applied and +that persist_host_docker scopes containers/metrics by host so same-named +containers on different hosts no longer collide. Requires STEWARD_DATABASE_URL. +""" +from __future__ import annotations + +import asyncio +import os +import uuid +from datetime import datetime, timezone + +import pytest + +pytestmark = pytest.mark.integration + +_NEEDS_DB = pytest.mark.skipif( + not os.environ.get("STEWARD_DATABASE_URL"), + reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)", +) + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +@_NEEDS_DB +def test_docker_tables_are_host_scoped(app): + from sqlalchemy import text + + async def _go(): + async with app.db_sessionmaker() as s: + # Both tables carry a host_id column after docker_002. + for tbl in ("docker_containers", "docker_metrics"): + cols = {r[0] for r in (await s.execute(text( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = :t"), {"t": tbl})).all()} + assert "host_id" in cols, f"{tbl} missing host_id" + # docker_containers is keyed by (host_id, name), not name alone. + pk_cols = {r[0] for r in (await s.execute(text( + "SELECT a.attname FROM pg_index i " + "JOIN pg_attribute a ON a.attrelid = i.indrelid " + " AND a.attnum = ANY(i.indkey) " + "WHERE i.indrelid = 'docker_containers'::regclass AND i.indisprimary" + ))).all()} + assert pk_cols == {"host_id", "name"} + + asyncio.run(_go()) + + +def _persist_fn(app): + """Resolve persist_host_docker via the registered capability if the docker + plugin is loaded, else import it directly (the import is safe only when the + plugin is NOT loaded — otherwise its models are already mapped).""" + from steward.core.capabilities import has_capability, get_capability + if has_capability("docker.persist_host_samples"): + return get_capability("docker.persist_host_samples").fn + from plugins.docker.ingest import persist_host_docker + return persist_host_docker + + +@_NEEDS_DB +def test_persist_scopes_containers_by_host(app): + from sqlalchemy import text + from steward.models.hosts import Host + + persist = _persist_fn(app) + now = datetime.now(timezone.utc) + snapshot = [(now, [{ + "name": "web", "container_id": "abc123", "image": "nginx", + "status": "running", "cpu_pct": 12.5, "mem_pct": 30.0, + "mem_usage_bytes": 100, "mem_limit_bytes": 200, "ports": [], "started_at": None, + }])] + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + await s.execute(text("DELETE FROM docker_metrics")) + await s.execute(text("DELETE FROM docker_containers")) + await s.execute(text( + "DELETE FROM plugin_metrics WHERE source_module = 'docker'")) + h1 = Host(id=str(uuid.uuid4()), name="alpha", address="10.0.0.1") + h2 = Host(id=str(uuid.uuid4()), name="beta", address="10.0.0.2") + s.add_all([h1, h2]) + await s.flush() + # Same container name on two hosts — must not collide now. + await persist(s, h1, snapshot) + await persist(s, h2, snapshot) + + web_rows = (await s.execute(text( + "SELECT COUNT(*) FROM docker_containers WHERE name = 'web'"))).scalar() + metric_hosts = (await s.execute(text( + "SELECT COUNT(DISTINCT host_id) FROM docker_metrics " + "WHERE container_name = 'web'"))).scalar() + # Alert pipeline received host-scoped resource names. + resources = {r[0] for r in (await s.execute(text( + "SELECT DISTINCT resource_name FROM plugin_metrics " + "WHERE source_module = 'docker'"))).all()} + return web_rows, metric_hosts, resources + + web_rows, metric_hosts, resources = asyncio.run(_go()) + assert web_rows == 2 # one 'web' per host, keyed by (host_id, name) + assert metric_hosts == 2 # time-series rows scoped per host + assert resources == {"alpha/web", "beta/web"} diff --git a/tests/plugins/host_agent/test_agent_docker.py b/tests/plugins/host_agent/test_agent_docker.py new file mode 100644 index 0000000..6ab4b7f --- /dev/null +++ b/tests/plugins/host_agent/test_agent_docker.py @@ -0,0 +1,157 @@ +# tests/plugins/host_agent/test_agent_docker.py +"""Unit tests for the agent's stdlib Docker collector. + +No socket / no network — the UDS request layer is monkeypatched so we exercise +the cpu/mem math, chunked-body decoding, sample assembly, and the silent-skip +contract in isolation. +""" +from plugins.host_agent import agent as a + + +# ── chunked transfer decoding ───────────────────────────────────────────────── + +def test_dechunk_reassembles_body(): + chunked = b"4\r\nWiki\r\n5\r\npedia\r\n0\r\n\r\n" + assert a._dechunk(chunked) == b"Wikipedia" + + +def test_dechunk_ignores_chunk_extensions(): + assert a._dechunk(b"3;name=v\r\nabc\r\n0\r\n\r\n") == b"abc" + + +# ── cpu / mem math (ported from the old central scraper) ────────────────────── + +def test_docker_cpu_pct(): + stats = { + "cpu_stats": { + "cpu_usage": {"total_usage": 200_000_000}, + "system_cpu_usage": 2_000_000_000, + "online_cpus": 4, + }, + "precpu_stats": { + "cpu_usage": {"total_usage": 100_000_000}, + "system_cpu_usage": 1_000_000_000, + }, + } + # cpu_delta=1e8, sys_delta=1e9 → 0.1 * 4 cpus * 100 = 40.0% + assert a._docker_cpu_pct(stats) == 40.0 + + +def test_docker_cpu_pct_guards_bad_stats(): + assert a._docker_cpu_pct({}) == 0.0 + # No forward progress in system time → 0, never a divide error. + assert a._docker_cpu_pct({ + "cpu_stats": {"cpu_usage": {"total_usage": 5}, "system_cpu_usage": 10}, + "precpu_stats": {"cpu_usage": {"total_usage": 5}, "system_cpu_usage": 10}, + }) == 0.0 + + +def test_docker_mem_subtracts_cache(): + stats = {"memory_stats": { + "usage": 200 * 1024 * 1024, + "limit": 1000 * 1024 * 1024, + "stats": {"cache": 50 * 1024 * 1024}, + }} + usage, limit, pct = a._docker_mem(stats) + assert usage == 150 * 1024 * 1024 # working set = usage − cache + assert limit == 1000 * 1024 * 1024 + assert pct == 15.0 + + +def test_docker_ports_keeps_only_published(): + ports = [ + {"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"}, + {"PrivatePort": 5432, "Type": "tcp"}, # unpublished → dropped + ] + assert a._docker_ports(ports) == [ + {"host_port": 8080, "container_port": 80, "protocol": "tcp"}, + ] + + +# ── collect_docker assembly + silent skip ───────────────────────────────────── + +def test_collect_docker_silent_skip_when_no_socket(): + # Absent socket path must degrade to [] (zero-config on Docker-less hosts). + assert a.collect_docker("/nonexistent/does-not-exist.sock") == [] + + +def test_collect_docker_assembles_container(monkeypatch): + container = { + "Id": "abc123def4567890", + "Names": ["/web"], + "Image": "nginx:latest", + "State": "running", + "Created": 1_700_000_000, + "Ports": [{"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"}], + } + stats = { + "cpu_stats": {"cpu_usage": {"total_usage": 200_000_000}, + "system_cpu_usage": 2_000_000_000, "online_cpus": 4}, + "precpu_stats": {"cpu_usage": {"total_usage": 100_000_000}, + "system_cpu_usage": 1_000_000_000}, + "memory_stats": {"usage": 200 * 1024 * 1024, "limit": 1000 * 1024 * 1024, + "stats": {"cache": 50 * 1024 * 1024}}, + } + + def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): + return [container] if path.startswith("/containers/json") else stats + + monkeypatch.setattr(a, "_docker_request", fake_request) + out = a.collect_docker("/var/run/docker.sock") + assert len(out) == 1 + c = out[0] + assert c["name"] == "web" + assert c["container_id"] == "abc123def456" # 12-char short id + assert c["image"] == "nginx:latest" + assert c["status"] == "running" + assert c["cpu_pct"] == 40.0 + assert c["mem_pct"] == 15.0 + assert c["ports"] == [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}] + assert c["started_at"].startswith("2023-11-14T") # epoch → ISO-8601 + + +def test_collect_docker_skips_stats_for_stopped(monkeypatch): + container = {"Id": "x" * 16, "Names": ["/db"], "Image": "pg", "State": "exited", + "Created": None, "Ports": []} + + def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): + if path.startswith("/containers/json"): + return [container] + raise AssertionError("stats must not be fetched for a stopped container") + + monkeypatch.setattr(a, "_docker_request", fake_request) + out = a.collect_docker("/var/run/docker.sock") + assert out[0]["status"] == "exited" + assert out[0]["cpu_pct"] is None and out[0]["started_at"] is None + + +# ── build_sample wiring ─────────────────────────────────────────────────────── + +def test_build_sample_includes_docker_when_present(monkeypatch): + monkeypatch.setattr(a, "collect_docker", lambda _s: [{"name": "web", "status": "running"}]) + sample = a.build_sample(["/"], {}, "/var/run/docker.sock") + assert sample["docker"] == [{"name": "web", "status": "running"}] + + +def test_build_sample_omits_docker_when_empty(monkeypatch): + monkeypatch.setattr(a, "collect_docker", lambda _s: []) + sample = a.build_sample(["/"], {}, "/var/run/docker.sock") + assert "docker" not in sample + + +# ── config ──────────────────────────────────────────────────────────────────── + +def test_read_config_defaults_docker_socket(tmp_path): + cfg_file = tmp_path / "agent.conf" + cfg_file.write_text("url = https://steward.example\ntoken = abc\n") + cfg = a.read_config(str(cfg_file)) + assert cfg["docker_socket"] == a.DEFAULT_DOCKER_SOCKET + + +def test_read_config_honours_explicit_docker_socket(tmp_path): + cfg_file = tmp_path / "agent.conf" + cfg_file.write_text( + "url = https://steward.example\ntoken = abc\n" + "docker_socket = /run/user/1000/docker.sock\n") + cfg = a.read_config(str(cfg_file)) + assert cfg["docker_socket"] == "/run/user/1000/docker.sock" From 82c3d2cf36b8f03806e11e993552b7d6e902e715 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 20:37:40 -0400 Subject: [PATCH 088/126] =?UTF-8?q?feat(docker):=20per-container=20enrichm?= =?UTF-8?q?ent=20=E2=80=94=20health,=20restarts,=20exit=20code,=20I/O,=20g?= =?UTF-8?q?rouping?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First slice of milestone 77 (Docker monitoring depth). Surfaces real per-container stats beyond basic state, all read-only on the existing push model. - agent (→1.4.0): collect_docker now inspects each container (health, restart count, exit code, OOM) and reads net + block I/O from the stats payload; pulls compose project + swarm service/task/node from container labels. Per-container inspect+stats calls run over a small bounded ThreadPool so the ~1s-per-stats blocking doesn't stretch the sample on a busy host. - schema (docker_003): additive columns on docker_containers — health, exit_code, oom_killed, compose_project, service_name, task_id, node_id, and BigInteger net/blk byte counters. - ingest: persists the enrichment + restart_count (.get keeps older agents working). - ui: Docker page rows now show health badge, uptime ("up 3d 4h"), restart count, exit code (+OOM) for stopped containers, and compose/service grouping label. - tests: agent helpers (grouping, inspect fields, net/IO sum) + collect_docker assembly incl. inspect; integration asserts enrichment round-trips. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- plugins/docker/ingest.py | 14 ++ .../docker_003_container_enrichment.py | 41 +++++ plugins/docker/models.py | 21 ++- plugins/docker/routes.py | 22 +++ plugins/docker/templates/docker/rows.html | 20 ++- plugins/host_agent/agent.py | 158 ++++++++++++++---- tests/integration/test_docker.py | 14 +- tests/plugins/host_agent/test_agent_docker.py | 71 +++++++- 8 files changed, 317 insertions(+), 44 deletions(-) create mode 100644 plugins/docker/migrations/versions/docker_003_container_enrichment.py diff --git a/plugins/docker/ingest.py b/plugins/docker/ingest.py index 1b715d6..cad4a6b 100644 --- a/plugins/docker/ingest.py +++ b/plugins/docker/ingest.py @@ -73,6 +73,20 @@ async def persist_host_docker(session, host, snapshots) -> None: existing.ports_json = json.dumps(c.get("ports") or []) existing.started_at = _parse_started_at(c.get("started_at")) existing.scraped_at = latest_at + # Enrichment (agent ≥ 1.4.0; .get keeps older agents working — fields + # stay None/0 when absent from the payload). + existing.restart_count = c.get("restart_count", 0) or 0 + existing.health = c.get("health") + existing.exit_code = c.get("exit_code") + existing.oom_killed = bool(c.get("oom_killed", False)) + existing.compose_project = c.get("compose_project") + existing.service_name = c.get("service_name") + existing.task_id = c.get("task_id") + existing.node_id = c.get("node_id") + existing.net_rx_bytes = c.get("net_rx_bytes") + existing.net_tx_bytes = c.get("net_tx_bytes") + existing.blk_read_bytes = c.get("blk_read_bytes") + existing.blk_write_bytes = c.get("blk_write_bytes") # Alert pipeline — resource is host-scoped so containers of the same name # on different hosts don't collide in the metric/alert namespace. diff --git a/plugins/docker/migrations/versions/docker_003_container_enrichment.py b/plugins/docker/migrations/versions/docker_003_container_enrichment.py new file mode 100644 index 0000000..4a7f794 --- /dev/null +++ b/plugins/docker/migrations/versions/docker_003_container_enrichment.py @@ -0,0 +1,41 @@ +"""Docker container enrichment: health, exit/restart, grouping, I/O counters + +Adds the fields the agent (≥1.4.0) now reports per container beyond the basic +state: health status, exit code, OOM flag, compose/swarm grouping labels, and +cumulative network/block I/O counters. Additive columns — no data loss, so no +DROP+recreate needed here. + +Revision ID: docker_003_container_enrichment +Revises: docker_002_host_scoped +Create Date: 2026-06-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_003_container_enrichment" +down_revision: Union[str, None] = "docker_002_host_scoped" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("docker_containers", sa.Column("health", sa.String(16), nullable=True)) + op.add_column("docker_containers", sa.Column("exit_code", sa.Integer, nullable=True)) + op.add_column("docker_containers", + sa.Column("oom_killed", sa.Boolean, nullable=False, server_default=sa.false())) + op.add_column("docker_containers", sa.Column("compose_project", sa.String(255), nullable=True)) + op.add_column("docker_containers", sa.Column("service_name", sa.String(255), nullable=True)) + op.add_column("docker_containers", sa.Column("task_id", sa.String(64), nullable=True)) + op.add_column("docker_containers", sa.Column("node_id", sa.String(64), nullable=True)) + op.add_column("docker_containers", sa.Column("net_rx_bytes", sa.BigInteger, nullable=True)) + op.add_column("docker_containers", sa.Column("net_tx_bytes", sa.BigInteger, nullable=True)) + op.add_column("docker_containers", sa.Column("blk_read_bytes", sa.BigInteger, nullable=True)) + op.add_column("docker_containers", sa.Column("blk_write_bytes", sa.BigInteger, nullable=True)) + + +def downgrade() -> None: + for col in ("blk_write_bytes", "blk_read_bytes", "net_tx_bytes", "net_rx_bytes", + "node_id", "task_id", "service_name", "compose_project", + "oom_killed", "exit_code", "health"): + op.drop_column("docker_containers", col) diff --git a/plugins/docker/models.py b/plugins/docker/models.py index b966455..8fd2ae4 100644 --- a/plugins/docker/models.py +++ b/plugins/docker/models.py @@ -2,7 +2,9 @@ from __future__ import annotations import uuid from datetime import datetime, timezone -from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, String, Text +from sqlalchemy import ( + BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text, +) from sqlalchemy.orm import Mapped, mapped_column from steward.models.base import Base @@ -38,6 +40,23 @@ class DockerContainer(Base): default=lambda: datetime.now(timezone.utc), ) + # ── Enrichment (agent ≥ 1.4.0) ──────────────────────────────────────────── + # Health/exit/restart come from `docker inspect` (not the list endpoint); + # exit_code is only meaningful for stopped containers. + health: Mapped[str | None] = mapped_column(String(16), nullable=True) # healthy|unhealthy|starting + exit_code: Mapped[int | None] = mapped_column(Integer, nullable=True) + oom_killed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + # Grouping: compose project + swarm placement, read off container labels. + compose_project: Mapped[str | None] = mapped_column(String(255), nullable=True) + service_name: Mapped[str | None] = mapped_column(String(255), nullable=True) + task_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + node_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + # Cumulative-since-start I/O counters (BigInteger — they exceed 2^31 quickly). + net_rx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + net_tx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + blk_read_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + blk_write_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + class DockerMetric(Base): """Time-series CPU/memory per container — one row per sample per running diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index 5eaae65..a6b38f2 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -1,6 +1,8 @@ # plugins/docker/routes.py from __future__ import annotations import json +from datetime import datetime, timezone + from quart import Blueprint, current_app, render_template, request from sqlalchemy import select @@ -13,6 +15,25 @@ from .models import DockerContainer, DockerMetric docker_bp = Blueprint("docker", __name__, template_folder="templates") +def _human_uptime(started_at: datetime | None) -> str | None: + """Compact 'how long running' string (e.g. '3d 4h', '5h 12m', '8m').""" + if started_at is None: + return None + if started_at.tzinfo is None: + started_at = started_at.replace(tzinfo=timezone.utc) + secs = int((datetime.now(timezone.utc) - started_at).total_seconds()) + if secs < 0: + return None + d, rem = divmod(secs, 86400) + h, rem = divmod(rem, 3600) + m, _ = divmod(rem, 60) + if d: + return f"{d}d {h}h" + if h: + return f"{h}h {m}m" + return f"{m}m" + + def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: if len(values) < 2: return f'' @@ -119,6 +140,7 @@ async def rows(): g["containers"].append({ "container": c, "ports": json.loads(c.ports_json) if c.ports_json else [], + "uptime": _human_uptime(c.started_at) if c.status == "running" else None, "sparkline_cpu": _sparkline(cpu_hist), "sparkline_mem": _sparkline(mem_hist), }) diff --git a/plugins/docker/templates/docker/rows.html b/plugins/docker/templates/docker/rows.html index 686609e..6dbfca8 100644 --- a/plugins/docker/templates/docker/rows.html +++ b/plugins/docker/templates/docker/rows.html @@ -54,8 +54,24 @@
-
{{ c.name }}
-
{{ c.status }}
+
+ {{ c.name }} + {% if c.health == 'healthy' %}● + {% elif c.health == 'unhealthy' %}● + {% elif c.health == 'starting' %}◐{% endif %} +
+
+ {{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %} + {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %} + · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %} + {% endif %} + {% if c.restart_count %} · ⟳{{ c.restart_count }}{% endif %} +
+ {% if c.service_name or c.compose_project %} +
+ {{ c.service_name or c.compose_project }} +
+ {% endif %}
diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index 3e8356d..227bc2e 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -17,9 +17,10 @@ import time import urllib.error import urllib.request from collections import deque +from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone -AGENT_VERSION = "1.3.0" +AGENT_VERSION = "1.4.0" # Default path to the local Docker Engine socket. Overridable via the # `docker_socket` config key; collection is silently skipped if it's absent or @@ -515,6 +516,117 @@ def _docker_ports(ports: list) -> list: return out +def _docker_grouping(labels: dict) -> dict: + """Pull compose project + swarm service/task/node out of container labels. + + These are set by Docker itself (compose / swarm), so reading them off the + container costs nothing extra and lets Steward group + place containers + without a manager-node query. + """ + labels = labels or {} + return { + "compose_project": labels.get("com.docker.compose.project"), + "service_name": labels.get("com.docker.swarm.service.name"), + "task_id": labels.get("com.docker.swarm.task.id"), + "node_id": labels.get("com.docker.swarm.node.id"), + } + + +def _docker_inspect_fields(insp: dict): + """Return (health, restart_count, exit_code, oom_killed) from an inspect. + + health is None for containers without a HEALTHCHECK (no State.Health). + """ + state = (insp or {}).get("State") or {} + health = (state.get("Health") or {}).get("Status") # healthy|unhealthy|starting + restart_count = insp.get("RestartCount", 0) if isinstance(insp, dict) else 0 + exit_code = state.get("ExitCode") + oom_killed = bool(state.get("OOMKilled", False)) + return health, restart_count, exit_code, oom_killed + + +def _docker_net_io(stats: dict): + """Return cumulative (net_rx, net_tx, blk_read, blk_write) bytes from stats. + + Counters are cumulative since container start (rates can be derived later); + block I/O comes from the cgroup io_service_bytes list, absent on some setups. + """ + net_rx = net_tx = blk_read = blk_write = 0 + for iface in (stats.get("networks") or {}).values(): + net_rx += iface.get("rx_bytes", 0) or 0 + net_tx += iface.get("tx_bytes", 0) or 0 + for e in ((stats.get("blkio_stats") or {}).get("io_service_bytes_recursive") or []): + op = (e.get("op") or "").lower() + if op == "read": + blk_read += e.get("value", 0) or 0 + elif op == "write": + blk_write += e.get("value", 0) or 0 + return net_rx, net_tx, blk_read, blk_write + + +def _collect_one_container(socket_path: str, c: dict) -> dict: + """Build one container's enriched record (runs in a worker thread). + + Does a per-container inspect (health/restart/exit/oom — only available there) + plus a stats read for running containers (cpu/mem/net/io). All best-effort: + a failed call just leaves the affected fields at their defaults. + """ + cid = c.get("Id", "") or "" + names = c.get("Names") or [] + name = names[0].lstrip("/") if names else cid[:12] + state = c.get("State", "unknown") + + # Inspect every container (incl. stopped) — exit codes + restart counts only + # live here, and stopped containers are exactly where exit_code matters. + health = exit_code = None + restart_count = 0 + oom_killed = False + try: + insp = _docker_request(socket_path, f"/containers/{cid}/json") + health, restart_count, exit_code, oom_killed = _docker_inspect_fields(insp) + except (OSError, ValueError): + pass + + cpu_pct = mem_usage = mem_limit = mem_pct = None + net_rx = net_tx = blk_read = blk_write = None + if state == "running": + try: + stats = _docker_request(socket_path, f"/containers/{cid}/stats?stream=false") + cpu_pct = _docker_cpu_pct(stats) + mem_usage, mem_limit, mem_pct = _docker_mem(stats) + net_rx, net_tx, blk_read, blk_write = _docker_net_io(stats) + except (OSError, ValueError): + pass + + created = c.get("Created") + started_at = ( + datetime.fromtimestamp(created, tz=timezone.utc).isoformat() + if isinstance(created, (int, float)) else None + ) + record = { + "name": name, + "container_id": cid[:12], + "image": c.get("Image", ""), + "status": state, + "cpu_pct": cpu_pct, + "mem_usage_bytes": mem_usage, + "mem_limit_bytes": mem_limit, + "mem_pct": mem_pct, + "ports": _docker_ports(c.get("Ports", [])), + "started_at": started_at, + "health": health, + "restart_count": restart_count, + "exit_code": exit_code, + "oom_killed": oom_killed, + "net_rx_bytes": net_rx, + "net_tx_bytes": net_tx, + "blk_read_bytes": blk_read, + "blk_write_bytes": blk_write, + } + record.update(_docker_grouping(c.get("Labels"))) + return record + + def collect_docker(socket_path: str) -> list: """Per-container state from the local Docker socket, or [] if unavailable. @@ -526,44 +638,16 @@ def collect_docker(socket_path: str) -> list: containers = _docker_request(socket_path, "/containers/json?all=true") except (OSError, ValueError): return [] - if not isinstance(containers, list): + if not isinstance(containers, list) or not containers: return [] - out = [] - for c in containers: - cid = c.get("Id", "") or "" - names = c.get("Names") or [] - name = names[0].lstrip("/") if names else cid[:12] - state = c.get("State", "unknown") - - cpu_pct = mem_usage = mem_limit = mem_pct = None - if state == "running": - try: - stats = _docker_request( - socket_path, f"/containers/{cid}/stats?stream=false") - cpu_pct = _docker_cpu_pct(stats) - mem_usage, mem_limit, mem_pct = _docker_mem(stats) - except (OSError, ValueError): - pass # stats unavailable for this container — leave gauges None - - created = c.get("Created") - started_at = ( - datetime.fromtimestamp(created, tz=timezone.utc).isoformat() - if isinstance(created, (int, float)) else None - ) - out.append({ - "name": name, - "container_id": cid[:12], - "image": c.get("Image", ""), - "status": state, - "cpu_pct": cpu_pct, - "mem_usage_bytes": mem_usage, - "mem_limit_bytes": mem_limit, - "mem_pct": mem_pct, - "ports": _docker_ports(c.get("Ports", [])), - "started_at": started_at, - }) - return out + # Each container needs an inspect (+ a stats read if running), and the stats + # call blocks ~1s while the daemon computes the cpu delta. Done serially that + # stretches the whole sample on a busy host, so fan the per-container work out + # over a small bounded thread pool (I/O-bound → threads are enough). + workers = min(8, len(containers)) + with ThreadPoolExecutor(max_workers=workers) as ex: + return list(ex.map(lambda c: _collect_one_container(socket_path, c), containers)) # ─── ring buffer ───────────────────────────────────────────────────────────── diff --git a/tests/integration/test_docker.py b/tests/integration/test_docker.py index 9271e3c..6c5c280 100644 --- a/tests/integration/test_docker.py +++ b/tests/integration/test_docker.py @@ -75,6 +75,11 @@ def test_persist_scopes_containers_by_host(app): "name": "web", "container_id": "abc123", "image": "nginx", "status": "running", "cpu_pct": 12.5, "mem_pct": 30.0, "mem_usage_bytes": 100, "mem_limit_bytes": 200, "ports": [], "started_at": None, + # enrichment (agent ≥ 1.4.0) + "health": "healthy", "restart_count": 2, "exit_code": 0, "oom_killed": False, + "compose_project": "stack", "service_name": "web", + "net_rx_bytes": 1000, "net_tx_bytes": 2000, + "blk_read_bytes": 4096, "blk_write_bytes": 8192, }])] async def _go(): @@ -101,9 +106,14 @@ def test_persist_scopes_containers_by_host(app): resources = {r[0] for r in (await s.execute(text( "SELECT DISTINCT resource_name FROM plugin_metrics " "WHERE source_module = 'docker'"))).all()} - return web_rows, metric_hosts, resources + # Enrichment columns persisted (check one host's row). + enrich = (await s.execute(text( + "SELECT health, restart_count, service_name, net_rx_bytes " + "FROM docker_containers WHERE name = 'web' LIMIT 1"))).first() + return web_rows, metric_hosts, resources, enrich - web_rows, metric_hosts, resources = asyncio.run(_go()) + web_rows, metric_hosts, resources, enrich = asyncio.run(_go()) assert web_rows == 2 # one 'web' per host, keyed by (host_id, name) assert metric_hosts == 2 # time-series rows scoped per host assert resources == {"alpha/web", "beta/web"} + assert enrich == ("healthy", 2, "web", 1000) # enrichment round-trips diff --git a/tests/plugins/host_agent/test_agent_docker.py b/tests/plugins/host_agent/test_agent_docker.py index 6ab4b7f..e734c19 100644 --- a/tests/plugins/host_agent/test_agent_docker.py +++ b/tests/plugins/host_agent/test_agent_docker.py @@ -68,6 +68,47 @@ def test_docker_ports_keeps_only_published(): ] +# ── enrichment helpers ──────────────────────────────────────────────────────── + +def test_docker_grouping_from_labels(): + g = a._docker_grouping({ + "com.docker.compose.project": "myproj", + "com.docker.swarm.service.name": "web", + "com.docker.swarm.task.id": "t123", + "com.docker.swarm.node.id": "n456", + }) + assert g == {"compose_project": "myproj", "service_name": "web", + "task_id": "t123", "node_id": "n456"} + + +def test_docker_grouping_empty(): + assert a._docker_grouping(None) == { + "compose_project": None, "service_name": None, "task_id": None, "node_id": None} + + +def test_docker_inspect_fields(): + insp = {"RestartCount": 3, + "State": {"Health": {"Status": "unhealthy"}, "ExitCode": 137, "OOMKilled": True}} + assert a._docker_inspect_fields(insp) == ("unhealthy", 3, 137, True) + + +def test_docker_inspect_fields_no_healthcheck(): + # Containers without a HEALTHCHECK have no State.Health → health is None. + health, restarts, exit_code, oom = a._docker_inspect_fields( + {"RestartCount": 0, "State": {"ExitCode": 0, "OOMKilled": False}}) + assert health is None and restarts == 0 and exit_code == 0 and oom is False + + +def test_docker_net_io_sums_counters(): + stats = { + "networks": {"eth0": {"rx_bytes": 1000, "tx_bytes": 2000}, + "eth1": {"rx_bytes": 5, "tx_bytes": 7}}, + "blkio_stats": {"io_service_bytes_recursive": [ + {"op": "Read", "value": 4096}, {"op": "Write", "value": 8192}]}, + } + assert a._docker_net_io(stats) == (1005, 2007, 4096, 8192) + + # ── collect_docker assembly + silent skip ───────────────────────────────────── def test_collect_docker_silent_skip_when_no_socket(): @@ -83,7 +124,11 @@ def test_collect_docker_assembles_container(monkeypatch): "State": "running", "Created": 1_700_000_000, "Ports": [{"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"}], + "Labels": {"com.docker.compose.project": "myproj", + "com.docker.swarm.service.name": "web"}, } + inspect = {"RestartCount": 2, + "State": {"Health": {"Status": "healthy"}, "ExitCode": 0, "OOMKilled": False}} stats = { "cpu_stats": {"cpu_usage": {"total_usage": 200_000_000}, "system_cpu_usage": 2_000_000_000, "online_cpus": 4}, @@ -91,10 +136,19 @@ def test_collect_docker_assembles_container(monkeypatch): "system_cpu_usage": 1_000_000_000}, "memory_stats": {"usage": 200 * 1024 * 1024, "limit": 1000 * 1024 * 1024, "stats": {"cache": 50 * 1024 * 1024}}, + "networks": {"eth0": {"rx_bytes": 1000, "tx_bytes": 2000}}, + "blkio_stats": {"io_service_bytes_recursive": [ + {"op": "Read", "value": 4096}, {"op": "Write", "value": 8192}]}, } def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): - return [container] if path.startswith("/containers/json") else stats + if path.startswith("/containers/json"): + return [container] + if "/stats" in path: + return stats + if path.endswith("/json"): # inspect + return inspect + raise AssertionError(f"unexpected path {path}") monkeypatch.setattr(a, "_docker_request", fake_request) out = a.collect_docker("/var/run/docker.sock") @@ -108,21 +162,34 @@ def test_collect_docker_assembles_container(monkeypatch): assert c["mem_pct"] == 15.0 assert c["ports"] == [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}] assert c["started_at"].startswith("2023-11-14T") # epoch → ISO-8601 + # enrichment + assert c["health"] == "healthy" and c["restart_count"] == 2 + assert c["exit_code"] == 0 and c["oom_killed"] is False + assert c["net_rx_bytes"] == 1000 and c["net_tx_bytes"] == 2000 + assert c["blk_read_bytes"] == 4096 and c["blk_write_bytes"] == 8192 + assert c["compose_project"] == "myproj" and c["service_name"] == "web" def test_collect_docker_skips_stats_for_stopped(monkeypatch): container = {"Id": "x" * 16, "Names": ["/db"], "Image": "pg", "State": "exited", - "Created": None, "Ports": []} + "Created": None, "Ports": [], "Labels": {}} + inspect = {"RestartCount": 0, + "State": {"ExitCode": 137, "OOMKilled": True}} # no Health block def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): if path.startswith("/containers/json"): return [container] + if path.endswith("/json"): # inspect — fetched for stopped too (exit code) + return inspect raise AssertionError("stats must not be fetched for a stopped container") monkeypatch.setattr(a, "_docker_request", fake_request) out = a.collect_docker("/var/run/docker.sock") assert out[0]["status"] == "exited" assert out[0]["cpu_pct"] is None and out[0]["started_at"] is None + # inspect still gives us why it died + assert out[0]["exit_code"] == 137 and out[0]["oom_killed"] is True + assert out[0]["health"] is None # ── build_sample wiring ─────────────────────────────────────────────────────── From fee654b53a80ee5a799434cbe03d716ae1a96e68 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 20:47:46 -0400 Subject: [PATCH 089/126] feat(docker): schema for lifecycle events + swarm topology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the milestone-77 storage that doesn't fit on the per-container row: * docker_events — lifecycle (start/stop/die/oom/health_change), to be derived by diffing consecutive host snapshots; host-scoped, indexed for timeline lookups (host_id, container_name, at) and retention pruning (at). * docker_swarm_services / docker_swarm_nodes — manager-reported Swarm topology (desired-vs-running replicas, node role/availability/status). Migration docker_004 extends the docker branch (down_revision docker_003); purely additive, no DROP+recreate. event/mode/role are plain strings (no CHECK whitelist), matching how docker_containers models status. Integration guard asserts the three new host-scoped tables exist. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- .../versions/docker_004_events_swarm.py | 77 +++++++++++++++++ plugins/docker/models.py | 86 +++++++++++++++++++ tests/integration/test_docker.py | 22 +++++ 3 files changed, 185 insertions(+) create mode 100644 plugins/docker/migrations/versions/docker_004_events_swarm.py diff --git a/plugins/docker/migrations/versions/docker_004_events_swarm.py b/plugins/docker/migrations/versions/docker_004_events_swarm.py new file mode 100644 index 0000000..2f236ca --- /dev/null +++ b/plugins/docker/migrations/versions/docker_004_events_swarm.py @@ -0,0 +1,77 @@ +"""Docker lifecycle events + swarm topology tables + +Adds the storage for milestone-77 monitoring depth that doesn't fit on the +per-container row: + + * docker_events — lifecycle (start/stop/die/oom/health_change) derived by + diffing consecutive host snapshots; host-scoped, indexed for both timeline + lookups (host_id, container_name, at) and retention pruning (at). + * docker_swarm_services / docker_swarm_nodes — manager-reported Swarm + topology (desired-vs-running replicas, node role/availability/status). + +All new tables — purely additive, so no DROP+recreate. `event`/`mode`/`role` +etc. are plain strings (no CHECK whitelist), matching how docker_containers +models `status`; keeps the value sets open without a migration per addition +(so rule 36 doesn't apply here). + +Revision ID: docker_004_events_swarm +Revises: docker_003_container_enrichment +Create Date: 2026-06-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_004_events_swarm" +down_revision: Union[str, None] = "docker_003_container_enrichment" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "docker_events", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False), + sa.Column("container_name", sa.String(255), nullable=False), + sa.Column("event", sa.String(16), nullable=False), + sa.Column("detail", sa.String(255), nullable=True), + sa.Column("at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index("ix_docker_events_host_container_time", "docker_events", + ["host_id", "container_name", "at"]) + op.create_index("ix_docker_events_at", "docker_events", ["at"]) + + op.create_table( + "docker_swarm_services", + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True), + sa.Column("service_name", sa.String(255), primary_key=True), + sa.Column("mode", sa.String(16), nullable=False, server_default="replicated"), + sa.Column("desired", sa.Integer, nullable=False, server_default="0"), + sa.Column("running", sa.Integer, nullable=False, server_default="0"), + sa.Column("image", sa.String(512), nullable=True), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "docker_swarm_nodes", + sa.Column("host_id", sa.String(36), + sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True), + sa.Column("node_id", sa.String(64), primary_key=True), + sa.Column("hostname", sa.String(255), nullable=False, server_default=""), + sa.Column("role", sa.String(16), nullable=False, server_default="worker"), + sa.Column("availability", sa.String(16), nullable=False, server_default="active"), + sa.Column("status", sa.String(16), nullable=False, server_default="unknown"), + sa.Column("leader", sa.Boolean, nullable=False, server_default=sa.false()), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("docker_swarm_nodes") + op.drop_table("docker_swarm_services") + op.drop_index("ix_docker_events_at", table_name="docker_events") + op.drop_index("ix_docker_events_host_container_time", table_name="docker_events") + op.drop_table("docker_events") diff --git a/plugins/docker/models.py b/plugins/docker/models.py index 8fd2ae4..cf83bed 100644 --- a/plugins/docker/models.py +++ b/plugins/docker/models.py @@ -85,3 +85,89 @@ class DockerMetric(Base): Index("ix_docker_metrics_host_container_time", "host_id", "container_name", "scraped_at"), ) + + +class DockerEvent(Base): + """Lifecycle events derived by diffing consecutive host snapshots. + + The agent only reports current state, so Steward synthesises lifecycle by + comparing each new snapshot against the stored DockerContainer state for the + host: a container that appears → `start`; one that vanishes → `stop`; a + transition to a stopped status with a non-zero exit → `die`; an OOM-kill + flip → `oom`; a health status change → `health_change`. Scoped to the + reporting host (names are only unique within a host). `event` is a plain + string (no CHECK whitelist), matching how `status` is modelled on + DockerContainer — keeps the value set open without a migration per addition. + """ + __tablename__ = "docker_events" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False + ) + container_name: Mapped[str] = mapped_column(String(255), nullable=False) + event: Mapped[str] = mapped_column(String(16), nullable=False) + # start | stop | die | oom | health_change + detail: Mapped[str | None] = mapped_column(String(255), nullable=True) + # e.g. "exit 137", "healthy→unhealthy", image on start + at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + # Timeline lookups filter on (host_id, container_name) and sort by time; + # retention prunes by `at` alone — index both access paths. + __table_args__ = ( + Index("ix_docker_events_host_container_time", + "host_id", "container_name", "at"), + Index("ix_docker_events_at", "at"), + ) + + +class DockerSwarmService(Base): + """A Swarm service as seen by a manager host: desired vs running replicas. + + Manager-scoped — only a host that is a Swarm manager reports these, so a + single-manager cluster yields one row set keyed by that host. Service names + are unique within a swarm; we still key by (host_id, service_name) so two + managers reporting independently don't collide. + """ + __tablename__ = "docker_swarm_services" + + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True + ) + service_name: Mapped[str] = mapped_column(String(255), primary_key=True) + mode: Mapped[str] = mapped_column(String(16), nullable=False, default="replicated") + # replicated | global + desired: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + running: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + image: Mapped[str | None] = mapped_column(String(512), nullable=True) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + +class DockerSwarmNode(Base): + """A Swarm node as seen by a manager host: role / availability / status.""" + __tablename__ = "docker_swarm_nodes" + + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True + ) + node_id: Mapped[str] = mapped_column(String(64), primary_key=True) + hostname: Mapped[str] = mapped_column(String(255), nullable=False, default="") + role: Mapped[str] = mapped_column(String(16), nullable=False, default="worker") + # manager | worker + availability: Mapped[str] = mapped_column(String(16), nullable=False, default="active") + # active | pause | drain + status: Mapped[str] = mapped_column(String(16), nullable=False, default="unknown") + # ready | down | unknown + leader: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) diff --git a/tests/integration/test_docker.py b/tests/integration/test_docker.py index 6c5c280..2865d65 100644 --- a/tests/integration/test_docker.py +++ b/tests/integration/test_docker.py @@ -53,6 +53,28 @@ def test_docker_tables_are_host_scoped(app): asyncio.run(_go()) +@_NEEDS_DB +def test_events_and_swarm_tables_exist(app): + """docker_004 created the lifecycle + swarm topology tables with host_id FKs.""" + from sqlalchemy import text + + async def _go(): + async with app.db_sessionmaker() as s: + for tbl in ("docker_events", "docker_swarm_services", "docker_swarm_nodes"): + cols = {r[0] for r in (await s.execute(text( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = :t"), {"t": tbl})).all()} + assert cols, f"{tbl} missing entirely" + assert "host_id" in cols, f"{tbl} missing host_id" + # docker_events carries the lifecycle columns the ingest derivation writes. + ev_cols = {r[0] for r in (await s.execute(text( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = 'docker_events'"))).all()} + assert {"event", "container_name", "at", "detail"} <= ev_cols + + asyncio.run(_go()) + + def _persist_fn(app): """Resolve persist_host_docker via the registered capability if the docker plugin is loaded, else import it directly (the import is safe only when the From 448258c5b4ba349fddac595a02a7b75aa513aea4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 20:57:18 -0400 Subject: [PATCH 090/126] feat(docker): agent manager-only swarm collector (AGENT_VERSION 1.5.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds collect_swarm(socket_path) to the host agent. Self-detects a Swarm manager via /info (Swarm.ControlAvailable) — workers and non-swarm daemons return None and never touch the manager-only endpoints (one cheap /info call, no 503s). On a manager it queries /services, /tasks, /nodes and emits sample["swarm"] = {services, nodes}: * services roll desired-vs-running replicas up from the task list (replica health isn't on the service object), handle replicated + global mode, strip the @sha256 image digest, and carry cross-node task→node placement. * nodes normalise role / availability / status + the manager leader flag. build_sample omits the swarm key entirely off managers, same silent contract as collect_docker. Unit tests cover manager detection, replica roll-up + placement (replicated & global), node normalisation, worker silent-skip, and build_sample wiring. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- plugins/host_agent/agent.py | 122 +++++++++++++++++- tests/plugins/host_agent/test_agent_docker.py | 115 +++++++++++++++++ 2 files changed, 236 insertions(+), 1 deletion(-) diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index 227bc2e..d1acc08 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -20,7 +20,7 @@ from collections import deque from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone -AGENT_VERSION = "1.4.0" +AGENT_VERSION = "1.5.0" # Default path to the local Docker Engine socket. Overridable via the # `docker_socket` config key; collection is silently skipped if it's absent or @@ -650,6 +650,117 @@ def collect_docker(socket_path: str) -> list: return list(ex.map(lambda c: _collect_one_container(socket_path, c), containers)) +# ─── swarm (manager-only) ──────────────────────────────────────────────────── + + +def _swarm_is_manager(info: dict) -> bool: + """True only on a reachable Swarm manager. + + `/services`, `/tasks`, `/nodes` are manager-only — a worker (or a non-swarm + daemon) returns 503 for them. `Swarm.ControlAvailable` in `/info` is exactly + "this node can serve the control-plane API", so we gate on it and skip the + topology query everywhere else (no errors, near-zero cost on workers). + """ + return bool((info or {}).get("Swarm", {}).get("ControlAvailable")) + + +def _swarm_services(services: list, tasks: list) -> list: + """Roll desired-vs-running replicas per service up from the task list. + + Replica health isn't on the service object — it's derived by counting tasks: + `running` = tasks currently in state running; `desired` = the service's + configured replica count (replicated) or the count of tasks the scheduler + wants running (global, which has no fixed number). + """ + running_by_svc: dict = {} + desired_by_svc: dict = {} # only used for global-mode services + placement: dict = {} # svc_id -> {node_id: running_count} + for t in (tasks or []): + sid = t.get("ServiceID") + if not sid: + continue + if ((t.get("Status") or {}).get("State") or "").lower() == "running": + running_by_svc[sid] = running_by_svc.get(sid, 0) + 1 + node = t.get("NodeID") + if node: + placement.setdefault(sid, {})[node] = \ + placement.setdefault(sid, {}).get(node, 0) + 1 + if (t.get("DesiredState") or "").lower() == "running": + desired_by_svc[sid] = desired_by_svc.get(sid, 0) + 1 + + out = [] + for s in (services or []): + sid = s.get("ID", "") or "" + spec = s.get("Spec") or {} + name = spec.get("Name") or sid[:12] + mode_obj = spec.get("Mode") or {} + if "Global" in mode_obj: + mode = "global" + desired = desired_by_svc.get(sid, 0) + else: + mode = "replicated" + desired = (mode_obj.get("Replicated") or {}).get("Replicas", 0) or 0 + image = (((spec.get("TaskTemplate") or {}).get("ContainerSpec") or {}) + .get("Image") or "") + image = image.split("@", 1)[0] # drop the @sha256:… digest suffix + out.append({ + "service_name": name, "mode": mode, + "desired": desired, "running": running_by_svc.get(sid, 0), + "image": image, + # task→node placement of the running replicas (cross-node — a manager + # sees every task, even ones whose containers live on other hosts). + "placement": [{"node_id": n, "running": cnt} + for n, cnt in sorted(placement.get(sid, {}).items())], + }) + return out + + +def _swarm_nodes(nodes: list) -> list: + """Normalise node role / availability / status (+ manager leader flag).""" + out = [] + for n in (nodes or []): + spec = n.get("Spec") or {} + desc = n.get("Description") or {} + status = n.get("Status") or {} + mgr = n.get("ManagerStatus") or {} + out.append({ + "node_id": n.get("ID", "") or "", + "hostname": desc.get("Hostname", "") or "", + "role": (spec.get("Role") or "worker").lower(), + "availability": (spec.get("Availability") or "active").lower(), + "status": (status.get("State") or "unknown").lower(), + "leader": bool(mgr.get("Leader", False)), + }) + return out + + +def collect_swarm(socket_path: str): + """Swarm topology from a manager node, or None on workers / non-swarm hosts. + + Self-detects manager via `/info` (one cheap call); only then queries the + manager-only endpoints. Any transport failure degrades to None, same silent + contract as collect_docker — a non-swarm host adds nothing to the payload. + """ + try: + info = _docker_request(socket_path, "/info") + except (OSError, ValueError): + return None + if not _swarm_is_manager(info): + return None + try: + services = _docker_request(socket_path, "/services") + tasks = _docker_request(socket_path, "/tasks") + nodes = _docker_request(socket_path, "/nodes") + except (OSError, ValueError): + return None + return { + "services": _swarm_services( + services if isinstance(services, list) else [], + tasks if isinstance(tasks, list) else []), + "nodes": _swarm_nodes(nodes if isinstance(nodes, list) else []), + } + + # ─── ring buffer ───────────────────────────────────────────────────────────── @@ -741,6 +852,15 @@ def build_sample(mounts: list[str], state: dict, if docker: sample["docker"] = docker + # Swarm topology is only emitted by managers (collect_swarm returns None + # everywhere else), so the key is absent on workers / non-swarm hosts. + try: + swarm = collect_swarm(docker_socket) + except Exception: + swarm = None + if swarm is not None: + sample["swarm"] = swarm + return sample diff --git a/tests/plugins/host_agent/test_agent_docker.py b/tests/plugins/host_agent/test_agent_docker.py index e734c19..67b3654 100644 --- a/tests/plugins/host_agent/test_agent_docker.py +++ b/tests/plugins/host_agent/test_agent_docker.py @@ -192,20 +192,135 @@ def test_collect_docker_skips_stats_for_stopped(monkeypatch): assert out[0]["health"] is None +# ── swarm (manager-only) ────────────────────────────────────────────────────── + +def test_swarm_is_manager(): + assert a._swarm_is_manager({"Swarm": {"ControlAvailable": True}}) is True + # Worker: in a swarm but no control plane. + assert a._swarm_is_manager({"Swarm": {"ControlAvailable": False}}) is False + # Non-swarm daemon: no Swarm block at all. + assert a._swarm_is_manager({}) is False + + +def test_swarm_services_rolls_up_replicas(): + services = [ + {"ID": "svc1", "Spec": { + "Name": "web", "Mode": {"Replicated": {"Replicas": 3}}, + "TaskTemplate": {"ContainerSpec": {"Image": "nginx:1@sha256:deadbeef"}}}}, + {"ID": "svc2", "Spec": { + "Name": "agent", "Mode": {"Global": {}}, + "TaskTemplate": {"ContainerSpec": {"Image": "agent:latest"}}}}, + ] + tasks = [ + # web: 2 of 3 desired replicas actually running, both on node n1 + {"ServiceID": "svc1", "NodeID": "n1", "DesiredState": "running", + "Status": {"State": "running"}}, + {"ServiceID": "svc1", "NodeID": "n1", "DesiredState": "running", + "Status": {"State": "running"}}, + {"ServiceID": "svc1", "NodeID": "n2", "DesiredState": "running", + "Status": {"State": "failed"}}, + # agent: global, 2 tasks desired+running across 2 nodes + {"ServiceID": "svc2", "NodeID": "n1", "DesiredState": "running", + "Status": {"State": "running"}}, + {"ServiceID": "svc2", "NodeID": "n2", "DesiredState": "running", + "Status": {"State": "running"}}, + ] + out = {s["service_name"]: s for s in a._swarm_services(services, tasks)} + assert out["web"] == {"service_name": "web", "mode": "replicated", + "desired": 3, "running": 2, "image": "nginx:1", + "placement": [{"node_id": "n1", "running": 2}]} + # Global mode has no fixed replica count → desired derived from desired tasks. + assert out["agent"] == {"service_name": "agent", "mode": "global", + "desired": 2, "running": 2, "image": "agent:latest", + "placement": [{"node_id": "n1", "running": 1}, + {"node_id": "n2", "running": 1}]} + + +def test_swarm_nodes_normalises_fields(): + nodes = [ + {"ID": "n1", "Description": {"Hostname": "mgr"}, + "Spec": {"Role": "manager", "Availability": "active"}, + "Status": {"State": "ready"}, "ManagerStatus": {"Leader": True}}, + {"ID": "n2", "Description": {"Hostname": "wrk"}, + "Spec": {"Role": "worker", "Availability": "drain"}, + "Status": {"State": "down"}}, + ] + out = {n["node_id"]: n for n in a._swarm_nodes(nodes)} + assert out["n1"] == {"node_id": "n1", "hostname": "mgr", "role": "manager", + "availability": "active", "status": "ready", "leader": True} + assert out["n2"] == {"node_id": "n2", "hostname": "wrk", "role": "worker", + "availability": "drain", "status": "down", "leader": False} + + +def test_collect_swarm_skips_on_worker(monkeypatch): + # A worker reports ControlAvailable=False → None, and never hits /services. + def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): + if path == "/info": + return {"Swarm": {"ControlAvailable": False}} + raise AssertionError("manager-only endpoint queried on a worker") + + monkeypatch.setattr(a, "_docker_request", fake_request) + assert a.collect_swarm("/var/run/docker.sock") is None + + +def test_collect_swarm_assembles_on_manager(monkeypatch): + services = [{"ID": "svc1", "Spec": { + "Name": "web", "Mode": {"Replicated": {"Replicas": 1}}, + "TaskTemplate": {"ContainerSpec": {"Image": "nginx"}}}}] + tasks = [{"ServiceID": "svc1", "DesiredState": "running", + "Status": {"State": "running"}}] + nodes = [{"ID": "n1", "Description": {"Hostname": "mgr"}, + "Spec": {"Role": "manager", "Availability": "active"}, + "Status": {"State": "ready"}, "ManagerStatus": {"Leader": True}}] + + def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT): + return { + "/info": {"Swarm": {"ControlAvailable": True}}, + "/services": services, "/tasks": tasks, "/nodes": nodes, + }[path] + + monkeypatch.setattr(a, "_docker_request", fake_request) + out = a.collect_swarm("/var/run/docker.sock") + assert out["services"][0]["service_name"] == "web" + assert out["services"][0]["running"] == 1 and out["services"][0]["desired"] == 1 + assert out["nodes"][0]["hostname"] == "mgr" and out["nodes"][0]["leader"] is True + + +def test_collect_swarm_silent_skip_when_no_socket(): + assert a.collect_swarm("/nonexistent/does-not-exist.sock") is None + + # ── build_sample wiring ─────────────────────────────────────────────────────── def test_build_sample_includes_docker_when_present(monkeypatch): monkeypatch.setattr(a, "collect_docker", lambda _s: [{"name": "web", "status": "running"}]) + monkeypatch.setattr(a, "collect_swarm", lambda _s: None) sample = a.build_sample(["/"], {}, "/var/run/docker.sock") assert sample["docker"] == [{"name": "web", "status": "running"}] def test_build_sample_omits_docker_when_empty(monkeypatch): monkeypatch.setattr(a, "collect_docker", lambda _s: []) + monkeypatch.setattr(a, "collect_swarm", lambda _s: None) sample = a.build_sample(["/"], {}, "/var/run/docker.sock") assert "docker" not in sample +def test_build_sample_includes_swarm_on_manager(monkeypatch): + monkeypatch.setattr(a, "collect_docker", lambda _s: []) + monkeypatch.setattr(a, "collect_swarm", + lambda _s: {"services": [], "nodes": [{"node_id": "n1"}]}) + sample = a.build_sample(["/"], {}, "/var/run/docker.sock") + assert sample["swarm"]["nodes"] == [{"node_id": "n1"}] + + +def test_build_sample_omits_swarm_off_manager(monkeypatch): + monkeypatch.setattr(a, "collect_docker", lambda _s: []) + monkeypatch.setattr(a, "collect_swarm", lambda _s: None) + sample = a.build_sample(["/"], {}, "/var/run/docker.sock") + assert "swarm" not in sample + + # ── config ──────────────────────────────────────────────────────────────────── def test_read_config_defaults_docker_socket(tmp_path): From 578cc33cc0a3f7cd4213929a519edf80f58daab4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 21:07:40 -0400 Subject: [PATCH 091/126] feat(docker): ingest swarm topology + lifecycle events + health/restart alerts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the agent's enriched + swarm payloads through the docker.persist_host_ samples capability: * Swarm topology — persist sample["swarm"] into docker_swarm_services / docker_swarm_nodes (upsert + prune stale, host-scoped so two managers don't clobber). Migration docker_005 adds services.placement_json for the task→node placement the agent now reports. * Lifecycle events — _derive_events (pure, unit-tested) diffs the newest snapshot against stored per-container state: start / stop / die (non-zero exit) / oom / health_change → docker_events rows. Skipped on a host's first snapshot so the baseline doesn't emit a start per existing container. * Alerts — record restart_count (always) and is_healthy (1.0/0.0, only when a HEALTHCHECK exists) alongside cpu/mem, under host-scoped resource names; METRIC_CATALOG[docker] gains restart_count + is_healthy so they're alertable. host_agent ingest captures the newest sample's swarm object and threads it to the capability (now persist_host_docker(session, host, snapshots, swarm=None)); invoked when containers OR swarm are present, under the same SAVEPOINT. Unit tests cover the event-diff matrix; integration tests cover event derivation across two snapshots and swarm topology round-trip (incl. placement). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- plugins/docker/ingest.py | 192 ++++++++++++++++-- .../versions/docker_005_swarm_placement.py | 29 +++ plugins/docker/models.py | 4 + plugins/host_agent/routes.py | 12 +- steward/alerts/routes.py | 4 +- tests/integration/test_docker.py | 80 ++++++++ tests/plugins/docker/__init__.py | 0 tests/plugins/docker/test_ingest_events.py | 67 ++++++ 8 files changed, 372 insertions(+), 16 deletions(-) create mode 100644 plugins/docker/migrations/versions/docker_005_swarm_placement.py create mode 100644 tests/plugins/docker/__init__.py create mode 100644 tests/plugins/docker/test_ingest_events.py diff --git a/plugins/docker/ingest.py b/plugins/docker/ingest.py index cad4a6b..1e57619 100644 --- a/plugins/docker/ingest.py +++ b/plugins/docker/ingest.py @@ -2,16 +2,23 @@ """Persist host-scoped Docker samples pushed by the host agent. Published as the "docker.persist_host_samples" capability (see __init__.setup), -so the host_agent plugin can hand off the `docker` array from a sample WITHOUT -importing the docker models — the coupling is opportunistic and degrades to a -no-op when the docker plugin is disabled. Runs inside the caller's open -transaction (the ingest handler's session); never opens or commits its own. +so the host_agent plugin can hand off a sample's `docker` array (and, on a swarm +manager, its `swarm` object) WITHOUT importing the docker models — the coupling +is opportunistic and degrades to a no-op when the docker plugin is disabled. +Runs inside the caller's open transaction (the ingest handler's SAVEPOINT); +never opens or commits its own. """ from __future__ import annotations import json from datetime import datetime +from sqlalchemy import delete, select + + +# Stopped/terminal container states (anything not "running"/"paused"/"restarting"). +_STOPPED_STATES = {"exited", "dead", "stopped"} + def _parse_started_at(value) -> datetime | None: """Parse the agent's ISO-8601 started_at string back to a datetime.""" @@ -23,18 +30,142 @@ def _parse_started_at(value) -> datetime | None: return None -async def persist_host_docker(session, host, snapshots) -> None: - """Upsert containers + append time-series for one host's docker snapshots. +def _derive_events(old_state: dict, new_containers: list) -> list: + """Diff a fresh snapshot against stored per-container state → lifecycle events. + + Pure function (no DB) so it's unit-testable. `old_state` maps name → the + previously stored {status, health, oom_killed, exit_code}; `new_containers` + is the newest snapshot's list of container dicts. Returns a list of + (container_name, event, detail) tuples: + + start — a container transitions into running (or a genuinely new + container appears already running) + stop — running → not-running, or a running container is removed + die — same as stop but with a non-zero exit code (abnormal) + oom — OOMKilled flips False→True + health_change — HEALTHCHECK status string changes + + The CALLER skips this entirely on a host's first-ever snapshot (empty + old_state) so the baseline doesn't emit a spurious "start" per existing + container — a later-appearing container still gets one because by then + old_state is populated and that container's prior entry is simply absent. + """ + events: list = [] + new_by_name = {c["name"]: c for c in new_containers if c.get("name")} + + for name, c in new_by_name.items(): + new_status = (c.get("status") or "").lower() + new_running = new_status == "running" + new_oom = bool(c.get("oom_killed")) + new_health = c.get("health") + new_exit = c.get("exit_code") + old = old_state.get(name) + + if old is None: + # Newly observed container — only a start is meaningful (we have no + # prior state to diff a death against). + if new_running: + events.append((name, "start", c.get("image") or None)) + continue + + old_running = (old.get("status") or "").lower() == "running" + if new_running and not old_running: + events.append((name, "start", c.get("image") or None)) + elif old_running and not new_running: + if new_exit not in (None, 0): + events.append((name, "die", f"exit {new_exit}")) + else: + events.append((name, "stop", None)) + + if new_oom and not bool(old.get("oom_killed")): + events.append((name, "oom", None)) + + if new_health != old.get("health") and (new_health or old.get("health")): + events.append((name, "health_change", + f"{old.get('health') or '?'}→{new_health or '?'}")) + + # A running container that vanished from the listing entirely (removed). + for name, old in old_state.items(): + if name not in new_by_name and (old.get("status") or "").lower() == "running": + events.append((name, "stop", "removed")) + + return events + + +async def _persist_swarm(session, host, swarm: dict) -> None: + """Upsert this manager's swarm topology; drop rows no longer reported. + + Current-state tables (not time-series): a manager re-reports its full + services/nodes set every sample, so we upsert what's present and delete what + isn't (scoped to this host so two managers don't clobber each other). + """ + from datetime import timezone + from .models import DockerSwarmNode, DockerSwarmService + + now = datetime.now(timezone.utc) + + services = swarm.get("services") or [] + seen_services: set[str] = set() + for s in services: + name = s.get("service_name") + if not name: + continue + seen_services.add(name) + row = await session.get(DockerSwarmService, (host.id, name)) + if row is None: + row = DockerSwarmService(host_id=host.id, service_name=name) + session.add(row) + row.mode = s.get("mode") or "replicated" + row.desired = int(s.get("desired") or 0) + row.running = int(s.get("running") or 0) + row.image = s.get("image") + row.placement_json = json.dumps(s.get("placement") or []) + row.updated_at = now + stale_services = delete(DockerSwarmService).where( + DockerSwarmService.host_id == host.id) + if seen_services: + stale_services = stale_services.where( + DockerSwarmService.service_name.notin_(seen_services)) + await session.execute(stale_services) + + nodes = swarm.get("nodes") or [] + seen_nodes: set[str] = set() + for n in nodes: + nid = n.get("node_id") + if not nid: + continue + seen_nodes.add(nid) + row = await session.get(DockerSwarmNode, (host.id, nid)) + if row is None: + row = DockerSwarmNode(host_id=host.id, node_id=nid) + session.add(row) + row.hostname = n.get("hostname") or "" + row.role = n.get("role") or "worker" + row.availability = n.get("availability") or "active" + row.status = n.get("status") or "unknown" + row.leader = bool(n.get("leader", False)) + row.updated_at = now + stale_nodes = delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id) + if seen_nodes: + stale_nodes = stale_nodes.where(DockerSwarmNode.node_id.notin_(seen_nodes)) + await session.execute(stale_nodes) + + +async def persist_host_docker(session, host, snapshots, swarm=None) -> None: + """Upsert containers + time-series + lifecycle events + swarm for one host. `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) — - one entry per ingested sample carrying docker data (usually just one; more - when the agent flushes a backlog). Every snapshot contributes time-series - DockerMetric rows; the newest snapshot drives current container state and - the alert pipeline (record_metric writes "now", so feeding it stale buffered - snapshots would be misleading). + one entry per ingested sample carrying docker data (usually one; more when + the agent flushes a backlog). Every snapshot contributes time-series + DockerMetric rows; the newest snapshot drives current container state, the + alert pipeline, and lifecycle-event derivation. `swarm` is the newest + sample's swarm object (or None off managers) — persisted when present. """ from steward.core.alerts import record_metric - from .models import DockerContainer, DockerMetric + from .models import DockerContainer, DockerEvent, DockerMetric + + if swarm is not None: + await _persist_swarm(session, host, swarm) if not snapshots: return @@ -54,6 +185,24 @@ async def persist_host_docker(session, host, snapshots) -> None: mem_usage_bytes=c.get("mem_usage_bytes") or 0, )) + # Snapshot of stored per-container state BEFORE the upsert overwrites it — + # used to diff lifecycle events. Skip event derivation on the host's first + # snapshot (empty state) so we don't emit a start per pre-existing container. + old_rows = (await session.execute( + select(DockerContainer).where(DockerContainer.host_id == host.id) + )).scalars().all() + old_state = { + r.name: {"status": r.status, "health": r.health, + "oom_killed": r.oom_killed, "exit_code": r.exit_code} + for r in old_rows + } + if old_state: + for name, event, detail in _derive_events(old_state, latest_containers): + session.add(DockerEvent( + host_id=host.id, container_name=name, + event=event, detail=detail, at=latest_at, + )) + # Current state + alerts from the newest snapshot only. for c in latest_containers: name = c.get("name") @@ -90,8 +239,8 @@ async def persist_host_docker(session, host, snapshots) -> None: # Alert pipeline — resource is host-scoped so containers of the same name # on different hosts don't collide in the metric/alert namespace. + resource = f"{host.name}/{name}" if c.get("status") == "running" and c.get("cpu_pct") is not None: - resource = f"{host.name}/{name}" await record_metric( session=session, source_module="docker", resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"], @@ -101,3 +250,20 @@ async def persist_host_docker(session, host, snapshots) -> None: session=session, source_module="docker", resource_name=resource, metric_name="mem_pct", value=c["mem_pct"], ) + # Restart count is alertable regardless of state (crash-looping matters + # most while the container is down/restarting). + await record_metric( + session=session, source_module="docker", + resource_name=resource, metric_name="restart_count", + value=float(c.get("restart_count", 0) or 0), + ) + # Health → 1.0/0.0 only for containers that actually define a HEALTHCHECK + # (health is None otherwise — recording 0 would false-alarm every plain + # container). + health = c.get("health") + if health in ("healthy", "unhealthy", "starting"): + await record_metric( + session=session, source_module="docker", + resource_name=resource, metric_name="is_healthy", + value=1.0 if health == "healthy" else 0.0, + ) diff --git a/plugins/docker/migrations/versions/docker_005_swarm_placement.py b/plugins/docker/migrations/versions/docker_005_swarm_placement.py new file mode 100644 index 0000000..0e90bd5 --- /dev/null +++ b/plugins/docker/migrations/versions/docker_005_swarm_placement.py @@ -0,0 +1,29 @@ +"""Docker swarm service placement column + +Adds docker_swarm_services.placement_json — the task→node placement of a +service's running replicas, captured from the agent's swarm payload (a manager +sees every task, so this records cross-node placement the local container rows +can't). Additive column; no DROP+recreate. + +Revision ID: docker_005_swarm_placement +Revises: docker_004_events_swarm +Create Date: 2026-06-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_005_swarm_placement" +down_revision: Union[str, None] = "docker_004_events_swarm" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("docker_swarm_services", + sa.Column("placement_json", sa.Text, nullable=False, + server_default="[]")) + + +def downgrade() -> None: + op.drop_column("docker_swarm_services", "placement_json") diff --git a/plugins/docker/models.py b/plugins/docker/models.py index cf83bed..fb3dd63 100644 --- a/plugins/docker/models.py +++ b/plugins/docker/models.py @@ -145,6 +145,10 @@ class DockerSwarmService(Base): desired: Mapped[int] = mapped_column(Integer, nullable=False, default=0) running: Mapped[int] = mapped_column(Integer, nullable=False, default=0) image: Mapped[str | None] = mapped_column(String(512), nullable=True) + # task→node placement of running replicas: [{"node_id", "running"}], JSON. + # A manager sees every task, so this captures cross-node placement that the + # local docker_containers rows (this host only) can't. + placement_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc), diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 9670569..3e6c486 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -197,6 +197,8 @@ async def ingest(): accepted = 0 latest_ts: datetime | None = None docker_snapshots: list[tuple[datetime, list]] = [] + latest_swarm: dict | None = None + latest_swarm_ts: datetime | None = None for sample in samples: try: recorded_at = _parse_ts(sample["ts"]) @@ -208,6 +210,12 @@ async def ingest(): docker = sample.get("docker") if isinstance(docker, list) and docker: docker_snapshots.append((recorded_at, docker)) + # Swarm is current-state, not time-series — keep only the newest + # sample's topology (a manager re-reports it every interval). + swarm = sample.get("swarm") + if isinstance(swarm, dict) and ( + latest_swarm_ts is None or recorded_at > latest_swarm_ts): + latest_swarm, latest_swarm_ts = swarm, recorded_at accepted += 1 if latest_ts is None or recorded_at > latest_ts: latest_ts = recorded_at @@ -220,7 +228,7 @@ async def ingest(): # (opportunistic synergy via the capability registry — no hard import, # no-op when docker is disabled). A failure here must never sink the # whole ingest, so the metrics above still land. - if docker_snapshots: + if docker_snapshots or latest_swarm is not None: from steward.core.capabilities import has_capability, invoke_capability if has_capability("docker.persist_host_samples"): try: @@ -229,7 +237,7 @@ async def ingest(): async with session.begin_nested(): await invoke_capability( "docker.persist_host_samples", UserRole.admin, - session, host, docker_snapshots, + session, host, docker_snapshots, latest_swarm, ) except Exception: current_app.logger.exception( diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py index f1396d4..621c58a 100644 --- a/steward/alerts/routes.py +++ b/steward/alerts/routes.py @@ -24,7 +24,9 @@ METRIC_CATALOG: dict[str, list[str]] = { "traefik": ["request_rate", "error_rate", "latency_p50_ms", "latency_p95_ms", "latency_p99_ms", "response_bytes_rate", "cert_expiry_days"], "unifi": ["is_up", "latency_ms", "total_clients"], - "docker": ["cpu_pct", "mem_pct"], + # restart_count = cumulative restarts (alert on crash-looping); is_healthy = + # 1.0 healthy / 0.0 unhealthy from the container HEALTHCHECK (alert on <1). + "docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy"], "host_agent": [ "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes", "disk_used_pct_worst", "load_1m", "load_5m", "load_15m", "uptime_secs", diff --git a/tests/integration/test_docker.py b/tests/integration/test_docker.py index 2865d65..21accd5 100644 --- a/tests/integration/test_docker.py +++ b/tests/integration/test_docker.py @@ -139,3 +139,83 @@ def test_persist_scopes_containers_by_host(app): assert metric_hosts == 2 # time-series rows scoped per host assert resources == {"alpha/web", "beta/web"} assert enrich == ("healthy", 2, "web", 1000) # enrichment round-trips + + +@_NEEDS_DB +def test_lifecycle_events_derived_across_snapshots(app): + from sqlalchemy import text + from steward.models.hosts import Host + + persist = _persist_fn(app) + t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc) + t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc) + running = {"name": "web", "container_id": "abc", "image": "nginx", + "status": "running", "cpu_pct": 5.0, "mem_pct": 10.0, + "restart_count": 0, "exit_code": None, "oom_killed": False, + "health": "healthy"} + died = {**running, "status": "exited", "cpu_pct": None, + "exit_code": 137, "oom_killed": True, "health": None} + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + await s.execute(text("DELETE FROM docker_events")) + await s.execute(text("DELETE FROM docker_metrics")) + await s.execute(text("DELETE FROM docker_containers")) + h = Host(id=str(uuid.uuid4()), name="evt", address="10.9.9.9") + s.add(h) + await s.flush() + # Baseline snapshot establishes state — must NOT emit a start. + await persist(s, h, [(t1, [running])]) + # Second snapshot: the container OOM-died. + await persist(s, h, [(t2, [died])]) + hid = h.id + rows = (await s.execute(text( + "SELECT event, detail FROM docker_events WHERE host_id = :h " + "ORDER BY event"), {"h": hid})).all() + return [(r[0], r[1]) for r in rows] + + events = asyncio.run(_go()) + # baseline emitted nothing; the death emitted both die (exit 137) and oom. + assert ("die", "exit 137") in events + assert ("oom", None) in events + assert not any(e[0] == "start" for e in events) + + +@_NEEDS_DB +def test_swarm_topology_persisted(app): + from sqlalchemy import text + from steward.models.hosts import Host + + persist = _persist_fn(app) + swarm = { + "services": [{"service_name": "web", "mode": "replicated", + "desired": 3, "running": 2, "image": "nginx", + "placement": [{"node_id": "n1", "running": 2}]}], + "nodes": [{"node_id": "n1", "hostname": "mgr", "role": "manager", + "availability": "active", "status": "ready", "leader": True}], + } + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + await s.execute(text("DELETE FROM docker_swarm_services")) + await s.execute(text("DELETE FROM docker_swarm_nodes")) + h = Host(id=str(uuid.uuid4()), name="mgr", address="10.8.8.8") + s.add(h) + await s.flush() + # No containers in this sample — swarm must still persist. + await persist(s, h, [], swarm) + hid = h.id + svc = (await s.execute(text( + "SELECT mode, desired, running, image, placement_json " + "FROM docker_swarm_services WHERE host_id = :h"), {"h": hid})).first() + node = (await s.execute(text( + "SELECT role, availability, status, leader " + "FROM docker_swarm_nodes WHERE host_id = :h"), {"h": hid})).first() + return svc, node + + svc, node = asyncio.run(_go()) + assert svc[0] == "replicated" and svc[1] == 3 and svc[2] == 2 and svc[3] == "nginx" + assert "n1" in svc[4] # placement JSON carries the node id + assert node[0] == "manager" and node[2] == "ready" and node[3] is True diff --git a/tests/plugins/docker/__init__.py b/tests/plugins/docker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/plugins/docker/test_ingest_events.py b/tests/plugins/docker/test_ingest_events.py new file mode 100644 index 0000000..b5b50e8 --- /dev/null +++ b/tests/plugins/docker/test_ingest_events.py @@ -0,0 +1,67 @@ +"""Unit tests for the docker plugin's lifecycle-event derivation. + +_derive_events is a pure function (no DB), and ingest.py imports its models +lazily inside functions, so importing it here doesn't register ORM tables — +safe for the no-DB unit lane. +""" +from plugins.docker.ingest import _derive_events + + +def _c(name, status, **kw): + return {"name": name, "status": status, **kw} + + +def test_start_on_running_after_stopped(): + old = {"web": {"status": "exited", "health": None, "oom_killed": False, "exit_code": 0}} + events = _derive_events(old, [_c("web", "running", image="nginx")]) + assert events == [("web", "start", "nginx")] + + +def test_start_for_new_container_when_state_exists(): + # old_state is non-empty (host already seen), a fresh container appears running. + old = {"other": {"status": "running", "health": None, "oom_killed": False, "exit_code": None}} + events = _derive_events(old, [ + _c("other", "running"), + _c("new", "running", image="redis"), + ]) + assert ("new", "start", "redis") in events + # the unchanged 'other' produces nothing + assert all(e[0] != "other" for e in events) + + +def test_stop_on_clean_exit(): + old = {"job": {"status": "running", "health": None, "oom_killed": False, "exit_code": None}} + events = _derive_events(old, [_c("job", "exited", exit_code=0)]) + assert events == [("job", "stop", None)] + + +def test_die_on_nonzero_exit(): + old = {"job": {"status": "running", "health": None, "oom_killed": False, "exit_code": None}} + events = _derive_events(old, [_c("job", "exited", exit_code=137)]) + assert events == [("job", "die", "exit 137")] + + +def test_oom_on_flip_to_true(): + old = {"hog": {"status": "running", "health": None, "oom_killed": False, "exit_code": None}} + # OOM-killed → also no longer running with a non-zero exit, so die + oom both fire. + events = _derive_events(old, [_c("hog", "exited", exit_code=137, oom_killed=True)]) + assert ("hog", "oom", None) in events + assert ("hog", "die", "exit 137") in events + + +def test_health_change_emitted(): + old = {"svc": {"status": "running", "health": "healthy", "oom_killed": False, "exit_code": None}} + events = _derive_events(old, [_c("svc", "running", health="unhealthy")]) + assert events == [("svc", "health_change", "healthy→unhealthy")] + + +def test_removed_running_container_stops(): + old = {"gone": {"status": "running", "health": None, "oom_killed": False, "exit_code": None}} + events = _derive_events(old, []) # vanished from the listing entirely + assert events == [("gone", "stop", "removed")] + + +def test_no_event_when_unchanged(): + old = {"web": {"status": "running", "health": "healthy", "oom_killed": False, "exit_code": None}} + events = _derive_events(old, [_c("web", "running", health="healthy")]) + assert events == [] From faecac3ec6eb675e851e1b98d3a8f0b4eca0686d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 21:40:57 -0400 Subject: [PATCH 092/126] feat(docker): retention + hourly rollup for metrics/events with Settings windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bounds Docker time-series growth (the main scaling concern). New docker_metrics_hourly table + docker_006 migration; a plugin retention module (docker.run_retention capability) rolls raw docker_metrics older than the raw window into hourly averages (idempotent upsert), deletes the rolled raw rows, then prunes stale rollups + lifecycle events. Core cleanup.py drives it each hourly run via the capability (no plugin-model import), reading the three retention windows fresh from settings so changes apply without restart (rule 25). Settings → "Thresholds & Retention" gains a Docker retention card (raw / rolled-up / events windows, working defaults 7/90/30 days). Unit tests cover the hour-aligned cutoff/bucketing helpers; integration test exercises the real rollup-average + prune across both windows. Milestone 77 task #941. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- plugins/docker/__init__.py | 9 ++ .../versions/docker_006_metric_rollup.py | 45 ++++++ plugins/docker/models.py | 34 +++++ plugins/docker/retention.py | 128 ++++++++++++++++++ steward/core/cleanup.py | 37 ++++- steward/core/settings.py | 6 + steward/settings/routes.py | 17 +++ steward/templates/settings/_tabs.html | 2 +- steward/templates/settings/thresholds.html | 30 +++- tests/integration/test_docker.py | 96 +++++++++++++ tests/plugins/docker/test_retention.py | 45 ++++++ 11 files changed, 445 insertions(+), 4 deletions(-) create mode 100644 plugins/docker/migrations/versions/docker_006_metric_rollup.py create mode 100644 plugins/docker/retention.py create mode 100644 tests/plugins/docker/test_retention.py diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py index 4272f51..7183961 100644 --- a/plugins/docker/__init__.py +++ b/plugins/docker/__init__.py @@ -21,12 +21,21 @@ def setup(app: "Quart") -> None: from steward.core.capabilities import register_capability from steward.models.users import UserRole from .ingest import persist_host_docker + from .retention import run_docker_retention register_capability( "docker.persist_host_samples", persist_host_docker, label="Persist host Docker samples", description="Store per-host container state + metrics pushed by the host agent.", required_role=UserRole.viewer, ) + # Roll up + prune Docker time-series, driven by the core cleanup task without + # it importing our models. Same trusted server-side data-plane role as above. + register_capability( + "docker.run_retention", run_docker_retention, + label="Run Docker retention", + description="Roll up old docker_metrics to hourly + prune stale metrics/events.", + required_role=UserRole.viewer, + ) def get_scheduled_tasks() -> list: diff --git a/plugins/docker/migrations/versions/docker_006_metric_rollup.py b/plugins/docker/migrations/versions/docker_006_metric_rollup.py new file mode 100644 index 0000000..782abfd --- /dev/null +++ b/plugins/docker/migrations/versions/docker_006_metric_rollup.py @@ -0,0 +1,45 @@ +"""Docker hourly metric rollup table + +Adds docker_metrics_hourly — the coarse series that retention rolls raw +docker_metrics into before pruning them, so multi-day history stays cheap. +One row per (host, container, hour bucket); the unique constraint is the +conflict target for the idempotent rollup upsert. Additive create_table. + +Revision ID: docker_006_metric_rollup +Revises: docker_005_swarm_placement +Create Date: 2026-06-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_006_metric_rollup" +down_revision: Union[str, None] = "docker_005_swarm_placement" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "docker_metrics_hourly", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("host_id", sa.String(length=36), nullable=False), + sa.Column("container_name", sa.String(length=255), nullable=False), + sa.Column("bucket", sa.DateTime(timezone=True), nullable=False), + sa.Column("cpu_pct", sa.Float(), nullable=False, server_default="0"), + sa.Column("mem_pct", sa.Float(), nullable=False, server_default="0"), + sa.Column("mem_usage_bytes", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("sample_count", sa.Integer(), nullable=False, server_default="0"), + sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("host_id", "container_name", "bucket", + name="uq_docker_metrics_hourly_bucket"), + ) + op.create_index("ix_docker_metrics_hourly_bucket", + "docker_metrics_hourly", ["bucket"]) + + +def downgrade() -> None: + op.drop_index("ix_docker_metrics_hourly_bucket", + table_name="docker_metrics_hourly") + op.drop_table("docker_metrics_hourly") diff --git a/plugins/docker/models.py b/plugins/docker/models.py index fb3dd63..cfbfee8 100644 --- a/plugins/docker/models.py +++ b/plugins/docker/models.py @@ -4,6 +4,7 @@ import uuid from datetime import datetime, timezone from sqlalchemy import ( BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text, + UniqueConstraint, ) from sqlalchemy.orm import Mapped, mapped_column from steward.models.base import Base @@ -87,6 +88,39 @@ class DockerMetric(Base): ) +class DockerMetricHourly(Base): + """Hourly rollup of docker_metrics — avg cpu/mem per container per hour. + + Raw per-sample rows (~2880/container/day at 30s) are pruned beyond a short + window; before deletion they're aggregated here so multi-day history stays + cheap to store and query. One row per (host, container, hour bucket); the + unique constraint lets retention upsert idempotently if it re-runs before the + raw rows are deleted. `bucket` is the hour-truncated sample time. + """ + __tablename__ = "docker_metrics_hourly" + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + ) + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False + ) + container_name: Mapped[str] = mapped_column(String(255), nullable=False) + bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + __table_args__ = ( + # One bucket per container per host — the conflict target for the + # idempotent rollup upsert; doubles as the history-query index. + UniqueConstraint("host_id", "container_name", "bucket", + name="uq_docker_metrics_hourly_bucket"), + Index("ix_docker_metrics_hourly_bucket", "bucket"), + ) + + class DockerEvent(Base): """Lifecycle events derived by diffing consecutive host snapshots. diff --git a/plugins/docker/retention.py b/plugins/docker/retention.py new file mode 100644 index 0000000..9919f92 --- /dev/null +++ b/plugins/docker/retention.py @@ -0,0 +1,128 @@ +# plugins/docker/retention.py +"""Bound Docker time-series growth: roll up old metrics, prune old rows. + +Published as the "docker.run_retention" capability (see __init__.setup) so the +core cleanup task can drive it WITHOUT importing the docker models (same +opportunistic-coupling pattern as docker.persist_host_samples). Runs inside the +caller's open transaction; never opens or commits its own. + +The scaling concern is docker_metrics: ~2880 rows/container/day at a 30s sample. +We keep raw samples for a short window, then aggregate everything older into +hourly averages (docker_metrics_hourly) and delete the raw rows — so multi-day +history stays cheap to store and query. docker_events is light but unbounded +without a cutoff, so it gets a (longer) window too. +""" +from __future__ import annotations + +from datetime import datetime, timedelta + + +def _hour_floor(dt: datetime) -> datetime: + """Truncate a datetime down to the start of its hour (drops min/sec/µs).""" + return dt.replace(minute=0, second=0, microsecond=0) + + +def _rollup_cutoff(now: datetime, raw_days: int) -> datetime: + """Hour-aligned boundary below which raw metrics get rolled up + deleted. + + Aligning to the hour means we only ever roll up *whole* elapsed hours — a + bucket is never split across the keep/roll boundary, so re-running can't + produce a partial-then-complete duplicate for the same hour. + """ + return _hour_floor(now - timedelta(days=raw_days)) + + +async def run_docker_retention( + session, + *, + events_days: int, + metrics_raw_days: int, + metrics_rollup_days: int, + now: datetime | None = None, +) -> dict: + """Roll up + prune Docker time-series. Returns a counts dict for logging. + + 1. Aggregate docker_metrics older than the (hour-aligned) raw window into + docker_metrics_hourly (avg cpu/mem per container per hour), upserting so a + re-run is idempotent, then delete those raw rows. + 2. Prune rolled-up rows older than the rollup window. + 3. Prune docker_events older than the events window. + """ + from datetime import timezone + + from sqlalchemy import delete, func, select + from sqlalchemy.dialects.postgresql import insert as pg_insert + + from .models import DockerEvent, DockerMetric, DockerMetricHourly + + if now is None: + now = datetime.now(timezone.utc) + + rolled = rolled_rows = events_pruned = rollup_pruned = 0 + + # ── 1. Roll up raw metrics older than the raw window into hourly buckets ── + raw_cutoff = _rollup_cutoff(now, metrics_raw_days) + hour = func.date_trunc("hour", DockerMetric.scraped_at) + agg = ( + select( + DockerMetric.host_id, + DockerMetric.container_name, + hour.label("bucket"), + func.avg(DockerMetric.cpu_pct).label("cpu_pct"), + func.avg(DockerMetric.mem_pct).label("mem_pct"), + func.avg(DockerMetric.mem_usage_bytes).label("mem_usage_bytes"), + func.count().label("sample_count"), + ) + .where(DockerMetric.scraped_at < raw_cutoff) + .group_by(DockerMetric.host_id, DockerMetric.container_name, hour) + ) + for r in (await session.execute(agg)).all(): + stmt = ( + pg_insert(DockerMetricHourly) + .values( + host_id=r.host_id, + container_name=r.container_name, + bucket=r.bucket, + cpu_pct=float(r.cpu_pct or 0.0), + mem_pct=float(r.mem_pct or 0.0), + mem_usage_bytes=int(r.mem_usage_bytes or 0), + sample_count=int(r.sample_count or 0), + ) + .on_conflict_do_update( + constraint="uq_docker_metrics_hourly_bucket", + set_={ + "cpu_pct": float(r.cpu_pct or 0.0), + "mem_pct": float(r.mem_pct or 0.0), + "mem_usage_bytes": int(r.mem_usage_bytes or 0), + "sample_count": int(r.sample_count or 0), + }, + ) + ) + await session.execute(stmt) + rolled += 1 + rolled_rows += int(r.sample_count or 0) + if rolled: + await session.execute( + delete(DockerMetric).where(DockerMetric.scraped_at < raw_cutoff) + ) + + # ── 2. Prune rolled-up rows beyond the rollup window ── + rollup_cutoff = now - timedelta(days=metrics_rollup_days) + res = await session.execute( + delete(DockerMetricHourly).where(DockerMetricHourly.bucket < rollup_cutoff) + ) + rollup_pruned = res.rowcount or 0 + + # ── 3. Prune lifecycle events beyond the events window ── + events_cutoff = now - timedelta(days=events_days) + res = await session.execute( + delete(DockerEvent).where(DockerEvent.at < events_cutoff) + ) + events_pruned = res.rowcount or 0 + + return { + "buckets_rolled": rolled, + "raw_rows_rolled": rolled_rows, + "rollup_pruned": rollup_pruned, + "events_pruned": events_pruned, + } diff --git a/steward/core/cleanup.py b/steward/core/cleanup.py index 0a9e556..563a918 100644 --- a/steward/core/cleanup.py +++ b/steward/core/cleanup.py @@ -16,9 +16,11 @@ logger = logging.getLogger(__name__) async def run_cleanup(app: "Quart") -> None: - """Delete rows older than DATA_RETENTION_DAYS from time-series tables.""" + """Delete rows older than DATA_RETENTION_DAYS from time-series tables, then + run Docker-specific rollup + retention (delegated to the docker plugin).""" retention_days: int = app.config.get("DATA_RETENTION_DAYS", 90) - cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days) + now = datetime.now(timezone.utc) + cutoff = now - timedelta(days=retention_days) async with app.db_sessionmaker() as session: async with session.begin(): @@ -32,3 +34,34 @@ async def run_cleanup(app: "Quart") -> None: ) if result.rowcount: logger.info(f"Pruned {result.rowcount} rows from {model.__tablename__}") + + await _run_docker_retention(session, now) + + +async def _run_docker_retention(session, now: datetime) -> None: + """Drive the docker plugin's rollup + prune via its capability, if loaded. + + Windows are read fresh from settings each run (rule 25 — a change in the + Settings UI takes effect on the next hourly cleanup, no restart). Kept in its + own transaction so a docker-side failure can't roll back the generic prune + above. No-op when the docker plugin is disabled (capability absent). + """ + from steward.core.capabilities import has_capability, invoke_capability + if not has_capability("docker.run_retention"): + return + from steward.core.settings import get_setting + from steward.models.users import UserRole + + # Reads + rollup/prune share one transaction — get_setting's SELECT would + # otherwise autobegin one, making a later session.begin() raise. + async with session.begin(): + raw_days = int(await get_setting(session, "docker.retention.metrics_raw_days") or 7) + rollup_days = int(await get_setting(session, "docker.retention.metrics_rollup_days") or 90) + events_days = int(await get_setting(session, "docker.retention.events_days") or 30) + counts = await invoke_capability( + "docker.run_retention", UserRole.viewer, session, + events_days=events_days, metrics_raw_days=raw_days, + metrics_rollup_days=rollup_days, now=now, + ) + if counts and any(counts.values()): + logger.info("Docker retention: %s", counts) diff --git a/steward/core/settings.py b/steward/core/settings.py index ef7f0bc..23bece8 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -78,6 +78,12 @@ DEFAULTS: dict[str, Any] = { "thresholds.load_warn": 80, "thresholds.load_crit": 100, "thresholds.temp_warn": 70, "thresholds.temp_crit": 85, "thresholds.uptime_warn": 99.0, "thresholds.uptime_crit": 95.0, + # Docker time-series retention (rule 25 — tunable, no restart). Raw 30s + # samples are heavy, so keep a short raw window then roll up to hourly + # averages kept much longer; lifecycle events are light, keep a month. + "docker.retention.metrics_raw_days": 7, + "docker.retention.metrics_rollup_days": 90, + "docker.retention.events_days": 30, "plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml", # Default-enabled plugins. These are the generic, non-vendor-specific # bundled plugins (protocols/standards, not a single product) — useful on diff --git a/steward/settings/routes.py b/steward/settings/routes.py index 8b1f0f6..517d2e1 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -127,6 +127,14 @@ _THRESHOLD_FIELDS = [ ("latency_warn", "ping.threshold.good_ms", False), ("latency_crit", "ping.threshold.warn_ms", False), ] +# Docker time-series retention windows (days). Read fresh by the cleanup task, +# so a save takes effect on the next hourly run — no app.config wiring needed. +_RETENTION_FIELDS = [ + ("docker_metrics_raw_days", "docker.retention.metrics_raw_days"), + ("docker_metrics_rollup_days", "docker.retention.metrics_rollup_days"), + ("docker_events_days", "docker.retention.events_days"), +] + @settings_bp.get("/thresholds/") @require_role(UserRole.admin) @@ -151,6 +159,15 @@ async def save_thresholds(): except (TypeError, ValueError): continue await set_setting(db, key, val) + for field, key in _RETENTION_FIELDS: + raw = form.get(field, "") + if raw == "": + continue + try: + val = max(1, int(raw)) # at least a day — 0 would prune everything + except (TypeError, ValueError): + continue + await set_setting(db, key, val) await _reload_app_config() await log_audit(current_app, session.get("user_id"), session.get("username", ""), "settings.saved", detail={"section": "thresholds"}) diff --git a/steward/templates/settings/_tabs.html b/steward/templates/settings/_tabs.html index 65bb7fd..97e1201 100644 --- a/steward/templates/settings/_tabs.html +++ b/steward/templates/settings/_tabs.html @@ -3,7 +3,7 @@
{% set tabs = [ ("general", "General", "/settings/general/"), - ("thresholds", "Thresholds", "/settings/thresholds/"), + ("thresholds", "Thresholds & Retention", "/settings/thresholds/"), ("notifications", "Notifications", "/settings/notifications/"), ("reports", "Reports", "/settings/reports/"), ("auth", "Auth", "/settings/auth/"), diff --git a/steward/templates/settings/thresholds.html b/steward/templates/settings/thresholds.html index 1a7509e..204b049 100644 --- a/steward/templates/settings/thresholds.html +++ b/steward/templates/settings/thresholds.html @@ -1,6 +1,6 @@ {# steward/templates/settings/thresholds.html — tunable degraded/critical cutoffs #} {% extends "base.html" %} -{% block title %}Settings — Thresholds — Steward{% endblock %} +{% block title %}Settings — Thresholds & Retention — Steward{% endblock %} {% block content %} {% set active_tab = "thresholds" %} {% include "settings/_tabs.html" %} @@ -42,6 +42,34 @@ {{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }}
+{% macro days(label, field, key, hint) %} +
+ +
+ +
+ {% if hint %}
{{ hint }}
{% endif %} +
+{% endmacro %} + +
+

Docker data retention

+

+ Bounds how much Docker history is stored. Raw per-sample container metrics are + kept for the raw window, then rolled up into hourly averages kept for the + rollup window; lifecycle events are kept for the events window. Applied by the + hourly cleanup task — a change takes effect on its next run. +

+ + {{ days("Raw metrics", "docker_metrics_raw_days", "docker.retention.metrics_raw_days", + "Keep per-sample container CPU/memory points this long, then roll up to hourly.") }} + {{ days("Rolled-up metrics", "docker_metrics_rollup_days", "docker.retention.metrics_rollup_days", + "Keep the hourly-averaged series this long for multi-day history.") }} + {{ days("Lifecycle events", "docker_events_days", "docker.retention.events_days", + "Keep container start/stop/die/health history this long.") }} +
+
Takes effect immediately — no restart. diff --git a/tests/integration/test_docker.py b/tests/integration/test_docker.py index 21accd5..8e98265 100644 --- a/tests/integration/test_docker.py +++ b/tests/integration/test_docker.py @@ -86,6 +86,15 @@ def _persist_fn(app): return persist_host_docker +def _retention_fn(app): + """Resolve run_docker_retention via capability (or direct import if unloaded).""" + from steward.core.capabilities import has_capability, get_capability + if has_capability("docker.run_retention"): + return get_capability("docker.run_retention").fn + from plugins.docker.retention import run_docker_retention + return run_docker_retention + + @_NEEDS_DB def test_persist_scopes_containers_by_host(app): from sqlalchemy import text @@ -219,3 +228,90 @@ def test_swarm_topology_persisted(app): assert svc[0] == "replicated" and svc[1] == 3 and svc[2] == 2 and svc[3] == "nginx" assert "n1" in svc[4] # placement JSON carries the node id assert node[0] == "manager" and node[2] == "ready" and node[3] is True + + +@_NEEDS_DB +def test_retention_rollup_and_prune(app): + """Old raw metrics roll up to hourly averages then delete; stale rollup + + events prune; recent rows survive.""" + from sqlalchemy import text + from steward.models.hosts import Host + + run_retention = _retention_fn(app) + now = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc) + # One old hour (10 days back) with three samples → one rolled-up bucket. + old_hour = datetime(2026, 6, 9, 9, 0, 0, tzinfo=timezone.utc) + recent = datetime(2026, 6, 19, 11, 0, 0, tzinfo=timezone.utc) # inside raw window + ancient_bucket = datetime(2026, 3, 1, 0, 0, 0, tzinfo=timezone.utc) # > rollup window + old_event = datetime(2026, 5, 1, 0, 0, 0, tzinfo=timezone.utc) # > events window + new_event = datetime(2026, 6, 18, 0, 0, 0, tzinfo=timezone.utc) # inside events window + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + await s.execute(text("DELETE FROM docker_metrics_hourly")) + await s.execute(text("DELETE FROM docker_metrics")) + await s.execute(text("DELETE FROM docker_events")) + h = Host(id=str(uuid.uuid4()), name="ret", address="10.7.7.7") + s.add(h) + await s.flush() + hid = h.id + # Three raw samples in the old hour: cpu 10/20/30, mem 40/50/60. + for i, (ts, cpu, mem, usage) in enumerate([ + (old_hour, 10.0, 40.0, 100), + (old_hour.replace(second=30), 20.0, 50.0, 200), + (old_hour.replace(minute=1), 30.0, 60.0, 300), + ]): + await s.execute(text( + "INSERT INTO docker_metrics " + "(id, host_id, container_name, scraped_at, cpu_pct, mem_pct, mem_usage_bytes) " + "VALUES (:id,:h,'web',:ts,:cpu,:mem,:usage)"), + {"id": str(uuid.uuid4()), "h": hid, "ts": ts, + "cpu": cpu, "mem": mem, "usage": usage}) + # A recent sample (within the 7-day raw window) — must survive raw. + await s.execute(text( + "INSERT INTO docker_metrics " + "(id, host_id, container_name, scraped_at, cpu_pct, mem_pct, mem_usage_bytes) " + "VALUES (:id,:h,'web',:ts,5.0,5.0,50)"), + {"id": str(uuid.uuid4()), "h": hid, "ts": recent}) + # A pre-existing rollup row older than the 90-day rollup window. + await s.execute(text( + "INSERT INTO docker_metrics_hourly " + "(id, host_id, container_name, bucket, cpu_pct, mem_pct, mem_usage_bytes, sample_count) " + "VALUES (:id,:h,'ancient',:b,1,1,1,1)"), + {"id": str(uuid.uuid4()), "h": hid, "b": ancient_bucket}) + # Events either side of the 30-day events window. + for ev_at, ev in [(old_event, "stop"), (new_event, "start")]: + await s.execute(text( + "INSERT INTO docker_events (id, host_id, container_name, event, at) " + "VALUES (:id,:h,'web',:ev,:at)"), + {"id": str(uuid.uuid4()), "h": hid, "ev": ev, "at": ev_at}) + + async with s.begin(): + counts = await run_retention( + s, events_days=30, metrics_raw_days=7, + metrics_rollup_days=90, now=now, + ) + + raw_left = (await s.execute(text( + "SELECT COUNT(*) FROM docker_metrics WHERE host_id=:h"), {"h": hid})).scalar() + bucket = (await s.execute(text( + "SELECT cpu_pct, mem_pct, mem_usage_bytes, sample_count " + "FROM docker_metrics_hourly WHERE host_id=:h AND container_name='web'"), + {"h": hid})).first() + ancient_left = (await s.execute(text( + "SELECT COUNT(*) FROM docker_metrics_hourly " + "WHERE host_id=:h AND container_name='ancient'"), {"h": hid})).scalar() + events_left = (await s.execute(text( + "SELECT COUNT(*) FROM docker_events WHERE host_id=:h"), {"h": hid})).scalar() + return counts, raw_left, bucket, ancient_left, events_left + + counts, raw_left, bucket, ancient_left, events_left = asyncio.run(_go()) + assert raw_left == 1 # only the recent sample survives raw + assert bucket is not None + assert bucket[0] == 20.0 and bucket[1] == 50.0 # avg cpu / mem over the 3 samples + assert bucket[2] == 200 and bucket[3] == 3 # avg usage + sample_count + assert ancient_left == 0 # stale rollup pruned + assert events_left == 1 # only the in-window event survives + assert counts["buckets_rolled"] == 1 and counts["raw_rows_rolled"] == 3 + assert counts["events_pruned"] == 1 and counts["rollup_pruned"] == 1 diff --git a/tests/plugins/docker/test_retention.py b/tests/plugins/docker/test_retention.py new file mode 100644 index 0000000..262d9c3 --- /dev/null +++ b/tests/plugins/docker/test_retention.py @@ -0,0 +1,45 @@ +"""Unit tests for the docker plugin's retention cutoff/bucketing helpers. + +_hour_floor and _rollup_cutoff are pure (no DB); retention.py imports its models +lazily inside run_docker_retention, so importing them here doesn't register ORM +tables — safe for the no-DB unit lane. The DB-backed rollup/prune itself is +covered in tests/integration/test_docker.py. +""" +from datetime import datetime, timedelta, timezone + +from plugins.docker.retention import _hour_floor, _rollup_cutoff + + +def test_hour_floor_drops_sub_hour_components(): + dt = datetime(2026, 6, 19, 14, 37, 52, 123456, tzinfo=timezone.utc) + assert _hour_floor(dt) == datetime(2026, 6, 19, 14, 0, 0, 0, tzinfo=timezone.utc) + + +def test_hour_floor_on_exact_hour_is_identity(): + dt = datetime(2026, 6, 19, 14, 0, 0, tzinfo=timezone.utc) + assert _hour_floor(dt) == dt + + +def test_rollup_cutoff_is_hour_aligned(): + now = datetime(2026, 6, 19, 14, 37, 52, tzinfo=timezone.utc) + cutoff = _rollup_cutoff(now, raw_days=7) + # 7 days back from 14:37 is 14:37 on the 12th, floored to 14:00. + assert cutoff == datetime(2026, 6, 12, 14, 0, 0, tzinfo=timezone.utc) + # No sub-hour remainder — only whole elapsed hours get rolled up. + assert cutoff.minute == 0 and cutoff.second == 0 and cutoff.microsecond == 0 + + +def test_rollup_cutoff_scales_with_raw_days(): + now = datetime(2026, 6, 19, 0, 0, 0, tzinfo=timezone.utc) + assert _rollup_cutoff(now, 1) == now - timedelta(days=1) + assert _rollup_cutoff(now, 30) == now - timedelta(days=30) + + +def test_rollup_cutoff_sample_classification(): + """A sample inside the raw window is kept; one before the aligned cutoff rolls up.""" + now = datetime(2026, 6, 19, 14, 37, 0, tzinfo=timezone.utc) + cutoff = _rollup_cutoff(now, raw_days=7) + recent = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc) # within window + old = datetime(2026, 6, 10, 9, 0, 0, tzinfo=timezone.utc) # older than cutoff + assert not (recent < cutoff) # kept raw + assert old < cutoff # rolled up From a840d6f8232f743d9aab114de91bf442769a6898 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 21:51:14 -0400 Subject: [PATCH 093/126] feat(docker): collect + persist /system/df image/disk usage (agent 1.6.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend for the image/disk panel (milestone 77 #942). Agent gains collect_disk_usage() — one /system/df call (gated on containers existing, so Docker-less hosts pay nothing), surfacing reclaimable bytes (image size held by unreferenced images), layers/containers/volumes/build-cache sizes, and the top 50 images by size. Emitted as sample["docker_disk"]; host_agent ingest tracks the newest sample's copy and hands it to the docker capability as a 5th arg. New current-state tables docker_disk_usage (one row/host) + docker_images (per-host image rows), docker_007 migration; ingest upserts the summary and replaces the image set per host (stale images pruned). Unit tests for the df parsing/reclaimable math + build_sample gating; integration test for persistence + image-set replacement. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- plugins/docker/ingest.py | 56 ++++++++++++- .../versions/docker_007_disk_usage.py | 55 +++++++++++++ plugins/docker/models.py | 50 ++++++++++++ plugins/host_agent/agent.py | 81 ++++++++++++++++++- plugins/host_agent/routes.py | 10 ++- tests/integration/test_docker.py | 54 +++++++++++++ tests/plugins/host_agent/test_agent_docker.py | 64 +++++++++++++++ 7 files changed, 367 insertions(+), 3 deletions(-) create mode 100644 plugins/docker/migrations/versions/docker_007_disk_usage.py diff --git a/plugins/docker/ingest.py b/plugins/docker/ingest.py index 1e57619..bf6726f 100644 --- a/plugins/docker/ingest.py +++ b/plugins/docker/ingest.py @@ -151,7 +151,57 @@ async def _persist_swarm(session, host, swarm: dict) -> None: await session.execute(stale_nodes) -async def persist_host_docker(session, host, snapshots, swarm=None) -> None: +async def _persist_disk(session, host, disk: dict) -> None: + """Upsert this host's /system/df summary + replace its image storage rows. + + Current-state (not time-series): the agent re-reports the full summary each + interval, so we overwrite the single summary row and replace the image set + (delete rows no longer present, upsert the rest), scoped to this host. + """ + from datetime import timezone + from .models import DockerDiskUsage, DockerImage + + now = datetime.now(timezone.utc) + + summary = await session.get(DockerDiskUsage, host.id) + if summary is None: + summary = DockerDiskUsage(host_id=host.id) + session.add(summary) + summary.layers_size = int(disk.get("layers_size") or 0) + summary.images_size = int(disk.get("images_size") or 0) + summary.images_reclaimable = int(disk.get("images_reclaimable") or 0) + summary.containers_size = int(disk.get("containers_size") or 0) + summary.volumes_size = int(disk.get("volumes_size") or 0) + summary.build_cache_size = int(disk.get("build_cache_size") or 0) + summary.images_total = int(disk.get("images_total") or 0) + summary.images_active = int(disk.get("images_active") or 0) + summary.containers_count = int(disk.get("containers_count") or 0) + summary.volumes_count = int(disk.get("volumes_count") or 0) + summary.updated_at = now + + images = disk.get("images") or [] + seen: set[str] = set() + for im in images: + iid = im.get("image_id") + if not iid or iid in seen: + continue + seen.add(iid) + row = await session.get(DockerImage, (host.id, iid)) + if row is None: + row = DockerImage(host_id=host.id, image_id=iid) + session.add(row) + row.repo_tag = (im.get("repo_tag") or "")[:512] + row.size = int(im.get("size") or 0) + row.shared_size = int(im.get("shared_size") or 0) + row.containers = int(im.get("containers") or 0) + row.updated_at = now + stale = delete(DockerImage).where(DockerImage.host_id == host.id) + if seen: + stale = stale.where(DockerImage.image_id.notin_(seen)) + await session.execute(stale) + + +async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -> None: """Upsert containers + time-series + lifecycle events + swarm for one host. `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) — @@ -160,12 +210,16 @@ async def persist_host_docker(session, host, snapshots, swarm=None) -> None: DockerMetric rows; the newest snapshot drives current container state, the alert pipeline, and lifecycle-event derivation. `swarm` is the newest sample's swarm object (or None off managers) — persisted when present. + `disk` is the newest sample's /system/df summary (or None on Docker-less + hosts) — persisted when present. """ from steward.core.alerts import record_metric from .models import DockerContainer, DockerEvent, DockerMetric if swarm is not None: await _persist_swarm(session, host, swarm) + if disk is not None: + await _persist_disk(session, host, disk) if not snapshots: return diff --git a/plugins/docker/migrations/versions/docker_007_disk_usage.py b/plugins/docker/migrations/versions/docker_007_disk_usage.py new file mode 100644 index 0000000..f6f0cb5 --- /dev/null +++ b/plugins/docker/migrations/versions/docker_007_disk_usage.py @@ -0,0 +1,55 @@ +"""Docker disk usage + image storage tables + +Adds docker_disk_usage (one per-host /system/df summary row) and docker_images +(the heavy-hitter image storage records). Both current-state, host-scoped, +re-reported each interval. Additive create_table. + +Revision ID: docker_007_disk_usage +Revises: docker_006_metric_rollup +Create Date: 2026-06-19 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_007_disk_usage" +down_revision: Union[str, None] = "docker_006_metric_rollup" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "docker_disk_usage", + sa.Column("host_id", sa.String(length=36), nullable=False), + sa.Column("layers_size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("images_size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("images_reclaimable", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("containers_size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("volumes_size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("build_cache_size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("images_total", sa.Integer(), nullable=False, server_default="0"), + sa.Column("images_active", sa.Integer(), nullable=False, server_default="0"), + sa.Column("containers_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("volumes_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("host_id"), + ) + op.create_table( + "docker_images", + sa.Column("host_id", sa.String(length=36), nullable=False), + sa.Column("image_id", sa.String(length=64), nullable=False), + sa.Column("repo_tag", sa.String(length=512), nullable=False, server_default=""), + sa.Column("size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("shared_size", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("containers", sa.Integer(), nullable=False, server_default="0"), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("host_id", "image_id"), + ) + + +def downgrade() -> None: + op.drop_table("docker_images") + op.drop_table("docker_disk_usage") diff --git a/plugins/docker/models.py b/plugins/docker/models.py index cfbfee8..04cb377 100644 --- a/plugins/docker/models.py +++ b/plugins/docker/models.py @@ -189,6 +189,56 @@ class DockerSwarmService(Base): ) +class DockerDiskUsage(Base): + """Per-host Docker disk usage summary from `/system/df` (current state). + + One row per host (the agent re-reports the full summary each interval). + Reclaimable = bytes held by images no container references. Sizes are bytes + (BigInteger — image caches exceed 2^31 easily). + """ + __tablename__ = "docker_disk_usage" + + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True + ) + layers_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + images_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + images_reclaimable: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + containers_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + volumes_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + build_cache_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + images_total: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + images_active: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + containers_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + volumes_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + +class DockerImage(Base): + """Per-host image storage record (the heavy hitters from `/system/df`). + + Keyed (host_id, image_id); `containers` is the reference count (0 ⇒ the + image's size is reclaimable). Replaced wholesale per host on each report. + """ + __tablename__ = "docker_images" + + host_id: Mapped[str] = mapped_column( + String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True + ) + image_id: Mapped[str] = mapped_column(String(64), primary_key=True) + repo_tag: Mapped[str] = mapped_column(String(512), nullable=False, default="") + size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + shared_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + containers: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + class DockerSwarmNode(Base): """A Swarm node as seen by a manager host: role / availability / status.""" __tablename__ = "docker_swarm_nodes" diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index d1acc08..98a237e 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -20,7 +20,7 @@ from collections import deque from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone -AGENT_VERSION = "1.5.0" +AGENT_VERSION = "1.6.0" # Default path to the local Docker Engine socket. Overridable via the # `docker_socket` config key; collection is silently skipped if it's absent or @@ -761,6 +761,74 @@ def collect_swarm(socket_path: str): } +# ─── disk usage (image storage) ────────────────────────────────────────────── + + +def _disk_images(images: list) -> list: + """Normalise /system/df image rows → compact per-image storage records. + + Sorted largest-first and capped so a host with a huge image cache doesn't + bloat the payload; the panel only needs the heavy hitters. `containers` is + how many containers reference the image (0 ⇒ reclaimable). + """ + out = [] + for im in images: + if not isinstance(im, dict): + continue + tags = im.get("RepoTags") or [] + # Untagged/dangling images report [":"] or null. + tag = next((t for t in tags if t and t != ":"), None) + raw_id = (im.get("Id") or "").split(":", 1)[-1] + out.append({ + "image_id": raw_id[:12], + "repo_tag": tag or "", + "size": int(im.get("Size") or 0), + "shared_size": int(im.get("SharedSize") or 0) if im.get("SharedSize", -1) >= 0 else 0, + "containers": int(im.get("Containers") or 0) if (im.get("Containers") or 0) > 0 else 0, + }) + out.sort(key=lambda r: r["size"], reverse=True) + return out[:50] + + +def collect_disk_usage(socket_path: str): + """Docker disk usage from `/system/df`, or None if unavailable. + + One bounded API call (the daemon walks images/layers/volumes). Reclaimable = + bytes held by images no container references — the same intent as + `docker system df`'s RECLAIMABLE column. Silent-skip (None) on any transport + failure, same contract as collect_docker/collect_swarm. + """ + try: + df = _docker_request(socket_path, "/system/df") + except (OSError, ValueError): + return None + if not isinstance(df, dict): + return None + + images = [im for im in (df.get("Images") or []) if isinstance(im, dict)] + containers = [c for c in (df.get("Containers") or []) if isinstance(c, dict)] + volumes = [v for v in (df.get("Volumes") or []) if isinstance(v, dict)] + build_cache = [b for b in (df.get("BuildCache") or []) if isinstance(b, dict)] + + def _vol_size(v): + return int((v.get("UsageData") or {}).get("Size") or 0) + + return { + "layers_size": int(df.get("LayersSize") or 0), + "images_total": len(images), + "images_active": sum(1 for im in images if (im.get("Containers") or 0) > 0), + "images_size": sum(int(im.get("Size") or 0) for im in images), + "images_reclaimable": sum( + int(im.get("Size") or 0) for im in images if (im.get("Containers") or 0) <= 0), + "containers_count": len(containers), + "containers_size": sum(int(c.get("SizeRw") or 0) for c in containers), + "volumes_count": len(volumes), + "volumes_size": sum(_vol_size(v) for v in volumes if _vol_size(v) > 0), + "build_cache_size": sum(int(b.get("Size") or 0) for b in build_cache), + "images": _disk_images(images), + } + + # ─── ring buffer ───────────────────────────────────────────────────────────── @@ -861,6 +929,17 @@ def build_sample(mounts: list[str], state: dict, if swarm is not None: sample["swarm"] = swarm + # Image/disk usage — host-level docker fact (one /system/df call), omitted on + # Docker-less hosts. Only meaningful when there's docker here at all, so skip + # the call entirely when collect_docker found nothing. + if docker: + try: + disk = collect_disk_usage(docker_socket) + except Exception: + disk = None + if disk is not None: + sample["docker_disk"] = disk + return sample diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 3e6c486..1b20286 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -199,6 +199,8 @@ async def ingest(): docker_snapshots: list[tuple[datetime, list]] = [] latest_swarm: dict | None = None latest_swarm_ts: datetime | None = None + latest_disk: dict | None = None + latest_disk_ts: datetime | None = None for sample in samples: try: recorded_at = _parse_ts(sample["ts"]) @@ -216,6 +218,11 @@ async def ingest(): if isinstance(swarm, dict) and ( latest_swarm_ts is None or recorded_at > latest_swarm_ts): latest_swarm, latest_swarm_ts = swarm, recorded_at + # Disk usage is current-state too — keep only the newest sample's. + disk = sample.get("docker_disk") + if isinstance(disk, dict) and ( + latest_disk_ts is None or recorded_at > latest_disk_ts): + latest_disk, latest_disk_ts = disk, recorded_at accepted += 1 if latest_ts is None or recorded_at > latest_ts: latest_ts = recorded_at @@ -228,7 +235,7 @@ async def ingest(): # (opportunistic synergy via the capability registry — no hard import, # no-op when docker is disabled). A failure here must never sink the # whole ingest, so the metrics above still land. - if docker_snapshots or latest_swarm is not None: + if docker_snapshots or latest_swarm is not None or latest_disk is not None: from steward.core.capabilities import has_capability, invoke_capability if has_capability("docker.persist_host_samples"): try: @@ -238,6 +245,7 @@ async def ingest(): await invoke_capability( "docker.persist_host_samples", UserRole.admin, session, host, docker_snapshots, latest_swarm, + latest_disk, ) except Exception: current_app.logger.exception( diff --git a/tests/integration/test_docker.py b/tests/integration/test_docker.py index 8e98265..1c4af60 100644 --- a/tests/integration/test_docker.py +++ b/tests/integration/test_docker.py @@ -230,6 +230,60 @@ def test_swarm_topology_persisted(app): assert node[0] == "manager" and node[2] == "ready" and node[3] is True +@_NEEDS_DB +def test_disk_usage_persisted(app): + """persist_host_docker stores the /system/df summary + image rows, and + replaces the image set on the next report (stale images pruned).""" + from sqlalchemy import text + from steward.models.hosts import Host + + persist = _persist_fn(app) + disk1 = { + "layers_size": 1000, "images_total": 2, "images_active": 1, + "images_size": 1000, "images_reclaimable": 700, + "containers_count": 2, "containers_size": 100, + "volumes_count": 1, "volumes_size": 40, "build_cache_size": 15, + "images": [ + {"image_id": "aaaaaaaaaaaa", "repo_tag": "app:1", "size": 300, + "shared_size": 50, "containers": 1}, + {"image_id": "bbbbbbbbbbbb", "repo_tag": "", "size": 700, + "shared_size": 0, "containers": 0}, + ], + } + disk2 = {**disk1, "images_reclaimable": 0, "images": [disk1["images"][0]]} + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + await s.execute(text("DELETE FROM docker_images")) + await s.execute(text("DELETE FROM docker_disk_usage")) + h = Host(id=str(uuid.uuid4()), name="dsk", address="10.6.6.6") + s.add(h) + await s.flush() + hid = h.id + await persist(s, h, [], None, disk1) + summary = (await s.execute(text( + "SELECT images_reclaimable, images_total, build_cache_size " + "FROM docker_disk_usage WHERE host_id=:h"), {"h": hid})).first() + img_count = (await s.execute(text( + "SELECT COUNT(*) FROM docker_images WHERE host_id=:h"), {"h": hid})).scalar() + # Re-report with one image dropped — the stale row must be pruned. + async with s.begin(): + await persist(s, await s.get(Host, hid), [], None, disk2) + after = (await s.execute(text( + "SELECT image_id FROM docker_images WHERE host_id=:h"), {"h": hid})).all() + reclaimable2 = (await s.execute(text( + "SELECT images_reclaimable FROM docker_disk_usage WHERE host_id=:h"), + {"h": hid})).scalar() + return summary, img_count, [r[0] for r in after], reclaimable2 + + summary, img_count, after_ids, reclaimable2 = asyncio.run(_go()) + assert summary[0] == 700 and summary[1] == 2 and summary[2] == 15 + assert img_count == 2 + assert after_ids == ["aaaaaaaaaaaa"] # the unreferenced image pruned on re-report + assert reclaimable2 == 0 # summary upserted in place + + @_NEEDS_DB def test_retention_rollup_and_prune(app): """Old raw metrics roll up to hourly averages then delete; stale rollup + diff --git a/tests/plugins/host_agent/test_agent_docker.py b/tests/plugins/host_agent/test_agent_docker.py index 67b3654..646f095 100644 --- a/tests/plugins/host_agent/test_agent_docker.py +++ b/tests/plugins/host_agent/test_agent_docker.py @@ -295,6 +295,7 @@ def test_collect_swarm_silent_skip_when_no_socket(): def test_build_sample_includes_docker_when_present(monkeypatch): monkeypatch.setattr(a, "collect_docker", lambda _s: [{"name": "web", "status": "running"}]) monkeypatch.setattr(a, "collect_swarm", lambda _s: None) + monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None) sample = a.build_sample(["/"], {}, "/var/run/docker.sock") assert sample["docker"] == [{"name": "web", "status": "running"}] @@ -321,6 +322,69 @@ def test_build_sample_omits_swarm_off_manager(monkeypatch): assert "swarm" not in sample +# ── disk usage (/system/df) ───────────────────────────────────────────────── + +def test_disk_images_normalises_and_caps(): + images = [ + {"Id": "sha256:aaaaaaaaaaaa1111", "RepoTags": ["nginx:latest"], + "Size": 200, "SharedSize": 50, "Containers": 2}, + {"Id": "sha256:bbbbbbbbbbbb2222", "RepoTags": [":"], + "Size": 500, "SharedSize": -1, "Containers": 0}, + ] + out = a._disk_images(images) + # Sorted largest-first; dangling tag normalised; short id; SharedSize<0 → 0. + assert out[0]["repo_tag"] == "" and out[0]["size"] == 500 + assert out[0]["shared_size"] == 0 and out[0]["containers"] == 0 + assert out[1]["repo_tag"] == "nginx:latest" and out[1]["image_id"] == "aaaaaaaaaaaa" + + +def test_collect_disk_usage_computes_reclaimable(monkeypatch): + df = { + "LayersSize": 1000, + "Images": [ + {"Id": "sha256:a", "RepoTags": ["app:1"], "Size": 300, "Containers": 1}, + {"Id": "sha256:b", "RepoTags": None, "Size": 700, "Containers": 0}, + ], + "Containers": [{"Id": "c1", "SizeRw": 25}, {"Id": "c2", "SizeRw": 75}], + "Volumes": [{"Name": "v1", "UsageData": {"Size": 40}}, + {"Name": "v2", "UsageData": {"Size": -1}}], + "BuildCache": [{"Size": 10}, {"Size": 5}], + } + monkeypatch.setattr(a, "_docker_request", + lambda s, p, timeout=a.DOCKER_API_TIMEOUT: df) + out = a.collect_disk_usage("/var/run/docker.sock") + assert out["layers_size"] == 1000 + assert out["images_total"] == 2 and out["images_active"] == 1 + assert out["images_size"] == 1000 + assert out["images_reclaimable"] == 700 # only the unreferenced image + assert out["containers_count"] == 2 and out["containers_size"] == 100 + assert out["volumes_count"] == 2 and out["volumes_size"] == 40 # negative skipped + assert out["build_cache_size"] == 15 + assert len(out["images"]) == 2 + + +def test_collect_disk_usage_silent_skip_when_no_socket(): + assert a.collect_disk_usage("/nonexistent/does-not-exist.sock") is None + + +def test_build_sample_includes_disk_when_docker_present(monkeypatch): + monkeypatch.setattr(a, "collect_docker", lambda _s: [{"name": "web", "status": "running"}]) + monkeypatch.setattr(a, "collect_swarm", lambda _s: None) + monkeypatch.setattr(a, "collect_disk_usage", lambda _s: {"images_reclaimable": 700}) + sample = a.build_sample(["/"], {}, "/var/run/docker.sock") + assert sample["docker_disk"] == {"images_reclaimable": 700} + + +def test_build_sample_omits_disk_when_no_docker(monkeypatch): + # No containers → the /system/df call is skipped entirely (cost guard). The + # sentinel would land under docker_disk if the gate ever called it. + monkeypatch.setattr(a, "collect_docker", lambda _s: []) + monkeypatch.setattr(a, "collect_swarm", lambda _s: None) + monkeypatch.setattr(a, "collect_disk_usage", lambda _s: {"sentinel": True}) + sample = a.build_sample(["/"], {}, "/var/run/docker.sock") + assert "docker_disk" not in sample + + # ── config ──────────────────────────────────────────────────────────────────── def test_read_config_defaults_docker_socket(tmp_path): From 277eb4016582c8eb89a1f8e983a444306fa4d14d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 21:54:54 -0400 Subject: [PATCH 094/126] fix(docker): close read txn before second begin in disk-usage test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_disk_usage_persisted opened a second session.begin() after interleaved SELECTs, which autobegin a transaction → "A transaction is already begun". Roll back the read transaction before the re-report write. Restores the integration lane green for the /system/df slice (a840d6f). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- tests/integration/test_docker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_docker.py b/tests/integration/test_docker.py index 1c4af60..722e9ef 100644 --- a/tests/integration/test_docker.py +++ b/tests/integration/test_docker.py @@ -267,6 +267,8 @@ def test_disk_usage_persisted(app): "FROM docker_disk_usage WHERE host_id=:h"), {"h": hid})).first() img_count = (await s.execute(text( "SELECT COUNT(*) FROM docker_images WHERE host_id=:h"), {"h": hid})).scalar() + # End the autobegun read transaction before opening the next write one. + await s.rollback() # Re-report with one image dropped — the stale row must be pruned. async with s.begin(): await persist(s, await s.get(Host, hid), [], None, disk2) From 3e4e35de960c1386a4b1e646287d0a2e2a4a10f6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 21:54:54 -0400 Subject: [PATCH 095/126] feat(docker): container detail page + lifecycle timeline (milestone 77 #942) New /plugins/docker/container// detail page (v1 quality): status/health badge, uptime, CPU/mem, restart count, last exit code (+OOM), net + block I/O (humanised), image/compose/swarm-service/node/ports, a range-toggled CPU/mem history graph (HTMX fragment reusing the time-range selector), and a lifecycle timeline rendered from docker_events (glyph+colour per event kind). Not-found and empty-history/empty-timeline states included. Container names in the main list and the per-host hub panel now link to it. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- plugins/docker/routes.py | 85 ++++++++++++- .../templates/docker/_container_history.html | 17 +++ .../templates/docker/container_detail.html | 112 ++++++++++++++++++ .../docker/templates/docker/host_panel.html | 2 +- plugins/docker/templates/docker/rows.html | 2 +- 5 files changed, 215 insertions(+), 3 deletions(-) create mode 100644 plugins/docker/templates/docker/_container_history.html create mode 100644 plugins/docker/templates/docker/container_detail.html diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index a6b38f2..7fe65c7 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -10,11 +10,25 @@ from steward.auth.middleware import require_role from steward.models.users import UserRole from steward.models.hosts import Host from steward.core.time_range import parse_range, DEFAULT_RANGE -from .models import DockerContainer, DockerMetric +from .models import DockerContainer, DockerEvent, DockerMetric docker_bp = Blueprint("docker", __name__, template_folder="templates") +def _human_bytes(n: int | None) -> str: + """Compact binary size string (e.g. '1.4 GiB', '512 MiB', '0 B').""" + if n is None: + return "—" + size = float(n) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if abs(size) < 1024.0 or unit == "TiB": + if unit == "B": + return f"{int(size)} {unit}" + return f"{size:.1f} {unit}" + size /= 1024.0 + return f"{size:.1f} TiB" + + def _human_uptime(started_at: datetime | None) -> str | None: """Compact 'how long running' string (e.g. '3d 4h', '5h 12m', '8m').""" if started_at is None: @@ -258,3 +272,72 @@ async def host_panel(host_id: str): running=running, stopped=len(containers) - running, ) + + +# Glyph + colour per lifecycle event, so the timeline reads at a glance. +_EVENT_STYLE = { + "start": ("▲", "var(--green)"), + "stop": ("■", "var(--text-muted)"), + "die": ("✕", "var(--red)"), + "oom": ("☠", "var(--red)"), + "health_change": ("◐", "var(--orange)"), +} + + +@docker_bp.get("/container//") +@require_role(UserRole.viewer) +async def container_detail(host_id: str, name: str): + """Full detail page for one container: facts, resource history, lifecycle.""" + async with current_app.db_sessionmaker() as db: + c = await db.get(DockerContainer, (host_id, name)) + host = await db.get(Host, host_id) + events = [] + if c is not None: + events = list((await db.execute( + select(DockerEvent) + .where(DockerEvent.host_id == host_id) + .where(DockerEvent.container_name == name) + .order_by(DockerEvent.at.desc()) + .limit(50) + )).scalars()) + + if c is None: + return await render_template( + "docker/container_detail.html", + container=None, host_id=host_id, name=name, + ), 404 + + return await render_template( + "docker/container_detail.html", + container=c, + host=host, + host_id=host_id, + name=name, + ports=json.loads(c.ports_json) if c.ports_json else [], + uptime=_human_uptime(c.started_at) if c.status == "running" else None, + net_rx=_human_bytes(c.net_rx_bytes), net_tx=_human_bytes(c.net_tx_bytes), + blk_read=_human_bytes(c.blk_read_bytes), blk_write=_human_bytes(c.blk_write_bytes), + events=[ + {"event": e.event, "detail": e.detail, "at": e.at, + "glyph": _EVENT_STYLE.get(e.event, ("•", "var(--text-muted)"))[0], + "colour": _EVENT_STYLE.get(e.event, ("•", "var(--text-muted)"))[1]} + for e in events + ], + current_range=request.args.get("range", DEFAULT_RANGE), + ) + + +@docker_bp.get("/container///history") +@require_role(UserRole.viewer) +async def container_history(host_id: str, name: str): + """HTMX fragment: CPU + memory sparklines for the selected time range.""" + since, range_key = parse_range(request.args.get("range")) + async with current_app.db_sessionmaker() as db: + cpu_hist, mem_hist = await _container_history(db, host_id, name, since) + return await render_template( + "docker/_container_history.html", + sparkline_cpu=_sparkline(cpu_hist, width=320, height=48), + sparkline_mem=_sparkline(mem_hist, width=320, height=48), + have_data=len(cpu_hist) >= 2, + range_key=range_key, + ) diff --git a/plugins/docker/templates/docker/_container_history.html b/plugins/docker/templates/docker/_container_history.html new file mode 100644 index 0000000..0a9b780 --- /dev/null +++ b/plugins/docker/templates/docker/_container_history.html @@ -0,0 +1,17 @@ +{# docker/_container_history.html — CPU/mem sparklines for the selected range #} +{% if have_data %} +
+
+
CPU %
+ {{ sparkline_cpu | safe }} +
+
+
Memory %
+ {{ sparkline_mem | safe }} +
+
+{% else %} +
+ Not enough samples in this range yet — history fills in as the agent reports. +
+{% endif %} diff --git a/plugins/docker/templates/docker/container_detail.html b/plugins/docker/templates/docker/container_detail.html new file mode 100644 index 0000000..377c6a2 --- /dev/null +++ b/plugins/docker/templates/docker/container_detail.html @@ -0,0 +1,112 @@ +{# docker/container_detail.html — full detail page for one container #} +{% extends "base.html" %} +{% block title %}{{ name }} — Docker — Steward{% endblock %} +{% block content %} + + + +{% if container is none %} +
+
Container not found
+
+ No container named {{ name }} is currently reported for this host. + It may have been removed, or the host agent hasn't reported recently. +
+
+{% else %} + +{# ── Header ──────────────────────────────────────────────────────────────── #} +
+ +

{{ container.name }}

+ {% if container.health == 'healthy' %}healthy + {% elif container.health == 'unhealthy' %}unhealthy + {% elif container.health == 'starting' %}starting{% endif %} +
+
+ {{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %} + {% if host %} · on {{ host.name }}{% endif %} +
+ +{# ── Facts grid ──────────────────────────────────────────────────────────── #} +{% macro fact(label, value, colour="") %} +
+
{{ label }}
+
{{ value }}
+
+{% endmacro %} + +
+ {{ fact("CPU", "%.1f%%" | format(container.cpu_pct) if container.cpu_pct is not none else "—") }} + {{ fact("Memory", "%.1f%%" | format(container.mem_pct) if container.mem_pct is not none else "—") }} + {{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }} + {{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—", + "var(--red)" if (container.exit_code is not none and container.exit_code != 0) else "") }} + {{ fact("Net in", net_rx) }} + {{ fact("Net out", net_tx) }} + {{ fact("Block read", blk_read) }} + {{ fact("Block write", blk_write) }} +
+ +{# ── Image / placement ───────────────────────────────────────────────────── #} +
+
+ Image + {{ container.image or "—" }} + {% if container.compose_project %} + Compose project{{ container.compose_project }} + {% endif %} + {% if container.service_name %} + Swarm service{{ container.service_name }} + {% endif %} + {% if container.node_id %} + Node{{ container.node_id }} + {% endif %} + {% if ports %} + Ports + {% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %} + {% endif %} +
+
+ +{# ── Resource history (range-toggled) ────────────────────────────────────── #} +
+
+

Resource history

+ {% include "_time_range.html" %} +
+
+
Loading…
+
+
+ +{# ── Lifecycle timeline ──────────────────────────────────────────────────── #} +
+

Lifecycle

+ {% if events %} +
+ {% for e in events %} +
+ {{ e.glyph }} + {{ e.event }} + {{ e.detail or "" }} + {{ e.at.strftime("%Y-%m-%d %H:%M") }} +
+ {% endfor %} +
+ {% else %} +
+ No lifecycle events recorded yet. Start/stop/health changes appear here as the + agent reports them over time. +
+ {% endif %} +
+ +{% endif %} +{% endblock %} diff --git a/plugins/docker/templates/docker/host_panel.html b/plugins/docker/templates/docker/host_panel.html index 1666174..942aef4 100644 --- a/plugins/docker/templates/docker/host_panel.html +++ b/plugins/docker/templates/docker/host_panel.html @@ -11,7 +11,7 @@ {% for c in containers %}
- {{ c.name }} + {{ c.name }} {% if c.cpu_pct is not none %}{{ "%.1f" | format(c.cpu_pct) }}%{% endif %} {% if c.mem_pct is not none %} · {{ "%.1f" | format(c.mem_pct) }}%{% endif %} diff --git a/plugins/docker/templates/docker/rows.html b/plugins/docker/templates/docker/rows.html index 6dbfca8..66088c0 100644 --- a/plugins/docker/templates/docker/rows.html +++ b/plugins/docker/templates/docker/rows.html @@ -55,7 +55,7 @@
- {{ c.name }} + {{ c.name }} {% if c.health == 'healthy' %}● {% elif c.health == 'unhealthy' %}● {% elif c.health == 'starting' %}◐{% endif %} From 114262dbf9ae602f0ade639687eb43d66c8288bc Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 21:58:19 -0400 Subject: [PATCH 096/126] feat(docker): swarm topology view + image/disk usage page (milestone 77 #942) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new read-only sub-pages, linked from the Docker index header only when the data exists (non-swarm / Docker-less installs aren't offered empty pages): - /plugins/docker/swarm — services with replica health (running/desired, colour-coded green/amber/red), Swarm nodes (role/availability/status, leader badge), and task→node placement with node ids resolved to hostnames. Grouped by reporting manager host. Empty state explains manager-only collection. - /plugins/docker/disk — per-host reclaimable space, image/layer/container/ volume/build-cache sizes, stopped-container count, and a per-image table (size, shared, ref count, reclaimable badge). Notes prune actions are deferred. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- plugins/docker/routes.py | 125 ++++++++++++++++++++- plugins/docker/templates/docker/disk.html | 84 ++++++++++++++ plugins/docker/templates/docker/index.html | 10 +- plugins/docker/templates/docker/swarm.html | 86 ++++++++++++++ 4 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 plugins/docker/templates/docker/disk.html create mode 100644 plugins/docker/templates/docker/swarm.html diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index 7fe65c7..98d10a8 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -10,7 +10,10 @@ from steward.auth.middleware import require_role from steward.models.users import UserRole from steward.models.hosts import Host from steward.core.time_range import parse_range, DEFAULT_RANGE -from .models import DockerContainer, DockerEvent, DockerMetric +from .models import ( + DockerContainer, DockerDiskUsage, DockerEvent, DockerImage, DockerMetric, + DockerSwarmNode, DockerSwarmService, +) docker_bp = Blueprint("docker", __name__, template_folder="templates") @@ -106,10 +109,19 @@ async def _container_history(db, host_id: str, name: str, since) -> tuple[list, async def index(): poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) current_range = request.args.get("range", DEFAULT_RANGE) + # Show the Swarm link only when a manager has actually reported topology, so + # non-swarm installs aren't offered an always-empty page. + async with current_app.db_sessionmaker() as db: + has_swarm = (await db.execute( + select(DockerSwarmService.host_id).limit(1))).first() is not None + has_disk = (await db.execute( + select(DockerDiskUsage.host_id).limit(1))).first() is not None return await render_template( "docker/index.html", poll_interval=poll_interval, current_range=current_range, + has_swarm=has_swarm, + has_disk=has_disk, ) @@ -327,6 +339,117 @@ async def container_detail(host_id: str, name: str): ) +@docker_bp.get("/swarm") +@require_role(UserRole.viewer) +async def swarm(): + """Swarm topology page: services with replica health, nodes, placement. + + Swarm tables are manager-scoped (each manager reports its own view), so we + group by reporting host — usually one group for a single-manager cluster. + """ + async with current_app.db_sessionmaker() as db: + services = list((await db.execute( + select(DockerSwarmService).order_by(DockerSwarmService.service_name) + )).scalars()) + nodes = list((await db.execute( + select(DockerSwarmNode).order_by( + DockerSwarmNode.role, DockerSwarmNode.hostname) + )).scalars()) + hosts = await _host_map(db, {s.host_id for s in services} | {n.host_id for n in nodes}) + + # node_id → hostname per reporting host, so placement reads as names not ids. + node_names: dict[tuple[str, str], str] = { + (n.host_id, n.node_id): (n.hostname or n.node_id) for n in nodes + } + + groups: dict[str, dict] = {} + for s in services: + g = groups.setdefault(s.host_id, {"services": [], "nodes": []}) + placement = [] + for p in (json.loads(s.placement_json) if s.placement_json else []): + nid = p.get("node_id", "") + placement.append({ + "node": node_names.get((s.host_id, nid), nid[:12] or "?"), + "running": p.get("running", 0), + }) + g["services"].append({ + "name": s.service_name, "mode": s.mode, + "running": s.running, "desired": s.desired, "image": s.image, + "healthy": s.running >= s.desired and s.desired > 0, + "degraded": 0 < s.running < s.desired, + "down": s.running == 0 and s.desired > 0, + "placement": placement, + }) + for n in nodes: + g = groups.setdefault(n.host_id, {"services": [], "nodes": []}) + g["nodes"].append(n) + + host_groups = [ + {"host_id": hid, + "host_name": hosts[hid].name if hid in hosts else hid, + "host": hosts.get(hid), + **g} + for hid, g in groups.items() + ] + host_groups.sort(key=lambda g: g["host_name"].lower()) + return await render_template("docker/swarm.html", host_groups=host_groups) + + +@docker_bp.get("/disk") +@require_role(UserRole.viewer) +async def disk(): + """Image/disk usage page: reclaimable space, per-image sizes, stopped count. + + Read-only — prune actions are deferred to the cleanup-actions milestone, so + this surfaces the numbers and notes where reclaim lives. + """ + async with current_app.db_sessionmaker() as db: + summaries = list((await db.execute(select(DockerDiskUsage))).scalars()) + images = list((await db.execute( + select(DockerImage).order_by(DockerImage.size.desc()) + )).scalars()) + # Stopped-container count per host, surfaced alongside reclaimable space. + stopped_rows = (await db.execute( + select(DockerContainer.host_id) + .where(DockerContainer.status != "running") + )).scalars() + stopped_by_host: dict[str, int] = {} + for hid in stopped_rows: + stopped_by_host[hid] = stopped_by_host.get(hid, 0) + 1 + hosts = await _host_map(db, {s.host_id for s in summaries}) + + images_by_host: dict[str, list] = {} + for im in images: + images_by_host.setdefault(im.host_id, []).append({ + "repo_tag": im.repo_tag, "size": _human_bytes(im.size), + "shared": _human_bytes(im.shared_size), "containers": im.containers, + "reclaimable": im.containers == 0, + }) + + host_groups = [ + { + "host_id": s.host_id, + "host_name": hosts[s.host_id].name if s.host_id in hosts else s.host_id, + "summary": { + "images_total": s.images_total, "images_active": s.images_active, + "images_size": _human_bytes(s.images_size), + "images_reclaimable": _human_bytes(s.images_reclaimable), + "layers_size": _human_bytes(s.layers_size), + "containers_count": s.containers_count, + "containers_size": _human_bytes(s.containers_size), + "volumes_count": s.volumes_count, + "volumes_size": _human_bytes(s.volumes_size), + "build_cache_size": _human_bytes(s.build_cache_size), + }, + "stopped": stopped_by_host.get(s.host_id, 0), + "images": images_by_host.get(s.host_id, []), + } + for s in summaries + ] + host_groups.sort(key=lambda g: g["host_name"].lower()) + return await render_template("docker/disk.html", host_groups=host_groups) + + @docker_bp.get("/container///history") @require_role(UserRole.viewer) async def container_history(host_id: str, name: str): diff --git a/plugins/docker/templates/docker/disk.html b/plugins/docker/templates/docker/disk.html new file mode 100644 index 0000000..8d50149 --- /dev/null +++ b/plugins/docker/templates/docker/disk.html @@ -0,0 +1,84 @@ +{# docker/disk.html — image/disk usage: reclaimable space, per-image sizes #} +{% extends "base.html" %} +{% block title %}Disk — Docker — Steward{% endblock %} +{% block content %} + + +

Image & disk usage

+

+ Reclaimable = space held by images no container references. Cleanup actions + (prune) arrive in a later release — these are read-only figures for now. +

+ +{% if host_groups %} +{% for g in host_groups %} +
+
{{ g.host_name }}
+ + {# ── Summary stats ────────────────────────────────────────────────────── #} +
+
+
Reclaimable
+ {{ g.summary.images_reclaimable }} +
+
+
Images
+ {{ g.summary.images_size }} +
{{ g.summary.images_active }}/{{ g.summary.images_total }} in use
+
+
+
Stopped
+ {{ g.stopped }} +
+
+
Writable layers
+ {{ g.summary.containers_size }} +
{{ g.summary.containers_count }} containers
+
+
+
Volumes
+ {{ g.summary.volumes_size }} +
{{ g.summary.volumes_count }} volumes
+
+
+
Build cache
+ {{ g.summary.build_cache_size }} +
+
+ + {# ── Per-image table ──────────────────────────────────────────────────── #} +
+ + + + + + {% for im in g.images %} + + + + + + + {% else %} + + {% endfor %} + +
ImageSizeSharedContainers
+ {{ im.repo_tag }} + {% if im.reclaimable %}reclaimable{% endif %} + {{ im.size }}{{ im.shared }}{{ im.containers }}
No images reported.
+
+
+{% endfor %} +{% else %} +
+
+ No disk usage reported yet. The host agent reports image/disk usage from + docker system df on hosts running Docker. +
+
+{% endif %} +{% endblock %} diff --git a/plugins/docker/templates/docker/index.html b/plugins/docker/templates/docker/index.html index 9b33918..950dd06 100644 --- a/plugins/docker/templates/docker/index.html +++ b/plugins/docker/templates/docker/index.html @@ -2,7 +2,15 @@ {% block title %}Docker — Steward{% endblock %} {% block content %}
-

Docker

+
+

Docker

+ {% if has_swarm %} + Swarm → + {% endif %} + {% if has_disk %} + Disk → + {% endif %} +
{% include "_time_range.html" %}
diff --git a/plugins/docker/templates/docker/swarm.html b/plugins/docker/templates/docker/swarm.html new file mode 100644 index 0000000..b8314d1 --- /dev/null +++ b/plugins/docker/templates/docker/swarm.html @@ -0,0 +1,86 @@ +{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement #} +{% extends "base.html" %} +{% block title %}Swarm — Docker — Steward{% endblock %} +{% block content %} + + +

Swarm

+ +{% if host_groups %} +{% for g in host_groups %} +
+
+ {{ g.host_name }} + manager view +
+ + {# ── Services ─────────────────────────────────────────────────────────── #} +
+ + + + + + + + {% for s in g.services %} + + + + + + + + {% else %} + + {% endfor %} + +
ServiceModeReplicasImagePlacement
{{ s.name }}{{ s.mode }} + + {{ s.running }}/{{ s.desired }} + + {{ s.image or "—" }} + {% for p in s.placement %}{{ p.node }} ({{ p.running }}){% if not loop.last %}, {% endif %}{% else %}—{% endfor %} +
No services on this manager.
+
+ + {# ── Nodes ────────────────────────────────────────────────────────────── #} +
+ + + + + + {% for n in g.nodes %} + + + + + + + {% else %} + + {% endfor %} + +
NodeRoleAvailabilityStatus
+ {{ n.hostname or n.node_id }} + {% if n.leader %}leader{% endif %} + {{ n.role }}{{ n.availability }} + + {{ n.status }} +
No nodes reported.
+
+
+{% endfor %} +{% else %} +
+
+ No Swarm topology reported. A Steward host agent running on a Swarm + manager reports services, nodes, and task placement here — + workers and non-swarm hosts contribute only their local containers. +
+
+{% endif %} +{% endblock %} From 9615f9abcdfbce20adeda9e0a3f943cb706c4900 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 22:01:12 -0400 Subject: [PATCH 097/126] feat(docker): group containers by compose/swarm + enrich widget (milestone 77 #942) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Container list now sub-groups each host's containers by compose project (or swarm service) with a small subheading, preserving the running-first order; hosts with no such labels render flat as before. The dashboard status widget links each container to its detail page and surfaces enriched state inline — health dot (healthy/unhealthy), restart count, and last non-zero exit code for stopped containers. Completes the #942 UI surface. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- plugins/docker/routes.py | 17 +++++++++++++++++ plugins/docker/templates/docker/rows.html | 7 ++++++- plugins/docker/templates/docker/widget.html | 9 ++++++--- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index 98d10a8..3dc9f55 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -175,6 +175,23 @@ async def rows(): else: g["stopped"] += 1 + # Sub-group each host's containers by compose project (or swarm service), + # preserving the running-first ordering. Containers with neither label fall + # into an unlabelled bucket rendered flat, so plain hosts look unchanged. + for g in groups.values(): + subs: dict[str, dict] = {} + order: list[str] = [] + for item in g["containers"]: + c = item["container"] + label = c.compose_project or c.service_name or None + key = label or "" + if key not in subs: + subs[key] = {"label": label, "containers": []} + order.append(key) + subs[key]["containers"].append(item) + g["subgroups"] = [subs[k] for k in order] + g["grouped"] = any(s["label"] for s in g["subgroups"]) + host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower()) running = sum(g["running"] for g in host_groups) total = len(containers) diff --git a/plugins/docker/templates/docker/rows.html b/plugins/docker/templates/docker/rows.html index 66088c0..9d3b058 100644 --- a/plugins/docker/templates/docker/rows.html +++ b/plugins/docker/templates/docker/rows.html @@ -47,7 +47,11 @@ - {% for item in g.containers %} + {% for sub in g.subgroups %} + {% if g.grouped and sub.label %} + {{ sub.label }} + {% endif %} + {% for item in sub.containers %} {% set c = item.container %} @@ -105,6 +109,7 @@ {{ item.sparkline_mem | safe }} {% endfor %} + {% endfor %}
diff --git a/plugins/docker/templates/docker/widget.html b/plugins/docker/templates/docker/widget.html index 9a8a5a4..cd66b22 100644 --- a/plugins/docker/templates/docker/widget.html +++ b/plugins/docker/templates/docker/widget.html @@ -24,9 +24,12 @@ {% for c in g.containers %}
- - {{ c.name }} - + + {{ c.name }}{% if c.health == 'unhealthy' %} ●{% elif c.health == 'healthy' %} ●{% endif %}{% if c.restart_count %} ⟳{{ c.restart_count }}{% endif %} + + {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %} + exit {{ c.exit_code }} + {% endif %} {% if c.cpu_pct is not none %} {{ "%.1f" | format(c.cpu_pct) }}% From 626ba699346df0ae0acced3a9492725b25f2ccc2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 22:04:11 -0400 Subject: [PATCH 098/126] test(docker): parse-check templates + smoke routes/_human_bytes (milestone 77 #943) CI never renders docker templates through the app (the unit-lane app uses testing=True, which skips plugin loading), so a Jinja syntax error could ship green. Add unit tests that parse every docker template, smoke-import the routes module + confirm the #942 view functions exist, and cover the _human_bytes / _human_uptime presentation helpers. Closes the render-regression gap the new UI pages introduced. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- tests/plugins/docker/test_routes.py | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/plugins/docker/test_routes.py diff --git a/tests/plugins/docker/test_routes.py b/tests/plugins/docker/test_routes.py new file mode 100644 index 0000000..d74776b --- /dev/null +++ b/tests/plugins/docker/test_routes.py @@ -0,0 +1,50 @@ +"""Unit tests for the docker plugin's presentation layer (no DB). + +CI never renders these templates through the running app (the unit-lane app is +created with testing=True, which skips plugin loading), so a Jinja syntax error +would otherwise ship green. These tests parse every docker template and smoke +the routes module so a broken tag or import is caught in the unit lane. +""" +import pathlib + +import jinja2 + +from plugins.docker import routes as r + +_TEMPLATES = pathlib.Path(r.__file__).parent / "templates" / "docker" + + +def test_all_docker_templates_parse(): + env = jinja2.Environment() + files = sorted(_TEMPLATES.glob("*.html")) + assert files, "no docker templates found" + for f in files: + # Raises TemplateSyntaxError on an unbalanced/typo'd tag. + env.parse(f.read_text()) + + +def test_routes_module_exposes_new_views(): + # Import-smoke + confirms the #942 view functions are defined. + for name in ("container_detail", "container_history", "swarm", "disk", "index", "rows"): + assert callable(getattr(r, name)), name + assert r.docker_bp.name == "docker" + + +def test_human_bytes_formats_binary_units(): + assert r._human_bytes(None) == "—" + assert r._human_bytes(0) == "0 B" + assert r._human_bytes(512) == "512 B" + assert r._human_bytes(1024) == "1.0 KiB" + assert r._human_bytes(1536) == "1.5 KiB" + assert r._human_bytes(1024 ** 2) == "1.0 MiB" + assert r._human_bytes(int(1.5 * 1024 ** 3)) == "1.5 GiB" + assert r._human_bytes(1024 ** 4) == "1.0 TiB" + + +def test_human_uptime_compact(): + from datetime import datetime, timedelta, timezone + now = datetime.now(timezone.utc) + assert r._human_uptime(None) is None + assert r._human_uptime(now - timedelta(minutes=8)).endswith("m") + assert "h" in r._human_uptime(now - timedelta(hours=5, minutes=12)) + assert "d" in r._human_uptime(now - timedelta(days=3, hours=4)) From 88091936c523eb64e1bc0bef7996560b907fb717 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 18 Jun 2026 23:07:12 -0400 Subject: [PATCH 099/126] fix(ui): unify header/breadcrumb treatment, drop redundant back buttons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page-top chrome was inconsistent: most nested views used the breadcrumb kicker + page-title pattern, but plugin sub-pages used ad-hoc "← back" links, ~11 pages stacked a breadcrumb AND a redundant ancestor back button, and two (monitors/edit, hosts/uptime) had a back button but no breadcrumb. The settings/plugin_detail page stacked all of it at once. Unify on the breadcrumb-led model: - settings/_tabs.html: drop the hardcoded "Settings" h1; the breadcrumb ("Settings › …") plus the tab strip is the header. - settings/plugin_detail: drop the "← Plugins" back button. - docker container_detail/swarm/disk + snmp/device: replace ad-hoc back links with the standard crumbs() breadcrumb. - host_agent, ansible/*, alerts/maintenance: remove redundant ancestor back buttons (the breadcrumb's parent crumbs already link there); keep lateral shortcuts (Inventory/Schedules/Browse/Targets/Groups/New). - monitors/edit, hosts/uptime: add the missing breadcrumb, drop the back link. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- plugins/docker/templates/docker/container_detail.html | 6 ++---- plugins/docker/templates/docker/disk.html | 5 ++--- plugins/docker/templates/docker/swarm.html | 5 ++--- plugins/host_agent/templates/host_detail.html | 1 - plugins/host_agent/templates/settings_list.html | 1 - plugins/snmp/templates/snmp/device.html | 3 ++- steward/templates/alerts/maintenance.html | 1 - steward/templates/ansible/browse.html | 2 -- steward/templates/ansible/inventory/group_detail.html | 1 - steward/templates/ansible/inventory/groups.html | 1 - steward/templates/ansible/inventory/target_detail.html | 1 - steward/templates/ansible/inventory/targets.html | 1 - steward/templates/ansible/playbook_editor.html | 1 - steward/templates/ansible/run_detail.html | 1 - steward/templates/ansible/schedules.html | 1 - steward/templates/hosts/uptime.html | 3 ++- steward/templates/monitors/edit.html | 5 +++-- steward/templates/settings/_tabs.html | 5 +++-- steward/templates/settings/plugin_detail.html | 1 - 19 files changed, 16 insertions(+), 29 deletions(-) diff --git a/plugins/docker/templates/docker/container_detail.html b/plugins/docker/templates/docker/container_detail.html index 377c6a2..bf4d76e 100644 --- a/plugins/docker/templates/docker/container_detail.html +++ b/plugins/docker/templates/docker/container_detail.html @@ -1,12 +1,10 @@ {# docker/container_detail.html — full detail page for one container #} {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}{{ name }} — Docker — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %} {% block content %} - - {% if container is none %}
Container not found
diff --git a/plugins/docker/templates/docker/disk.html b/plugins/docker/templates/docker/disk.html index 8d50149..22a0e16 100644 --- a/plugins/docker/templates/docker/disk.html +++ b/plugins/docker/templates/docker/disk.html @@ -1,11 +1,10 @@ {# docker/disk.html — image/disk usage: reclaimable space, per-image sizes #} {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Disk — Docker — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Image & disk usage", "")]) }}{% endblock %} {% block content %} -

Image & disk usage

Reclaimable = space held by images no container references. Cleanup actions diff --git a/plugins/docker/templates/docker/swarm.html b/plugins/docker/templates/docker/swarm.html index b8314d1..35f3caf 100644 --- a/plugins/docker/templates/docker/swarm.html +++ b/plugins/docker/templates/docker/swarm.html @@ -1,11 +1,10 @@ {# docker/swarm.html — Swarm topology: services, replica health, nodes, placement #} {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Swarm — Docker — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Swarm", "")]) }}{% endblock %} {% block content %} -

Swarm

{% if host_groups %} diff --git a/plugins/host_agent/templates/host_detail.html b/plugins/host_agent/templates/host_detail.html index 42df49e..9b18f86 100644 --- a/plugins/host_agent/templates/host_detail.html +++ b/plugins/host_agent/templates/host_detail.html @@ -27,7 +27,6 @@ {% block content %}
- ← Host

{{ host.name }}

{% if stale %}stale{% else %}live{% endif %}
diff --git a/plugins/host_agent/templates/settings_list.html b/plugins/host_agent/templates/settings_list.html index 80c027f..93cf0d4 100644 --- a/plugins/host_agent/templates/settings_list.html +++ b/plugins/host_agent/templates/settings_list.html @@ -5,7 +5,6 @@ {% block content %}

Agent fleet

- ← Hosts

Bulk agent operations across your inventory. For a single host, use its diff --git a/plugins/snmp/templates/snmp/device.html b/plugins/snmp/templates/snmp/device.html index b8fe625..b0b1a18 100644 --- a/plugins/snmp/templates/snmp/device.html +++ b/plugins/snmp/templates/snmp/device.html @@ -1,9 +1,10 @@ {# plugins/snmp/templates/snmp/device.html #} {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}SNMP — {{ device_name }} — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("SNMP", "/plugins/snmp/"), (device_name, "")]) }}{% endblock %} {% block content %}

- ← SNMP

{{ device_name }}

{{ device.host }}
diff --git a/steward/templates/alerts/maintenance.html b/steward/templates/alerts/maintenance.html index 6061a7e..1126e67 100644 --- a/steward/templates/alerts/maintenance.html +++ b/steward/templates/alerts/maintenance.html @@ -6,7 +6,6 @@

Maintenance Windows

diff --git a/steward/templates/ansible/browse.html b/steward/templates/ansible/browse.html index ba30058..d133869 100644 --- a/steward/templates/ansible/browse.html +++ b/steward/templates/ansible/browse.html @@ -17,7 +17,6 @@ {% endif %} Inventory Schedules - ← Run History
@@ -35,7 +34,6 @@ {% endif %} - ← Back to browse 
{{ view_contents }}
diff --git a/steward/templates/ansible/inventory/group_detail.html b/steward/templates/ansible/inventory/group_detail.html index 84e6975..c5373ff 100644 --- a/steward/templates/ansible/inventory/group_detail.html +++ b/steward/templates/ansible/inventory/group_detail.html @@ -5,7 +5,6 @@ {% block content %}

Group: {{ group.name }}

- ← Groups
diff --git a/steward/templates/ansible/inventory/groups.html b/steward/templates/ansible/inventory/groups.html index da2b1e1..5a1126b 100644 --- a/steward/templates/ansible/inventory/groups.html +++ b/steward/templates/ansible/inventory/groups.html @@ -7,7 +7,6 @@

Inventory Groups

diff --git a/steward/templates/ansible/inventory/target_detail.html b/steward/templates/ansible/inventory/target_detail.html index 4837f72..a2d4df6 100644 --- a/steward/templates/ansible/inventory/target_detail.html +++ b/steward/templates/ansible/inventory/target_detail.html @@ -5,7 +5,6 @@ {% block content %}

Target: {{ target.name }}

- ← Targets
diff --git a/steward/templates/ansible/inventory/targets.html b/steward/templates/ansible/inventory/targets.html index f92d3d4..750e6ab 100644 --- a/steward/templates/ansible/inventory/targets.html +++ b/steward/templates/ansible/inventory/targets.html @@ -7,7 +7,6 @@

Inventory Targets

diff --git a/steward/templates/ansible/playbook_editor.html b/steward/templates/ansible/playbook_editor.html index 4d5d51e..8628a2a 100644 --- a/steward/templates/ansible/playbook_editor.html +++ b/steward/templates/ansible/playbook_editor.html @@ -5,7 +5,6 @@ {% block content %}

{{ "Edit playbook" if editing else "New playbook" }}

- ← Browse
{% if error %}
{{ error }}
{% endif %} diff --git a/steward/templates/ansible/run_detail.html b/steward/templates/ansible/run_detail.html index 0545ac8..ec0b06f 100644 --- a/steward/templates/ansible/run_detail.html +++ b/steward/templates/ansible/run_detail.html @@ -4,7 +4,6 @@ {% block breadcrumb %}{{ crumbs([("Ansible", "/ansible/"), ("Run " ~ run.id[:8], "")]) }}{% endblock %} {% block content %}
- ← Runs

Run Detail

{% if run.status.value in ("running", "queued") and session.user_role in ("operator", "admin") %} diff --git a/steward/templates/ansible/schedules.html b/steward/templates/ansible/schedules.html index f9c9d9e..d54fd86 100644 --- a/steward/templates/ansible/schedules.html +++ b/steward/templates/ansible/schedules.html @@ -14,7 +14,6 @@
diff --git a/steward/templates/hosts/uptime.html b/steward/templates/hosts/uptime.html index 7a7230e..ff0d0ac 100644 --- a/steward/templates/hosts/uptime.html +++ b/steward/templates/hosts/uptime.html @@ -1,9 +1,10 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% block title %}Uptime / SLA — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), ("Uptime / SLA", "")]) }}{% endblock %} {% block content %}

Uptime / SLA

- ← Hosts
{# ── Summary pills ──────────────────────────────────────────────────────────── #} diff --git a/steward/templates/monitors/edit.html b/steward/templates/monitors/edit.html index ac383b8..4df8b02 100644 --- a/steward/templates/monitors/edit.html +++ b/steward/templates/monitors/edit.html @@ -1,10 +1,11 @@ {% extends "base.html" %} +{% from "_macros.html" import crumbs %} {% from "monitors/_fields.html" import type_fields, toggle_script %} {% block title %}Edit Monitor — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Monitors", "/monitors/"), ("Edit monitor", "")]) }}{% endblock %} {% block content %}
- ← Monitors -

Edit Monitor

+

Edit Monitor

{% set inp = "width:100%;padding:0.4rem 0.65rem;background:var(--bg);border:1px solid var(--border-mid);border-radius:4px;color:var(--text);font-size:0.88rem;" %} diff --git a/steward/templates/settings/_tabs.html b/steward/templates/settings/_tabs.html index 97e1201..490ad20 100644 --- a/steward/templates/settings/_tabs.html +++ b/steward/templates/settings/_tabs.html @@ -1,5 +1,6 @@ -{# settings/_tabs.html — include at top of each settings section #} -

Settings

+{# settings/_tabs.html — include at top of each settings section. + No section

: the breadcrumb kicker ("Settings › …") names the section and + the tab strip is the visual header, so we don't stack a redundant title. #}
{% set tabs = [ ("general", "General", "/settings/general/"), diff --git a/steward/templates/settings/plugin_detail.html b/steward/templates/settings/plugin_detail.html index b0a1fd7..91a7d76 100644 --- a/steward/templates/settings/plugin_detail.html +++ b/steward/templates/settings/plugin_detail.html @@ -8,7 +8,6 @@ {% include "settings/_tabs.html" %}
- ← Plugins

{{ plugin.get('name', plugin._dir) }}

v{{ plugin.get('version', '?') }} {% if fail_reason %} From 4fc8c96c41c3a08e8ca25d234bf92cc4fc69e81b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 19 Jun 2026 13:28:07 -0400 Subject: [PATCH 100/126] fix(snmp): bundle pysnmp in image and port poller to the asyncio HLAPI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SNMP plugin ships in the image but logged "pysnmp not installed — SNMP polling disabled" on every poll, so polling never worked. Two coupled defects: 1. The Dockerfile installed only `.[ansible]`, so the `snmp` extra (pysnmp) was never bundled even though the plugin is first-party and shipped. 2. poller.py used the synchronous pysnmp HLAPI (`next(getCmd(...))`), which pysnmp-lextudio 6.x removed — it's asyncio-only now — so even with the dep present, polling would have thrown and silently returned nothing. The 5.x line that still has the sync API isn't safe on the image's Python 3.13. Fix: - Dockerfile: install `.[ansible,snmp]`. - poller.py: `poll_device_sync` → `async def poll_device` on the asyncio HLAPI, with a dual-version import (pysnmp 7.x `pysnmp.hlapi.v3arch.asyncio`/`get_cmd` + async `UdpTransportTarget.create`; pysnmp-lextudio 6.2.x `pysnmp.hlapi.asyncio`/`getCmd` + direct `UdpTransportTarget`) so a dependency bump can't silently re-break it. - scheduler.py: await poll_device directly; drop the run_in_executor wrapper and the now-unused asyncio import. - Add tests/plugins/snmp/test_poller.py covering the version→mpModel mapping, that the poller is a coroutine, and the graceful no-pysnmp path. Note: CI confirms import/load and the no-pysnmp path, but has no SNMP target — live polling against real devices is verified after deploy. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- Dockerfile | 6 +- plugins/snmp/poller.py | 91 +++++++++++++++++++++---------- plugins/snmp/scheduler.py | 11 +--- tests/plugins/snmp/__init__.py | 0 tests/plugins/snmp/test_poller.py | 32 +++++++++++ 5 files changed, 101 insertions(+), 39 deletions(-) create mode 100644 tests/plugins/snmp/__init__.py create mode 100644 tests/plugins/snmp/test_poller.py diff --git a/Dockerfile b/Dockerfile index e74ceca..fc8a570 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,8 +19,10 @@ WORKDIR /app COPY pyproject.toml . COPY steward/ steward/ -# .[ansible] pulls the full Ansible package so the playbook runner works in-image. -RUN pip install --no-cache-dir '.[ansible]' +# Bundle the extras whose first-party plugins ship in the image: [ansible] for +# the playbook runner, [snmp] (pysnmp) for the SNMP poller. Without them those +# bundled plugins load but silently no-op for want of their runtime dependency. +RUN pip install --no-cache-dir '.[ansible,snmp]' COPY alembic.ini . # First-party plugins ship inside the image (bundled root at /app/plugins). diff --git a/plugins/snmp/poller.py b/plugins/snmp/poller.py index 6a837f6..22c7c84 100644 --- a/plugins/snmp/poller.py +++ b/plugins/snmp/poller.py @@ -1,11 +1,13 @@ # plugins/snmp/poller.py """ -Synchronous SNMP GET helper, run via executor. +Asynchronous SNMP GET helper. -Requires pysnmp-lextudio (maintained pysnmp fork): +Requires pysnmp-lextudio (the maintained pysnmp fork), bundled into the Docker +image via the `snmp` extra (`pip install .[snmp]`): pip install 'steward[snmp]' -If pysnmp is not installed, poll_device() returns an empty dict and logs a warning. +If pysnmp is not installed, poll_device() returns an empty dict and logs a +warning — SNMP polling is then simply disabled, nothing else breaks. """ from __future__ import annotations import logging @@ -22,39 +24,74 @@ def _pysnmp_available() -> bool: def _mp_model(version: str) -> int: - """Map version string to pysnmp mpModel integer.""" + """Map an SNMP version string to pysnmp's mpModel int (0 = v1, 1 = v2c).""" return 0 if version == "1" else 1 -def poll_device_sync( +async def poll_device( host: str, port: int, community: str, version: str, oids: list[dict], ) -> dict[str, float]: - """ - Perform SNMP GET for each OID and return {label: float_value}. - Non-numeric OIDs (strings, etc.) are skipped. - Returns empty dict on any error. + """Perform an SNMP GET for each OID and return ``{label: float_value}``. + + Non-numeric OIDs (strings, etc.) are skipped. Returns an empty dict on any + error (unreachable host, wrong community, …) so a flaky device never breaks + the poll loop. + + pysnmp's HLAPI is asyncio-only as of v6. The import path moved between major + versions, so we support both rather than let a dependency bump silently + re-break polling: + • pysnmp-lextudio 6.2.x → ``pysnmp.hlapi.asyncio`` (``getCmd`` + a directly + constructed ``UdpTransportTarget``). + • canonical pysnmp 7.x → ``pysnmp.hlapi.v3arch.asyncio`` (``get_cmd`` + the + async ``UdpTransportTarget.create``). """ if not _pysnmp_available(): logger.warning("pysnmp not installed — SNMP polling disabled. " "Install with: pip install 'steward[snmp]'") return {} - from pysnmp.hlapi import ( - CommunityData, - ContextData, - ObjectIdentity, - ObjectType, - SnmpEngine, - UdpTransportTarget, - getCmd, - ) + try: + # canonical pysnmp 7.x + from pysnmp.hlapi.v3arch.asyncio import ( + CommunityData, + ContextData, + ObjectIdentity, + ObjectType, + SnmpEngine, + UdpTransportTarget, + get_cmd as _get_cmd, + ) + _transport_is_async = True + except ImportError: + # pysnmp-lextudio 6.2.x + from pysnmp.hlapi.asyncio import ( + CommunityData, + ContextData, + ObjectIdentity, + ObjectType, + SnmpEngine, + UdpTransportTarget, + getCmd as _get_cmd, + ) + _transport_is_async = False + + engine = SnmpEngine() + + # Same host/port for every OID on this device, so build the transport once. + try: + if _transport_is_async: + transport = await UdpTransportTarget.create((host, port), timeout=5, retries=1) + else: + transport = UdpTransportTarget((host, port), timeout=5, retries=1) + except Exception as exc: + logger.debug("SNMP transport setup failed for %s:%s: %s", host, port, exc) + return {} results: dict[str, float] = {} - engine = SnmpEngine() for oid_cfg in oids: oid = oid_cfg["oid"] @@ -62,14 +99,12 @@ def poll_device_sync( scale = float(oid_cfg.get("scale", 1.0)) try: - error_indication, error_status, error_index, var_binds = next( - getCmd( - engine, - CommunityData(community, mpModel=_mp_model(version)), - UdpTransportTarget((host, port), timeout=5, retries=1), - ContextData(), - ObjectType(ObjectIdentity(oid)), - ) + error_indication, error_status, error_index, var_binds = await _get_cmd( + engine, + CommunityData(community, mpModel=_mp_model(version)), + transport, + ContextData(), + ObjectType(ObjectIdentity(oid)), ) except Exception as exc: logger.debug("SNMP GET %s@%s OID %s failed: %s", host, port, oid, exc) @@ -88,7 +123,7 @@ def poll_device_sync( try: results[label] = float(val) * scale except (TypeError, ValueError): - # Non-numeric type (e.g. OctetString description) — skip + # Non-numeric type (e.g. OctetString description) — skip. logger.debug("SNMP non-numeric value for %s label=%s: %r", oid, label, val) return results diff --git a/plugins/snmp/scheduler.py b/plugins/snmp/scheduler.py index d1a1d85..e0d4df7 100644 --- a/plugins/snmp/scheduler.py +++ b/plugins/snmp/scheduler.py @@ -1,6 +1,5 @@ # plugins/snmp/scheduler.py from __future__ import annotations -import asyncio import logging from typing import TYPE_CHECKING @@ -27,15 +26,13 @@ def make_poll_task(app: "Quart") -> ScheduledTask: async def _do_poll(app: "Quart") -> None: - from .poller import poll_device_sync + from .poller import poll_device from steward.core.alerts import record_metric devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", []) if not devices: return - loop = asyncio.get_event_loop() - async with app.db_sessionmaker() as session: async with session.begin(): for device in devices: @@ -52,11 +49,7 @@ async def _do_poll(app: "Quart") -> None: continue try: - readings = await loop.run_in_executor( - None, - poll_device_sync, - host, port, community, version, oids, - ) + readings = await poll_device(host, port, community, version, oids) except Exception: logger.exception("SNMP poll failed for device %s (%s)", name, host) continue diff --git a/tests/plugins/snmp/__init__.py b/tests/plugins/snmp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/plugins/snmp/test_poller.py b/tests/plugins/snmp/test_poller.py new file mode 100644 index 0000000..151f928 --- /dev/null +++ b/tests/plugins/snmp/test_poller.py @@ -0,0 +1,32 @@ +"""Unit tests for the SNMP poller (no pysnmp / no network). + +The unit lane doesn't install the `snmp` extra, so these exercise the parts that +don't need pysnmp: the version→mpModel mapping, that the poller is async, and the +graceful "pysnmp missing → empty result" path. Live polling against a real device +is verified out-of-band (CI has no SNMP target). Uses asyncio.run() directly so it +doesn't depend on the pytest-asyncio mode. +""" +import asyncio +import inspect + +from plugins.snmp import poller + + +def test_poll_device_is_coroutine(): + # The scheduler awaits it directly (no executor) — it must be async. + assert inspect.iscoroutinefunction(poller.poll_device) + + +def test_mp_model_maps_version_to_int(): + assert poller._mp_model("1") == 0 # SNMP v1 + assert poller._mp_model("2c") == 1 # SNMP v2c + assert poller._mp_model("2") == 1 # anything non-"1" → v2c model + + +def test_poll_device_without_pysnmp_returns_empty(monkeypatch): + # When the optional dep is absent, polling is disabled gracefully (no raise). + monkeypatch.setattr(poller, "_pysnmp_available", lambda: False) + out = asyncio.run( + poller.poll_device("192.0.2.1", 161, "public", "2c", [{"oid": "1.3.6.1.2.1.1.3.0"}]) + ) + assert out == {} From 0940dc6972af2ebe1ca7f4e7782c8d20f471c481 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 19 Jun 2026 13:52:27 -0400 Subject: [PATCH 101/126] feat(crypto): fail loudly on unpersistable secret key; flag undecryptable secrets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the incident where an unwritable /data made Steward silently mint a fresh ephemeral secret key on every boot, orphaning all encrypted secrets — only discovered when an Ansible run failed. Make both failure modes loud and visible. - config._resolve_secret_key: if a new key must be generated but can't be persisted (no STEWARD_SECRET_KEY, unwritable /data), raise RuntimeError and refuse to start, with an actionable fix (set STEWARD_SECRET_KEY, or make /data writable by uid 1000). Also raises a clear error if an existing key file can't be read. First-run on a writable volume still generates + persists normally. - core.settings: detect secrets stored as ciphertext that won't decrypt with the current key (scan_undecryptable_secrets at startup; _is_undecryptable helper). Cached in memory; set_setting discards a key on a fresh write so the banner clears without a restart. - app.py: run the scan after migrate_plaintext_secrets, log a warning, and inject undecryptable_secrets into the template context. - base.html: admin banner naming which secrets to re-enter, each linked to its settings tab. - compose.deploy.yml: document the uid-1000 /data write requirement and the STEWARD_SECRET_KEY option for multi-node Swarm. - Tests: secret-key resolver (reuse existing file, generate when writable, raise when unpersistable) and the undecryptable-secret detection helper. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC --- compose.deploy.yml | 12 ++++- steward/app.py | 17 ++++++- steward/config.py | 35 ++++++++++--- steward/core/settings.py | 65 ++++++++++++++++++++++++ steward/templates/base.html | 28 ++++++++++ tests/core/test_undecryptable_secrets.py | 29 +++++++++++ tests/test_config.py | 31 ++++++++++- 7 files changed, 206 insertions(+), 11 deletions(-) create mode 100644 tests/core/test_undecryptable_secrets.py diff --git a/compose.deploy.yml b/compose.deploy.yml index bb08e55..c5bd8fb 100644 --- a/compose.deploy.yml +++ b/compose.deploy.yml @@ -22,12 +22,22 @@ services: # /data holds the auto-generated secret key, third-party plugins, and the # Ansible playbook cache — all of which must survive image updates. No # source bind-mounts: core + first-party plugins live inside the image. + # + # The container runs as the unprivileged 'app' user (uid 1000), so /data + # MUST be writable by uid 1000. A named docker volume (below) gets that + # automatically. If you bind-mount a host/NFS path instead, chown it to + # 1000:1000 (and mind NFS root_squash) — otherwise the app can't persist + # its secret key, mints a fresh one each boot, and every encrypted secret + # (managed SSH key, SMTP/OIDC/LDAP creds) becomes unrecoverable. The app + # now refuses to start in that state rather than silently degrade. - app_data:/data environment: - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:${POSTGRES_PASSWORD:-steward}@db/steward # STEWARD_SECRET_KEY is intentionally absent — the app generates a random # key on first boot and persists it to /data/secret.key, which the app_data - # volume keeps across image updates. + # volume keeps across image updates. On multi-node Swarm with a shared bind + # mount, pin STEWARD_SECRET_KEY (env or a docker secret) instead, so the key + # never depends on filesystem ownership being correct on every node. depends_on: db: condition: service_healthy diff --git a/steward/app.py b/steward/app.py index 29b26f4..8e5915e 100644 --- a/steward/app.py +++ b/steward/app.py @@ -1,11 +1,14 @@ # steward/app.py from __future__ import annotations import asyncio +import logging from pathlib import Path from quart import Quart, render_template from .config import load_bootstrap from .database import init_db +logger = logging.getLogger(__name__) + def create_app( config_path: Path | str | None = None, @@ -50,9 +53,17 @@ def create_app( # ── 3b. Encryption-at-rest: bind the encryptor, then encrypt legacy secrets ─ if not testing: from .core.crypto import init_crypto - from .core.settings import migrate_plaintext_secrets + from .core.settings import migrate_plaintext_secrets, scan_undecryptable_secrets init_crypto(app.config["SECRET_KEY"]) migrate_plaintext_secrets(app.config["DATABASE_URL"]) + # Flag any secrets sealed under a now-lost key (see the admin banner). + undecryptable = scan_undecryptable_secrets(app.config["DATABASE_URL"]) + if undecryptable: + logger.warning( + "%d stored secret(s) cannot be decrypted with the current app key " + "(re-enter them in Settings): %s", + len(undecryptable), ", ".join(undecryptable), + ) # ── 4. Load all settings from DB → populate app.config ──────────────────── if not testing: @@ -167,6 +178,7 @@ def create_app( @app.context_processor def _inject_plugin_failures(): from .core.plugin_manager import get_plugin_failures + from .core.settings import get_undecryptable_secrets # enabled_plugins lets templates gate cross-plugin embeds (e.g. the # Hosts hub only fetches the docker fragment when docker is enabled), # avoiding a 404 to a route whose blueprint isn't registered. @@ -177,6 +189,9 @@ def create_app( return { "plugin_failures": get_plugin_failures(), "enabled_plugins": enabled_plugins, + # Read from an in-memory cache (no DB hit per render); kept current by + # the startup scan + set_setting's discard-on-write. + "undecryptable_secrets": get_undecryptable_secrets(), } # ── 11. Share-token middleware ───────────────────────────────────────────── diff --git a/steward/config.py b/steward/config.py index 9c8f1fd..678359e 100644 --- a/steward/config.py +++ b/steward/config.py @@ -75,13 +75,27 @@ def load_bootstrap(config_path: Path | str | None = None) -> dict[str, Any]: def _resolve_secret_key(raw: dict) -> str: - """Resolve secret_key: env var → file → auto-generate.""" + """Resolve secret_key: env var → file → auto-generate. + + Refuses to start if a new key must be generated but cannot be persisted: an + ephemeral key changes on every restart, which silently renders every + encrypted secret (managed SSH key, SMTP/OIDC/LDAP credentials) unrecoverable. + Failing loudly with a fix beats limping along and losing data on the next + boot — exactly the footgun that bit the vdnt-docker02 deployment. + """ from_env = _env("SECRET_KEY") or raw.get("secret_key") if from_env: return from_env if _SECRET_KEY_FILE.exists(): - key = _SECRET_KEY_FILE.read_text().strip() + try: + key = _SECRET_KEY_FILE.read_text().strip() + except OSError as exc: + raise RuntimeError( + f"App secret key file {_SECRET_KEY_FILE} exists but cannot be read " + f"({exc}). Make it readable by the container user (uid 1000), or set " + f"STEWARD_SECRET_KEY." + ) from exc if key: return key @@ -89,11 +103,16 @@ def _resolve_secret_key(raw: dict) -> str: try: _SECRET_KEY_FILE.parent.mkdir(parents=True, exist_ok=True) _SECRET_KEY_FILE.write_text(key) - logger.info("Generated new secret key and saved to %s", _SECRET_KEY_FILE) except OSError as exc: - logger.warning( - "Could not write secret key to %s (%s). " - "Key will not persist across restarts.", - _SECRET_KEY_FILE, exc, - ) + raise RuntimeError( + f"Generated a new app secret key but could not persist it to " + f"{_SECRET_KEY_FILE} ({exc}). An ephemeral key changes on every restart, " + f"which makes all encrypted secrets (managed SSH key, SMTP/OIDC/LDAP " + f"credentials) unrecoverable. Fix one of:\n" + f" • set STEWARD_SECRET_KEY to a stable value " + f"(recommended for Swarm / multi-node), or\n" + f" • make {_SECRET_KEY_FILE.parent} writable by the container user " + f"(uid 1000)." + ) from exc + logger.info("Generated new secret key and saved to %s", _SECRET_KEY_FILE) return key diff --git a/steward/core/settings.py b/steward/core/settings.py index 23bece8..339dc75 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -150,6 +150,34 @@ def _decode(value: Any, key: str = "") -> Any: return decrypt_secret(value, context=key) if is_encrypted(value) else value +# Secret settings that are stored as ciphertext but won't decrypt with the +# current app key (key rotated or lost). Surfaced as an admin banner so the +# operator knows precisely which secrets to re-enter — instead of finding out +# only when something that uses one fails. Refreshed at startup +# (scan_undecryptable_secrets) and kept live: re-entering a secret clears it +# without a restart (set_setting discards it on a fresh write). +_undecryptable_secrets: set[str] = set() + + +def get_undecryptable_secrets() -> list[str]: + """Sorted keys whose stored ciphertext won't decrypt (for the UI banner).""" + return sorted(_undecryptable_secrets) + + +def _is_undecryptable(stored: Any, key: str = "") -> bool: + """True iff `stored` is an encrypted token that fails to decrypt. + + A failed decrypt returns the ciphertext unchanged (still enc-prefixed), so a + value that is still encrypted after a decrypt attempt is undecryptable. + """ + from steward.core.crypto import decrypt_secret, is_encrypted + return ( + isinstance(stored, str) + and is_encrypted(stored) + and is_encrypted(decrypt_secret(stored, context=key)) + ) + + # ───────────────────────────────────────────────────────────────────────────── # Async helpers (use inside request handlers / scheduled tasks) # ───────────────────────────────────────────────────────────────────────────── @@ -182,6 +210,12 @@ async def set_setting(session: AsyncSession, key: str, value: Any) -> None: row.value_json = json.dumps(to_store) row.updated_at = now + # A fresh write of a secret is encrypted with the current key (or cleared to + # plaintext), so it's decryptable now — drop any stale "undecryptable" flag + # so the banner clears without a restart. + if key in SECRET_KEYS: + _undecryptable_secrets.discard(key) + async def get_all_settings(session: AsyncSession) -> dict[str, Any]: """Return flat key→value dict with defaults filled in for missing keys.""" @@ -359,6 +393,37 @@ def migrate_plaintext_secrets(db_url: str) -> int: return asyncio.run(_run()) +def scan_undecryptable_secrets(db_url: str) -> list[str]: + """Populate the undecryptable-secrets cache from the DB; return the keys found. + + Run once at startup, AFTER migrate_plaintext_secrets and init_crypto: any row + that's encrypted but won't decrypt with the current key was sealed under a + different (lost/rotated) key and must be re-entered. Surfaced via the admin + banner (get_undecryptable_secrets). + """ + async def _run() -> list[str]: + from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker + engine = create_async_engine(db_url, echo=False) + factory = async_sessionmaker(engine, expire_on_commit=False) + bad: list[str] = [] + try: + async with factory() as session: + result = await session.execute( + select(AppSetting).where(AppSetting.key.in_(SECRET_KEYS)) + ) + for row in result.scalars(): + if _is_undecryptable(json.loads(row.value_json), row.key): + bad.append(row.key) + finally: + await engine.dispose() + return bad + + global _undecryptable_secrets + found = asyncio.run(_run()) + _undecryptable_secrets = set(found) + return sorted(found) + + # ───────────────────────────────────────────────────────────────────────────── # External URL helper # ───────────────────────────────────────────────────────────────────────────── diff --git a/steward/templates/base.html b/steward/templates/base.html index 3a03dfa..7b7a95d 100644 --- a/steward/templates/base.html +++ b/steward/templates/base.html @@ -280,6 +280,34 @@ function setTimeRange(val) {
{% endif %} + {% if undecryptable_secrets and session.user_role == 'admin' %} + {% set _sec_tab = { + 'smtp.password': '/settings/notifications/', + 'oidc.client_secret': '/settings/auth/', + 'ldap.bind_password': '/settings/auth/', + 'ansible.ssh_private_key': '/settings/ansible/', + 'ansible.become_password': '/settings/ansible/', + 'ansible.vault_password': '/settings/ansible/', + } %} +
+ ⚠ + + + {{ undecryptable_secrets | length }} stored secret{{ 's' if undecryptable_secrets | length != 1 }} + can’t be decrypted + + — the app secret key changed, so {{ 'they' if undecryptable_secrets | length != 1 else 'it' }} + must be re-entered: + {% for k in undecryptable_secrets -%} + {%- if _sec_tab.get(k) %}{{ k }} + {%- else %}{{ k }}{% endif %}{% if not loop.last %}, {% endif %} + {%- endfor %}. + +
+ {% endif %} {% if error %}
{{ error }}
{% endif %} diff --git a/tests/core/test_undecryptable_secrets.py b/tests/core/test_undecryptable_secrets.py new file mode 100644 index 0000000..c515ca0 --- /dev/null +++ b/tests/core/test_undecryptable_secrets.py @@ -0,0 +1,29 @@ +"""Unit tests for undecryptable-secret detection (the admin-banner source).""" +from steward.core.crypto import encrypt_secret, init_crypto +from steward.core.settings import _is_undecryptable, get_undecryptable_secrets + + +def test_is_undecryptable_false_for_current_key(): + init_crypto("key-A") + tok = encrypt_secret("hunter2") + assert _is_undecryptable(tok, "smtp.password") is False + + +def test_is_undecryptable_true_after_key_rotation(): + init_crypto("key-A") + tok = encrypt_secret("hunter2") # sealed under key-A + init_crypto("key-B") # key changed/lost + assert _is_undecryptable(tok, "smtp.password") is True + + +def test_is_undecryptable_ignores_plaintext_and_empty(): + init_crypto("key-A") + assert _is_undecryptable("", "smtp.password") is False + assert _is_undecryptable("plain-value", "smtp.password") is False + assert _is_undecryptable(None, "smtp.password") is False + + +def test_get_undecryptable_secrets_returns_sorted_list(): + # Default (nothing scanned) is an empty list; the accessor is always safe to + # call from the template context processor. + assert isinstance(get_undecryptable_secrets(), list) diff --git a/tests/test_config.py b/tests/test_config.py index 40e9203..93d482a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,6 @@ import textwrap import pytest -from steward.config import load_bootstrap +from steward.config import load_bootstrap, _resolve_secret_key def test_load_bootstrap_from_yaml(tmp_path): @@ -36,3 +36,32 @@ def test_missing_database_url_raises(tmp_path): cfg_file.write_text("secret_key: s\n") with pytest.raises(ValueError, match="Database URL is required"): load_bootstrap(cfg_file) + + +# ── secret key resolution ────────────────────────────────────────────────────── + +def test_secret_key_existing_file_is_reused(tmp_path, monkeypatch): + key_file = tmp_path / "secret.key" + key_file.write_text("persisted-key\n") + monkeypatch.setattr("steward.config._SECRET_KEY_FILE", key_file) + monkeypatch.delenv("STEWARD_SECRET_KEY", raising=False) + assert _resolve_secret_key({}) == "persisted-key" + + +def test_secret_key_generated_and_persisted_when_writable(tmp_path, monkeypatch): + key_file = tmp_path / "sub" / "secret.key" # parent doesn't exist yet + monkeypatch.setattr("steward.config._SECRET_KEY_FILE", key_file) + monkeypatch.delenv("STEWARD_SECRET_KEY", raising=False) + key = _resolve_secret_key({}) + assert key and key_file.read_text().strip() == key + + +def test_secret_key_unpersistable_raises_instead_of_ephemeral(tmp_path, monkeypatch): + # A file sits where a directory is needed, so mkdir/write fails → must raise + # (an ephemeral key would silently orphan every encrypted secret). + blocker = tmp_path / "blocker" + blocker.write_text("not a dir") + monkeypatch.setattr("steward.config._SECRET_KEY_FILE", blocker / "secret.key") + monkeypatch.delenv("STEWARD_SECRET_KEY", raising=False) + with pytest.raises(RuntimeError, match="could not persist"): + _resolve_secret_key({}) From 95ebdf7045390a9b2fb78083e1483c01431255c3 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 19 Jun 2026 15:34:47 -0400 Subject: [PATCH 102/126] feat(ui): replace flat top nav with a grouped left sidebar (slice 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flat top bar was a set of ungrouped peers and gave plugin data no home. Move to a persistent left sidebar with grouped sections, per the navigation redesign (operator-chosen shell). - base.html: top