FabledSteward/steward/settings/routes.py

# steward/settings/routes.py
from __future__ import annotations
import html
import logging
from datetime import datetime
from pathlib import Path
from quart import Blueprint, current_app, render_template, request, redirect, url_for, session

from steward.auth.middleware import require_role
from steward.core.audit import audit_request, log_audit
from steward.core.logging_setup import LEVELS as LOG_LEVELS, set_log_level
from steward.models.users import UserRole
from steward.core.settings import (
    get_all_settings, set_setting, delete_setting,
    get_stored_plugin_names, find_orphaned_plugins,
    to_smtp_cfg, to_webhook_cfg, to_ansible_cfg, to_plugins_cfg,
    to_oidc_cfg, to_ldap_cfg, to_thresholds_cfg, to_access_cfg, DEFAULTS, seal_source_tokens,
    is_plugin_secret,
)

settings_bp = Blueprint("settings", __name__, url_prefix="/settings")
logger = logging.getLogger(__name__)


def _discover_plugins() -> list[dict]:
    """Scan every plugin root for plugin.yaml files.

    Roots are scanned in order (bundled first), and a plugin name found in an
    earlier root shadows the same name in a later one — matching load_plugins.
    """
    import yaml
    plugin_dirs = [
        Path(d).resolve()
        for d in current_app.config.get("PLUGIN_DIRS", ["plugins"])
    ]
    plugins: list[dict] = []
    seen: set[str] = set()
    for plugin_dir in plugin_dirs:
        if not plugin_dir.exists():
            continue
        for entry in sorted(plugin_dir.iterdir()):
            yaml_path = entry / "plugin.yaml"
            if entry.is_dir() and yaml_path.exists() and entry.name not in seen:
                try:
                    with yaml_path.open() as f:
                        meta = yaml.safe_load(f) or {}
                    meta["_dir"] = entry.name
                    plugins.append(meta)
                    seen.add(entry.name)
                except Exception:
                    logger.warning("Could not read %s", yaml_path)
    return plugins


def _merge_plugin_config(discovered: list[dict], plugins_cfg: dict) -> None:
    """Merge stored settings into discovered plugin dicts in-place."""
    for plugin in discovered:
        name = plugin["_dir"]
        stored = plugins_cfg.get(name, {})
        plugin["_enabled"] = stored.get("enabled", False)
        defaults = plugin.get("config", {})
        plugin["_config"] = {**defaults, **{k: v for k, v in stored.items() if k != "enabled"}}


async def _reload_app_config() -> None:
    """Reload app.config from DB so settings take effect without restart."""
    async with current_app.db_sessionmaker() as db:
        fresh = await get_all_settings(db)
    current_app.config.update(to_access_cfg(fresh))
    current_app.config.update(
        DATA_RETENTION_DAYS=fresh.get("data.retention_days", 90),
        MONITORS_POLL_INTERVAL=fresh.get("monitors.poll_interval_seconds", 60),
        PUBLIC_BASE_URL=fresh.get("general.public_base_url", ""),
        SMTP=to_smtp_cfg(fresh),
        WEBHOOK=to_webhook_cfg(fresh),
        ANSIBLE=to_ansible_cfg(fresh),
        PLUGINS=to_plugins_cfg(fresh),
        OIDC=to_oidc_cfg(fresh),
        LDAP=to_ldap_cfg(fresh),
        THRESHOLDS=to_thresholds_cfg(fresh),
    )


# ── Redirect root to general tab ──────────────────────────────────────────────

@settings_bp.get("/")
@require_role(UserRole.admin)
async def index():
    return redirect(url_for("settings.general"))


# ── General ───────────────────────────────────────────────────────────────────

@settings_bp.get("/general/")
@require_role(UserRole.admin)
async def general():
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    return await render_template("settings/general.html", settings=settings,
                                 log_levels=LOG_LEVELS)


@settings_bp.post("/general/")
@require_role(UserRole.admin)
async def save_general():
    form = await request.form
    whole: dict[str, int] = {}
    for key in ("session.lifetime_hours", "general.trusted_proxy_hops",
                "login.max_failures", "login.lockout_minutes", "audit.retention_days"):
        try:
            whole[key] = int(form.get(key, DEFAULTS[key]))
        except (TypeError, ValueError):
            return f"{key} must be a whole number", 400
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            await set_setting(db, "general.public_base_url",
                              (form.get("general.public_base_url", "") or "").strip())
            for key, value in whole.items():
                await set_setting(db, key, value)
            await set_setting(db, "data.retention_days",
                              int(form.get("data.retention_days", 90)))
            await set_setting(db, "monitors.poll_interval_seconds",
                              int(form.get("monitors.poll_interval_seconds", 60)))
            await set_setting(db, "log.level", set_log_level(form.get("log.level")))
    await _reload_app_config()
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "settings.saved", detail={"section": "general"})
    return redirect(url_for("settings.general"))


# ── Monitoring thresholds ─────────────────────────────────────────────────────

# (form field, setting key, is_float) for every tunable cutoff.
_THRESHOLD_FIELDS = [
    ("cpu_warn", "thresholds.cpu_warn", False), ("cpu_crit", "thresholds.cpu_crit", False),
    ("mem_warn", "thresholds.mem_warn", False), ("mem_crit", "thresholds.mem_crit", False),
    ("disk_warn", "thresholds.disk_warn", False), ("disk_crit", "thresholds.disk_crit", False),
    ("load_warn", "thresholds.load_warn", False), ("load_crit", "thresholds.load_crit", False),
    ("temp_warn", "thresholds.temp_warn", False), ("temp_crit", "thresholds.temp_crit", False),
    ("uptime_warn", "thresholds.uptime_warn", True), ("uptime_crit", "thresholds.uptime_crit", True),
    ("latency_warn", "ping.threshold.good_ms", False), ("latency_crit", "ping.threshold.warn_ms", False),
    ("tls_warn", "thresholds.tls_warn_days", False), ("tls_crit", "thresholds.tls_crit_days", False),
    ("clock_warn", "thresholds.clock_warn_s", False), ("clock_crit", "thresholds.clock_crit_s", False),
    ("crash_loop_deaths", "thresholds.crash_loop_deaths", False),
    ("error_5xx_pct", "traefik.threshold.error_5xx_pct", True),
    ("error_4xx_pct", "traefik.threshold.error_4xx_pct", True),
]

# Docker time-series retention windows (days). Read fresh by the cleanup task,
# so a save takes effect on the next hourly run — no app.config wiring needed.
_RETENTION_FIELDS = [
    ("docker_metrics_raw_days", "docker.retention.metrics_raw_days"),
    ("docker_metrics_rollup_days", "docker.retention.metrics_rollup_days"),
    ("docker_events_days", "docker.retention.events_days"),
    ("docker_logs_retention_days", "docker.logs.retention_days"),
    ("metrics_raw_days", "metrics.retention.raw_days"),
    ("metrics_rollup_days", "metrics.retention.rollup_days"),
]


@settings_bp.get("/thresholds/")
@require_role(UserRole.admin)
async def thresholds():
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    from steward.core.plugin_manager import _LOADED_PLUGINS
    return await render_template("settings/thresholds.html", settings=settings,
                                 traefik_on="traefik" in _LOADED_PLUGINS)


@settings_bp.post("/thresholds/")
@require_role(UserRole.admin)
async def save_thresholds():
    form = await request.form
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            for field, key, is_float in _THRESHOLD_FIELDS:
                raw = form.get(field, "")
                if raw == "":
                    continue
                try:
                    val = float(raw) if is_float else int(raw)
                except (TypeError, ValueError):
                    continue
                await set_setting(db, key, val)
            for field, key in _RETENTION_FIELDS:
                raw = form.get(field, "")
                if raw == "":
                    continue
                try:
                    val = max(1, int(raw))  # at least a day — 0 would prune everything
                except (TypeError, ValueError):
                    continue
                await set_setting(db, key, val)
            # Container-log controls (m79). Checkbox: present ⇒ on (this is a
            # full-page form, so absence is a genuine "off"). Exclude: comma-split
            # names. Size: entered in MB, stored as bytes.
            await set_setting(db, "docker.logs.enabled", "docker_logs_enabled" in form)
            exclude = [n.strip() for n in form.get("docker_logs_exclude", "").split(",")
                       if n.strip()]
            await set_setting(db, "docker.logs.exclude", exclude)
            max_mb = form.get("docker_logs_max_mb", "")
            if max_mb != "":
                try:
                    await set_setting(db, "docker.logs.max_bytes_per_container",
                                      max(1, int(max_mb)) * 1_000_000)
                except (TypeError, ValueError):
                    pass
    await _reload_app_config()
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "settings.saved", detail={"section": "thresholds"})
    return redirect(url_for("settings.thresholds"))


# ── Notifications (SMTP + Webhook) ────────────────────────────────────────────

@settings_bp.get("/notifications/")
@require_role(UserRole.admin)
async def notifications():
    from steward.core.formatting import zones_by_region
    from steward.core.mail import audience
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
        if not settings["smtp.tz"] and session.get("tz"):
            # "Also send to" reads times in the first admin's zone until one is chosen.
            # Reading the settings already began this session's transaction.
            await set_setting(db, "smtp.tz", session["tz"])
            await db.commit()
            settings["smtp.tz"] = session["tz"]
            await _reload_app_config()
        who = await audience(db)
    from urllib.parse import urlsplit
    webhook_host = urlsplit(settings.get("webhook.url") or "").hostname or ""
    return await render_template("settings/notifications.html", settings=settings,
                                 webhook_host=webhook_host, audience=who,
                                 roles=list(UserRole), zones=zones_by_region())


@settings_bp.post("/notifications/")
@require_role(UserRole.admin)
async def save_notifications():
    from steward.core.formatting import zone_names
    from steward.core.mail import KINDS
    form = await request.form
    extra_tz = form.get("smtp.tz", "")
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            recipients_raw = form.get("smtp.recipients", "")
            recipients = [r.strip() for r in recipients_raw.split(",") if r.strip()]
            await set_setting(db, "smtp.host", form.get("smtp.host", ""))
            await set_setting(db, "smtp.port", int(form.get("smtp.port", 587)))
            await set_setting(db, "smtp.tls", "smtp.tls" in form)
            await set_setting(db, "smtp.username", form.get("smtp.username", ""))
            if form.get("smtp.password"):
                await set_setting(db, "smtp.password", form.get("smtp.password"))
            await set_setting(db, "smtp.recipients", recipients)
            await set_setting(db, "smtp.from", form.get("smtp.from", "").strip())
            if extra_tz in zone_names():
                await set_setting(db, "smtp.tz", extra_tz)
            for kind in KINDS:
                for role in UserRole:
                    key = f"notify.{kind}.{role.value}"
                    await set_setting(db, key, key in form)
            if "webhook.remove" in form:
                await set_setting(db, "webhook.url", "")
            elif form.get("webhook.url", "").strip():
                await set_setting(db, "webhook.url", form.get("webhook.url").strip())
            await set_setting(db, "webhook.template", form.get("webhook.template", ""))
    await _reload_app_config()
    await audit_request("settings.saved", detail={"section": "notifications"})
    return redirect(url_for("settings.notifications"))


def _sample_alert() -> dict:
    """What a test notification carries: a made-up firing alert whose name
    has quotes in it, so a template that cannot take them shows it."""
    from datetime import datetime, timezone
    return {"rule_name": 'Test alert "from" Steward', "state": "TEST", "metric": "cpu_pct",
            "value": 97.5, "operator": ">", "threshold": 90.0, "resource": "example-host",
            "source_module": "host_agent", "note": "sent from Settings → Notifications",
            "timestamp": datetime.now(timezone.utc).isoformat()}


async def _notify_test_result(channel: str, ok: bool, result: str, payload: str | None = None):
    await audit_request("settings.notification.tested", detail={"channel": channel, "sent": ok})
    return await render_template("settings/_notify_test.html", ok=ok, result=result,
                                 payload=payload)


@settings_bp.post("/notifications/test-webhook")
@require_role(UserRole.admin)
async def test_webhook():
    """Render the webhook template (as in the form, unsaved) for a sample
    alert and send it to the URL in the form, or the saved one."""
    from steward.core.notifications import _send_webhook, render_webhook
    form = await request.form
    async with current_app.db_sessionmaker() as db:
        saved = to_webhook_cfg(await get_all_settings(db))
    url = (form.get("webhook.url") or "").strip() or saved["url"]
    template = form.get("webhook.template") or saved["template"]
    alert = _sample_alert()
    try:
        payload = render_webhook(template, alert)
    except ValueError as exc:
        return await _notify_test_result("webhook", False, str(exc))
    if not url:
        return await _notify_test_result(
            "webhook", False, "No webhook URL is saved or entered. This is the body it would get.",
            payload)
    sent = await _send_webhook({"url": url, "template": template}, alert)
    return await _notify_test_result(
        "webhook", sent["sent"], "Delivered." if sent["sent"] else sent["error"], payload)


@settings_bp.post("/notifications/test-email")
@require_role(UserRole.admin)
async def test_email():
    """Send a sample alert email using the SMTP settings in the form (unsaved):
    to you, in your zone, and to the Also send to addresses in the form, in
    the zone chosen for them. A blank password uses the saved one."""
    from steward.core.formatting import zone
    from steward.core.mail import Recipient, alert_email, is_real_address, pick_recipients
    from steward.core import outbox
    from steward.core.notifications import smtp_mode
    from steward.models.users import User
    form = await request.form
    async with current_app.db_sessionmaker() as db:
        saved = to_smtp_cfg(await get_all_settings(db))
        me = await db.get(User, session.get("user_id"))
    try:
        port = int(form.get("smtp.port") or 587)
    except ValueError:
        return await _notify_test_result("email", False, "The port must be a number.")
    extra = [r.strip() for r in form.get("smtp.recipients", "").split(",") if r.strip()]
    cfg = {"host": (form.get("smtp.host") or "").strip(), "port": port,
           "tls": "smtp.tls" in form, "username": form.get("smtp.username", ""),
           "password": form.get("smtp.password") or saved["password"],
           "from": (form.get("smtp.from") or "").strip(), "base_url": saved["base_url"]}
    to = ([Recipient(me.email.strip(), zone(me.time_zone), False, me.username)]
          if me is not None and is_real_address(me.email) else [])
    to += [r for r in pick_recipients([], "alerts", {}, extra, form.get("smtp.tz") or saved["tz"])
           if r.address.lower() not in {t.address.lower() for t in to}]
    if not cfg["host"]:
        return await _notify_test_result("email", False, "Enter an SMTP host.")
    if not to:
        return await _notify_test_result(
            "email", False, "Your account has no email address mail can reach, and no Also send to "
                            "addresses are entered. Set your address on the Account page.")
    alert = _sample_alert()
    # Queued and recorded like any email, tried once with the form's values.
    ok, said = outbox.said(await outbox.send_each(
        current_app, "test", to, lambda r: alert_email(cfg, alert, r), retry=False, cfg=cfg))
    how = {"ssl": "implicit TLS", "starttls": "STARTTLS", "plain": "no TLS"}[smtp_mode(port, cfg["tls"])]
    result = f"{said[:-1]} via {cfg['host']}:{port} ({how})." if ok else f"{cfg['host']}:{port} ({how}): {said}"
    return await _notify_test_result("email", ok, result, alert_email(cfg, alert, to[0])[1])


# ── Ansible Sources ───────────────────────────────────────────────────────────

async def _get_ansible_sources() -> list[dict]:
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    sources = settings.get("ansible.sources", [])
    return sources if isinstance(sources, list) else []


async def _save_ansible_sources(sources: list[dict]) -> None:
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            await set_setting(db, "ansible.sources", seal_source_tokens(sources)[0])
    await _reload_app_config()


async def _ansible_sources_partial(sources: list[dict]):
    return await render_template("settings/_ansible_sources.html", sources=sources)


@settings_bp.get("/ansible/")
@require_role(UserRole.admin)
async def ansible():
    sources = await _get_ansible_sources()
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    return await render_template(
        "settings/ansible.html",
        sources=sources,
        ssh_key_set=bool(settings.get("ansible.ssh_private_key")),
        become_set=bool(settings.get("ansible.become_password")),
        vault_set=bool(settings.get("ansible.vault_password")),
        ssh_public_key=settings.get("ansible.ssh_public_key", ""),
        ssh_user=settings.get("ansible.ssh_user", "steward"),
        host_key_checking=settings.get("ansible.host_key_checking", False),
        max_concurrent_runs=settings.get("ansible.max_concurrent_runs", 3),
    )


@settings_bp.post("/ansible/generate-key")
@require_role(UserRole.admin)
async def ansible_generate_key():
    """Mint a fresh managed ed25519 keypair (private encrypted, public shown).

    This is the reusable Steward identity: provisioning installs the public half
    on every host, and steady-state runs authenticate with the private half.
    Regenerating invalidates access to already-provisioned hosts until they are
    re-provisioned, so the UI confirms before calling this.
    """
    from steward.core.crypto import generate_ssh_keypair

    form = await request.form
    private_pem, public_line = generate_ssh_keypair()
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            await set_setting(db, "ansible.ssh_private_key", private_pem)
            await set_setting(db, "ansible.ssh_public_key", public_line)
    await _reload_app_config()
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "settings.saved", detail={"section": "ansible.managed_key"})
    # Allow the inline trigger on other pages (e.g. Host Agents) to return there.
    nxt = (form.get("next", "") or "").strip()
    if nxt.startswith("/") and not nxt.startswith("//"):
        return redirect(nxt)
    return redirect(url_for("settings.ansible"))


@settings_bp.post("/ansible/credentials")
@require_role(UserRole.admin)
async def ansible_save_credentials():
    """Save global Ansible credentials (masked-update: blank keeps the current value)."""
    form = await request.form
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            for field, key in (
                ("ssh_private_key", "ansible.ssh_private_key"),
                ("become_password", "ansible.become_password"),
                ("vault_password", "ansible.vault_password"),
            ):
                if f"clear_{field}" in form:
                    await set_setting(db, key, "")
                    continue
                raw = form.get(field, "") or ""
                # Preserve PEM whitespace for the key; trim passwords.
                val = raw.strip("\n") if field == "ssh_private_key" else raw.strip()
                if val:
                    await set_setting(db, key, val)
            ssh_user = (form.get("ssh_user", "") or "").strip()
            if ssh_user:
                await set_setting(db, "ansible.ssh_user", ssh_user)
            await set_setting(db, "ansible.host_key_checking", "host_key_checking" in form)
            try:
                mcr = max(1, min(50, int(form.get("max_concurrent_runs", 3))))
            except (TypeError, ValueError):
                mcr = 3
            await set_setting(db, "ansible.max_concurrent_runs", mcr)
    await _reload_app_config()
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "settings.saved", detail={"section": "ansible.credentials"})
    return redirect(url_for("settings.ansible"))


@settings_bp.post("/ansible/sources/add")
@require_role(UserRole.admin)
async def ansible_add_source():
    form = await request.form
    name = form.get("name", "").strip()
    src_type = form.get("type", "local").strip()
    if not name:
        sources = await _get_ansible_sources()
        return await _ansible_sources_partial(sources)
    entry: dict = {"name": name, "type": src_type}
    if src_type == "git":
        entry["url"] = form.get("url", "").strip()
        entry["branch"] = form.get("branch", "main").strip() or "main"
        try:
            entry["pull_interval_seconds"] = int(form.get("pull_interval_seconds", 3600))
        except ValueError:
            entry["pull_interval_seconds"] = 3600
        token = form.get("http_token", "").strip()
        if token:
            entry["http_token"] = token
    else:
        entry["path"] = form.get("path", "").strip()
    sources = await _get_ansible_sources()
    sources.append(entry)
    await _save_ansible_sources(sources)
    return await _ansible_sources_partial(sources)


@settings_bp.post("/ansible/sources/<int:idx>/remove")
@require_role(UserRole.admin)
async def ansible_remove_source(idx: int):
    sources = await _get_ansible_sources()
    if 0 <= idx < len(sources):
        sources.pop(idx)
        await _save_ansible_sources(sources)
    return await _ansible_sources_partial(sources)


@settings_bp.post("/ansible/sources/<int:idx>/save")
@require_role(UserRole.admin)
async def ansible_save_source(idx: int):
    sources = await _get_ansible_sources()
    if not (0 <= idx < len(sources)):
        return await _ansible_sources_partial(sources)
    form = await request.form
    src_type = form.get("type", "local").strip()
    entry: dict = {
        "name": form.get("name", sources[idx].get("name", "")).strip(),
        "type": src_type,
    }
    if src_type == "git":
        entry["url"] = form.get("url", "").strip()
        entry["branch"] = form.get("branch", "main").strip() or "main"
        try:
            entry["pull_interval_seconds"] = int(form.get("pull_interval_seconds", 3600))
        except ValueError:
            entry["pull_interval_seconds"] = 3600
        token = form.get("http_token", "").strip()
        if token:
            entry["http_token"] = token
        elif "http_token" in sources[idx]:
            # Preserve existing token — password fields don't round-trip via browser
            entry["http_token"] = sources[idx]["http_token"]
    else:
        entry["path"] = form.get("path", "").strip()
    sources[idx] = entry
    await _save_ansible_sources(sources)
    return await _ansible_sources_partial(sources)


@settings_bp.post("/ansible/sources/<int:idx>/sync")
@require_role(UserRole.admin)
async def ansible_sync_source(idx: int):
    """Trigger a git pull for a git source and return a status badge."""
    sources = await _get_ansible_sources()
    if not (0 <= idx < len(sources)):
        return ('<span style="color:var(--red);font-size:0.8rem;">Source not found</span>', 404)
    source = sources[idx]
    if source.get("type") != "git":
        return '<span style="color:var(--text-muted);font-size:0.8rem;">Not a git source</span>'
    from steward.ansible.sources import git_pull
    from steward.core.settings import to_ansible_cfg
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    ansible_cfg = to_ansible_cfg(settings)
    from steward.ansible.sources import get_sources
    resolved = next((s for s in get_sources(ansible_cfg) if s["name"] == source["name"]), None)
    if resolved is None:
        return '<span style="color:var(--red);font-size:0.8rem;">Could not resolve source path</span>'
    try:
        await git_pull(resolved)
        return '<span style="color:var(--green);font-size:0.8rem;">Synced</span>'
    except Exception as exc:
        logger.exception("git pull failed for source %r", source["name"])
        return ('<span style="color:var(--red);font-size:0.8rem;">Sync failed: '
                f'{html.escape(str(exc))}</span>')


# ── Auth (OIDC / LDAP) ────────────────────────────────────────────────────────

@settings_bp.get("/auth/")
@require_role(UserRole.admin)
async def auth_settings():
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    return await render_template("settings/auth.html", settings=settings)


@settings_bp.post("/auth/")
@require_role(UserRole.admin)
async def save_auth_settings():
    form = await request.form
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            # OIDC
            await set_setting(db, "oidc.enabled", "oidc.enabled" in form)
            await set_setting(db, "oidc.discovery_url", form.get("oidc.discovery_url", "").strip())
            await set_setting(db, "oidc.client_id", form.get("oidc.client_id", "").strip())
            if form.get("oidc.client_secret"):
                await set_setting(db, "oidc.client_secret", form.get("oidc.client_secret").strip())
            await set_setting(db, "oidc.scopes", form.get("oidc.scopes", "openid profile email").strip())
            await set_setting(db, "oidc.username_claim", form.get("oidc.username_claim", "preferred_username").strip())
            await set_setting(db, "oidc.email_claim", form.get("oidc.email_claim", "email").strip())
            await set_setting(db, "oidc.groups_claim", form.get("oidc.groups_claim", "groups").strip())
            await set_setting(db, "oidc.admin_group", form.get("oidc.admin_group", "").strip())
            await set_setting(db, "oidc.operator_group", form.get("oidc.operator_group", "").strip())
            # LDAP
            await set_setting(db, "ldap.enabled", "ldap.enabled" in form)
            await set_setting(db, "ldap.host", form.get("ldap.host", "").strip())
            await set_setting(db, "ldap.port", int(form.get("ldap.port") or 389))
            await set_setting(db, "ldap.tls", "ldap.tls" in form)
            await set_setting(db, "ldap.ca_cert", form.get("ldap.ca_cert", "").strip())
            await set_setting(db, "ldap.bind_dn", form.get("ldap.bind_dn", "").strip())
            if form.get("ldap.bind_password"):
                await set_setting(db, "ldap.bind_password", form.get("ldap.bind_password").strip())
            await set_setting(db, "ldap.base_dn", form.get("ldap.base_dn", "").strip())
            await set_setting(db, "ldap.user_filter", form.get("ldap.user_filter", "(uid={username})").strip())
            await set_setting(db, "ldap.admin_group_dn", form.get("ldap.admin_group_dn", "").strip())
            await set_setting(db, "ldap.operator_group_dn", form.get("ldap.operator_group_dn", "").strip())
            await set_setting(db, "ldap.attr_username", form.get("ldap.attr_username", "uid").strip())
            await set_setting(db, "ldap.attr_email", form.get("ldap.attr_email", "mail").strip())
    await _reload_app_config()
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "settings.saved", detail={"section": "auth"})
    # Clear OIDC discovery cache so new settings take effect
    from steward.auth.oidc import _discovery_cache
    _discovery_cache.clear()
    return redirect(url_for("settings.auth_settings"))


# ── Reports ───────────────────────────────────────────────────────────────────

async def _reports_page(settings: dict, flash):
    """Settings → Reports, with the schedule and the last send in the reader's
    zone: the hour is kept in the zone of whoever saved it (#5378)."""
    from steward.core.formatting import zone
    from steward.core.reports import schedule_in
    shown_day, shown_hour = schedule_in(
        int(settings.get("reports.schedule_day", 6)), int(settings.get("reports.schedule_hour", 8)),
        zone(settings.get("reports.schedule_tz")), zone(session.get("tz")))
    raw = settings.get("reports.last_sent_at") or ""
    try:
        last_sent = datetime.fromisoformat(raw) if raw else None
    except ValueError:
        last_sent = None
    return await render_template("settings/reports.html", settings=settings, flash=flash,
                                 shown_day=shown_day, shown_hour=shown_hour, last_sent=last_sent)


@settings_bp.get("/system/")
@require_role(UserRole.admin)
async def system():
    """Steward's own health (#5467): its process metrics now and over the day,
    each one an alert rule can watch (source "Steward itself"); then its
    tables' sizes and what the last hourly cleanup removed (#5469); and
    the email not yet sent (#5509), whose failures and slow retries join the
    page's verdict."""
    from steward.core import outbox
    from steward.core.health import last_cleanup, process_health, table_sizes
    from steward.core.status import sparkline_svg
    async with current_app.db_sessionmaker() as db:
        health = await process_health(db)
        # Table sizes and the last cleanup (#5469): Postgres's catalog and
        # what the cleanup kept, never a scan of the tables themselves.
        storage, cleanup = await table_sizes(db), await last_cleanup(db)
        email = await outbox.outgoing(db)
    health["problems"] = email["problems"] + health["problems"]
    for row in health["rows"]:
        # From zero to the row's own peak, with a floor (UI & Design ruling 8).
        row["spark"] = sparkline_svg(row["series"], width=120, height=20,
                                     stroke="currentColor", floor=1.0) if len(row["series"]) > 1 else ""
    return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup,
                                 email=email, kind_labels=outbox.KIND_LABELS)


@settings_bp.get("/reports/")
@require_role(UserRole.admin)
async def reports():
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    return await _reports_page(settings, None)


@settings_bp.post("/reports/")
@require_role(UserRole.admin)
async def save_reports():
    """The day and hour are what the reader picked on their own clock, so the
    schedule is kept in their zone."""
    form = await request.form
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            await set_setting(db, "reports.enabled", "reports.enabled" in form)
            await set_setting(db, "reports.schedule_day",
                              int(form.get("reports.schedule_day", 6)))
            await set_setting(db, "reports.schedule_hour",
                              int(form.get("reports.schedule_hour", 8)))
            await set_setting(db, "reports.schedule_tz", session.get("tz") or "")
    return redirect(url_for("settings.reports"))


@settings_bp.post("/reports/send-now/")
@require_role(UserRole.admin)
async def send_report_now():
    from steward.core.reports import send_report
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    ok, message = await send_report(current_app._get_current_object())
    return await _reports_page(settings, {"ok": ok, "message": message})


# ── Backups ───────────────────────────────────────────────────────────────────

async def _backups_page(flash=None):
    """Settings → Backups (#5673): how the newest backup went, the bucket and
    the daily hour (in the reader's zone), and the runs before."""
    from steward.backup import job
    from steward.core.formatting import zone
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    hour = job.shown_hour(int(settings.get("backup.schedule_hour", 3)),
                          zone(settings.get("backup.schedule_tz")), zone(session.get("tz")))
    return await render_template(
        "settings/backups.html", settings=settings, flash=flash, shown_hour=hour,
        verdict=job.verdict(settings, job.is_running()),
        history=job.history_rows(settings.get("backup.history")),
        has_secret=bool(settings.get("backup.s3_secret_key")))


@settings_bp.get("/backups/")
@require_role(UserRole.admin)
async def backups():
    return await _backups_page()


@settings_bp.post("/backups/")
@require_role(UserRole.admin)
async def save_backups():
    """The hour is what the reader picked on their own clock, so it is kept in
    their zone. A blank secret key keeps the stored one."""
    form = await request.form
    endpoint = form.get("backup.s3_endpoint", "").strip()
    if endpoint and not endpoint.startswith(("http://", "https://")):
        return await _backups_page({"ok": False, "message":
                                    "The endpoint starts with http:// or https://, e.g. http://archives:8081."})
    try:
        hour = int(form.get("backup.schedule_hour", 3))
    except ValueError:
        hour = 3
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            await set_setting(db, "backup.enabled", "backup.enabled" in form)
            await set_setting(db, "backup.s3_endpoint", endpoint)
            await set_setting(db, "backup.s3_bucket", form.get("backup.s3_bucket", "").strip())
            await set_setting(db, "backup.s3_region", form.get("backup.s3_region", "").strip() or "us-east-1")
            await set_setting(db, "backup.s3_access_key", form.get("backup.s3_access_key", "").strip())
            secret = form.get("backup.s3_secret_key", "")
            if secret.strip():
                await set_setting(db, "backup.s3_secret_key", secret.strip())
            elif "backup.clear_secret" in form:
                await set_setting(db, "backup.s3_secret_key", "")
            await set_setting(db, "backup.s3_verify_tls", "backup.s3_verify_tls" in form)
            await set_setting(db, "backup.schedule_hour", min(max(hour, 0), 23))
            await set_setting(db, "backup.schedule_tz", session.get("tz") or "")
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "settings.saved", detail={"section": "backups"})
    return redirect(url_for("settings.backups"))


@settings_bp.post("/backups/test/")
@require_role(UserRole.admin)
async def test_backups():
    """Reach the saved bucket with the saved keys (the operator asked, so the
    target is contacted)."""
    from steward.backup import job
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    target = job.target_from(settings)
    if target is None:
        return await _backups_page({"ok": False, "message":
                                    "Save the endpoint, bucket and both keys first."})
    ok, message = await job.test_target(target)
    return await _backups_page({"ok": ok, "message": message})


@settings_bp.post("/backups/now/")
@require_role(UserRole.admin)
async def backup_now():
    from steward.backup import job
    started = job.start_backup(current_app._get_current_object(),
                               by=session.get("username", "") or "an admin",
                               user_id=session.get("user_id"))
    message = ("Making a backup now; it shows under Recent backups when it is sent."
               if started else "A backup is already being made.")
    return await _backups_page({"ok": started, "message": message})


def _build_plugin_cfg_from_form(plugin: dict, form) -> dict:
    """Extract config values for one plugin from a form submission."""
    name = plugin["_dir"]
    enabled = f"plugin.{name}.enabled" in form
    plugin_cfg: dict = {"enabled": enabled}
    for cfg_key in plugin.get("config", {}).keys():
        field_name = f"plugin.{name}.{cfg_key}"
        default_val = plugin["config"][cfg_key]
        if isinstance(default_val, dict):
            sub_dict = {}
            for sub_key, sub_default in default_val.items():
                sub_field = f"plugin.{name}.{cfg_key}.{sub_key}"
                if isinstance(sub_default, bool):
                    sub_dict[sub_key] = sub_field in form
                elif sub_field in form:
                    raw_sub = form[sub_field]
                    if isinstance(sub_default, int):
                        try:
                            sub_dict[sub_key] = int(raw_sub)
                        except ValueError:
                            sub_dict[sub_key] = sub_default
                    else:
                        sub_dict[sub_key] = raw_sub
                else:
                    sub_dict[sub_key] = sub_default
            plugin_cfg[cfg_key] = sub_dict
        elif field_name in form:
            raw_val = form[field_name]
            if isinstance(default_val, bool):
                plugin_cfg[cfg_key] = raw_val.lower() in ("1", "true", "yes", "on")
            elif isinstance(default_val, int):
                try:
                    plugin_cfg[cfg_key] = int(raw_val)
                except ValueError:
                    plugin_cfg[cfg_key] = default_val
            else:
                plugin_cfg[cfg_key] = raw_val
    return plugin_cfg


# ── Plugins list ──────────────────────────────────────────────────────────────

# Bundled plugins that are really built-in *host/monitoring capabilities* (facets
# of a host), not discrete vendor integrations. Presentation-only split — they
# still load through the normal plugin mechanism. A plugin.yaml may set
# kind: capability|integration to override.
CAPABILITY_PLUGINS = {"host_agent", "snmp", "docker"}


@settings_bp.get("/plugins/")
@require_role(UserRole.admin)
async def plugins():
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
        stored_plugin_names = await get_stored_plugin_names(db)
    discovered = _discover_plugins()
    _merge_plugin_config(discovered, to_plugins_cfg(settings))
    for p in discovered:
        p["_kind"] = p.get("kind") or (
            "capability" if p["_dir"] in CAPABILITY_PLUGINS else "integration")
    # Config left behind by plugins that are no longer installed. Surfaced for an
    # explicit operator decision rather than cleaned up automatically: an external
    # plugin can be missing because /data/plugins isn't mounted, and silently
    # dropping its row would destroy stored credentials.
    #
    # Load failures are folded in so every plugin counted by the admin banner has
    # a row here. A plugin that failed BECAUSE it has no directory is not in
    # `discovered`, so without this it would be counted in the banner and shown
    # nowhere on the page that banner links to.
    from steward.core.plugin_manager import get_plugin_failures
    orphans = find_orphaned_plugins(
        stored_plugin_names,
        [p["_dir"] for p in discovered],
        get_plugin_failures(),
    )
    return await render_template(
        "settings/plugins.html",
        capabilities=[p for p in discovered if p["_kind"] == "capability"],
        integrations=[p for p in discovered if p["_kind"] != "capability"],
        discovered_plugins=discovered,
        orphaned_plugins=orphans,
        settings=settings,
    )


@settings_bp.post("/plugins/orphans/<name>/remove/")
@require_role(UserRole.admin)
async def plugin_orphan_remove(name: str):
    """Delete the stored settings row for a plugin that is no longer installed.

    Refuses if the plugin IS installed — that would silently wipe a live
    plugin's config, and disabling it is what the operator wants there instead.
    """
    installed = {p["_dir"] for p in _discover_plugins()}
    if name in installed:
        return redirect(url_for("settings.plugins"))

    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            removed = await delete_setting(db, f"plugin.{name}")

    if removed:
        await _reload_app_config()
        await log_audit(
            current_app, session.get("user_id"), session.get("username", ""),
            "plugin.settings_removed", entity_type="plugin", entity_id=name)
    return redirect(url_for("settings.plugins"))


# ── Per-plugin detail (settings) ──────────────────────────────────────────────

@settings_bp.get("/plugins/<name>/")
@require_role(UserRole.admin)
async def plugin_detail(name: str):
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    discovered = _discover_plugins()
    plugin = next((p for p in discovered if p["_dir"] == name), None)
    if plugin is None:
        return redirect(url_for("settings.plugins"))
    _merge_plugin_config([plugin], to_plugins_cfg(settings))
    from steward.core.plugin_manager import _LOADED_PLUGINS, get_plugin_failures
    return await render_template(
        "settings/plugin_detail.html",
        plugin=plugin,
        is_loaded=name in _LOADED_PLUGINS,
        fail_reason=get_plugin_failures().get(name),
    )


@settings_bp.post("/plugins/<name>/")
@require_role(UserRole.admin)
async def save_plugin_detail(name: str):
    form = await request.form
    discovered = _discover_plugins()
    plugin = next((p for p in discovered if p["_dir"] == name), None)
    if plugin is None:
        return redirect(url_for("settings.plugins"))

    async with current_app.db_sessionmaker() as db:
        old_settings = await get_all_settings(db)
    old_enabled = to_plugins_cfg(old_settings).get(name, {}).get("enabled", False)

    plugin_cfg = _build_plugin_cfg_from_form(plugin, form)
    # A secret is never sent back to the browser, so its box arrives blank
    # unless someone typed a new one: blank keeps the saved value.
    old_cfg = to_plugins_cfg(old_settings).get(name, {})
    for field, value in list(plugin_cfg.items()):
        if is_plugin_secret(field) and value == "" and old_cfg.get(field):
            plugin_cfg[field] = old_cfg[field]

    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            await set_setting(db, f"plugin.{name}", plugin_cfg)

    await _reload_app_config()

    new_enabled = plugin_cfg.get("enabled", False)
    if old_enabled != new_enabled:
        action = "plugin.enabled" if new_enabled else "plugin.disabled"
        await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                        action, entity_type="plugin", entity_id=name)
    if new_enabled and not old_enabled:
        from steward.core.plugin_manager import hot_reload_plugin
        hot_reload_plugin(current_app._get_current_object(), name)

    return redirect(url_for("settings.plugin_detail", name=name))


# ── App restart ───────────────────────────────────────────────────────────────

@settings_bp.post("/plugins/restart/")
@require_role(UserRole.admin)
async def restart_app_route():
    """Trigger an in-app restart (replaces the process via os.execv)."""
    import asyncio
    from steward.core.plugin_manager import restart_app

    # Schedule the restart slightly after we return the response
    async def _delayed_restart():
        await asyncio.sleep(0.5)
        restart_app()

    asyncio.create_task(_delayed_restart())
    return await render_template("settings/_restart_pending.html")