FabledSteward/steward/settings/routes.py

# steward/settings/routes.py
from __future__ import annotations
import html
import logging
from datetime import datetime
from pathlib import Path
from quart import Blueprint, current_app, render_template, request, redirect, url_for, session

from steward.auth.middleware import require_role
from steward.core.audit import audit_request, log_audit
from steward.core.logging_setup import LEVELS as LOG_LEVELS, set_log_level
from steward.models.users import UserRole
from steward.core.settings import (
    get_all_settings, set_setting, delete_setting,
    get_stored_plugin_names, find_orphaned_plugins,
    to_smtp_cfg, to_webhook_cfg, to_ansible_cfg, to_plugins_cfg,
    to_oidc_cfg, to_ldap_cfg, to_thresholds_cfg, to_access_cfg, DEFAULTS, seal_source_tokens,
    is_plugin_secret, NORM_FLOORS, ALWAYS_ON_PLUGINS,
)

settings_bp = Blueprint("settings", __name__, url_prefix="/settings")
logger = logging.getLogger(__name__)


def _discover_plugins() -> list[dict]:
    """Scan every plugin root for plugin.yaml files.

    Roots are scanned in order (bundled first), and a plugin name found in an
    earlier root shadows the same name in a later one — matching load_plugins.
    """
    import yaml
    plugin_dirs = [
        Path(d).resolve()
        for d in current_app.config.get("PLUGIN_DIRS", ["plugins"])
    ]
    plugins: list[dict] = []
    seen: set[str] = set()
    for plugin_dir in plugin_dirs:
        if not plugin_dir.exists():
            continue
        for entry in sorted(plugin_dir.iterdir()):
            yaml_path = entry / "plugin.yaml"
            if entry.is_dir() and yaml_path.exists() and entry.name not in seen:
                try:
                    with yaml_path.open() as f:
                        meta = yaml.safe_load(f) or {}
                    meta["_dir"] = entry.name
                    plugins.append(meta)
                    seen.add(entry.name)
                except Exception:
                    logger.warning("Could not read %s", yaml_path)
    return plugins


def _merge_plugin_config(discovered: list[dict], plugins_cfg: dict) -> None:
    """Merge stored settings into discovered plugin dicts in-place."""
    for plugin in discovered:
        name = plugin["_dir"]
        stored = plugins_cfg.get(name, {})
        plugin["_enabled"] = stored.get("enabled", False)
        plugin["_always_on"] = name in ALWAYS_ON_PLUGINS
        defaults = plugin.get("config", {})
        plugin["_config"] = {**defaults, **{k: v for k, v in stored.items() if k != "enabled"}}


async def _reload_app_config() -> None:
    """Reload app.config from DB so settings take effect without restart."""
    async with current_app.db_sessionmaker() as db:
        fresh = await get_all_settings(db)
    current_app.config.update(to_access_cfg(fresh))
    current_app.config.update(
        DATA_RETENTION_DAYS=fresh.get("data.retention_days", 90),
        MONITORS_POLL_INTERVAL=fresh.get("monitors.poll_interval_seconds", 60),
        PUBLIC_BASE_URL=fresh.get("general.public_base_url", ""),
        SMTP=to_smtp_cfg(fresh),
        WEBHOOK=to_webhook_cfg(fresh),
        ANSIBLE=to_ansible_cfg(fresh),
        PLUGINS=to_plugins_cfg(fresh),
        OIDC=to_oidc_cfg(fresh),
        LDAP=to_ldap_cfg(fresh),
        THRESHOLDS=to_thresholds_cfg(fresh),
    )


# ── Redirect root to general tab ──────────────────────────────────────────────

@settings_bp.get("/")
@require_role(UserRole.admin)
async def index():
    return redirect(url_for("settings.general"))


# ── General ───────────────────────────────────────────────────────────────────

@settings_bp.get("/general/")
@require_role(UserRole.admin)
async def general():
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    return await render_template("settings/general.html", settings=settings,
                                 log_levels=LOG_LEVELS)


@settings_bp.post("/general/")
@require_role(UserRole.admin)
async def save_general():
    form = await request.form
    whole: dict[str, int] = {}
    for key in ("session.lifetime_hours", "general.trusted_proxy_hops",
                "login.max_failures", "login.lockout_minutes", "audit.retention_days"):
        try:
            whole[key] = int(form.get(key, DEFAULTS[key]))
        except (TypeError, ValueError):
            return f"{key} must be a whole number", 400
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            await set_setting(db, "general.public_base_url",
                              (form.get("general.public_base_url", "") or "").strip())
            for key, value in whole.items():
                await set_setting(db, key, value)
            await set_setting(db, "data.retention_days",
                              int(form.get("data.retention_days", 90)))
            await set_setting(db, "monitors.poll_interval_seconds",
                              int(form.get("monitors.poll_interval_seconds", 60)))
            await set_setting(db, "log.level", set_log_level(form.get("log.level")))
    await _reload_app_config()
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "settings.saved", detail={"section": "general"})
    return redirect(url_for("settings.general"))


# ── Monitoring thresholds ─────────────────────────────────────────────────────

# (form field, setting key, is_float) for every tunable cutoff.
_THRESHOLD_FIELDS = [
    ("cpu_warn", "thresholds.cpu_warn", False), ("cpu_crit", "thresholds.cpu_crit", False),
    ("mem_warn", "thresholds.mem_warn", False), ("mem_crit", "thresholds.mem_crit", False),
    ("disk_warn", "thresholds.disk_warn", False), ("disk_crit", "thresholds.disk_crit", False),
    ("load_warn", "thresholds.load_warn", False), ("load_crit", "thresholds.load_crit", False),
    ("temp_warn", "thresholds.temp_warn", False), ("temp_crit", "thresholds.temp_crit", False),
    ("uptime_warn", "thresholds.uptime_warn", True), ("uptime_crit", "thresholds.uptime_crit", True),
    ("latency_warn", "ping.threshold.good_ms", False), ("latency_crit", "ping.threshold.warn_ms", False),
    ("tls_warn", "thresholds.tls_warn_days", False), ("tls_crit", "thresholds.tls_crit_days", False),
    ("clock_warn", "thresholds.clock_warn_s", False), ("clock_crit", "thresholds.clock_crit_s", False),
    ("crash_loop_deaths", "thresholds.crash_loop_deaths", False),
    ("error_5xx_pct", "traefik.threshold.error_5xx_pct", True),
    ("error_4xx_pct", "traefik.threshold.error_4xx_pct", True),
    ("min_requests", "traefik.threshold.min_requests", False),
    ("min_5xx", "traefik.threshold.min_5xx", False),
    # Each normal's floor (#5742), named by its own setting key.
    *[(f"norms.floor.{name}", f"norms.floor.{name}", True) for name in NORM_FLOORS],
]

# Docker time-series retention windows (days). Read fresh by the cleanup task,
# so a save takes effect on the next hourly run — no app.config wiring needed.
_RETENTION_FIELDS = [
    ("docker_events_days", "docker.retention.events_days"),
    ("docker_logs_retention_days", "docker.logs.retention_days"),
    ("metrics_raw_days", "metrics.retention.raw_days"),
    ("metrics_rollup_days", "metrics.retention.rollup_days"),
    ("anomalies_review_days", "anomalies.review_days"),
]


@settings_bp.get("/thresholds/")
@require_role(UserRole.admin)
async def thresholds():
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    from steward.core.plugin_manager import _LOADED_PLUGINS
    return await render_template("settings/thresholds.html", settings=settings,
                                 traefik_on="traefik" in _LOADED_PLUGINS,
                                 plugins_on=set(_LOADED_PLUGINS))


@settings_bp.post("/thresholds/")
@require_role(UserRole.admin)
async def save_thresholds():
    form = await request.form
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            for field, key, is_float in _THRESHOLD_FIELDS:
                raw = form.get(field, "")
                if raw == "":
                    continue
                try:
                    val = float(raw) if is_float else int(raw)
                except (TypeError, ValueError):
                    continue
                await set_setting(db, key, val)
            for field, key in _RETENTION_FIELDS:
                raw = form.get(field, "")
                if raw == "":
                    continue
                try:
                    val = max(1, int(raw))  # at least a day — 0 would prune everything
                except (TypeError, ValueError):
                    continue
                await set_setting(db, key, val)
            # Container-log controls (m79). Checkbox: present ⇒ on (this is a
            # full-page form, so absence is a genuine "off"). Exclude: comma-split
            # names. Size: entered in MB, stored as bytes.
            await set_setting(db, "docker.logs.enabled", "docker_logs_enabled" in form)
            exclude = [n.strip() for n in form.get("docker_logs_exclude", "").split(",")
                       if n.strip()]
            await set_setting(db, "docker.logs.exclude", exclude)
            max_mb = form.get("docker_logs_max_mb", "")
            if max_mb != "":
                try:
                    await set_setting(db, "docker.logs.max_bytes_per_container",
                                      max(1, int(max_mb)) * 1_000_000)
                except (TypeError, ValueError):
                    pass
    await _reload_app_config()
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "settings.saved", detail={"section": "thresholds"})
    return redirect(url_for("settings.thresholds"))


# ── Notifications (SMTP + Webhook) ────────────────────────────────────────────

@settings_bp.get("/notifications/")
@require_role(UserRole.admin)
async def notifications():
    from steward.core.formatting import zones_by_region
    from steward.core.mail import audience
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
        if not settings["smtp.tz"] and session.get("tz"):
            # "Also send to" reads times in the first admin's zone until one is chosen.
            # Reading the settings already began this session's transaction.
            await set_setting(db, "smtp.tz", session["tz"])
            await db.commit()
            settings["smtp.tz"] = session["tz"]
            await _reload_app_config()
        who = await audience(db)
    from urllib.parse import urlsplit
    webhook_host = urlsplit(settings.get("webhook.url") or "").hostname or ""
    return await render_template("settings/notifications.html", settings=settings,
                                 webhook_host=webhook_host, audience=who,
                                 roles=list(UserRole), zones=zones_by_region())


@settings_bp.post("/notifications/")
@require_role(UserRole.admin)
async def save_notifications():
    from steward.core.formatting import zone_names
    from steward.core.mail import KINDS
    form = await request.form
    extra_tz = form.get("smtp.tz", "")
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            recipients_raw = form.get("smtp.recipients", "")
            recipients = [r.strip() for r in recipients_raw.split(",") if r.strip()]
            await set_setting(db, "smtp.host", form.get("smtp.host", ""))
            await set_setting(db, "smtp.port", int(form.get("smtp.port", 587)))
            await set_setting(db, "smtp.tls", "smtp.tls" in form)
            await set_setting(db, "smtp.username", form.get("smtp.username", ""))
            if form.get("smtp.password"):
                await set_setting(db, "smtp.password", form.get("smtp.password"))
            await set_setting(db, "smtp.recipients", recipients)
            await set_setting(db, "smtp.from", form.get("smtp.from", "").strip())
            if extra_tz in zone_names():
                await set_setting(db, "smtp.tz", extra_tz)
            for kind in KINDS:
                for role in UserRole:
                    key = f"notify.{kind}.{role.value}"
                    await set_setting(db, key, key in form)
            if "webhook.remove" in form:
                await set_setting(db, "webhook.url", "")
            elif form.get("webhook.url", "").strip():
                await set_setting(db, "webhook.url", form.get("webhook.url").strip())
            await set_setting(db, "webhook.template", form.get("webhook.template", ""))
    await _reload_app_config()
    await audit_request("settings.saved", detail={"section": "notifications"})
    return redirect(url_for("settings.notifications"))


def _sample_alert() -> dict:
    """What a test notification carries: a made-up firing alert whose name
    has quotes in it, so a template that cannot take them shows it."""
    from datetime import datetime, timezone
    return {"rule_name": 'Test alert "from" Steward', "state": "TEST", "metric": "cpu_pct",
            "value": 97.5, "operator": ">", "threshold": 90.0, "resource": "example-host",
            "source_module": "host_agent", "note": "sent from Settings → Notifications",
            "timestamp": datetime.now(timezone.utc).isoformat()}


async def _notify_test_result(channel: str, ok: bool, result: str, payload: str | None = None):
    await audit_request("settings.notification.tested", detail={"channel": channel, "sent": ok})
    return await render_template("settings/_notify_test.html", ok=ok, result=result,
                                 payload=payload)


@settings_bp.post("/notifications/test-webhook")
@require_role(UserRole.admin)
async def test_webhook():
    """Render the webhook template (as in the form, unsaved) for a sample
    alert and send it to the URL in the form, or the saved one."""
    from steward.core.notifications import _send_webhook, render_webhook
    form = await request.form
    async with current_app.db_sessionmaker() as db:
        saved = to_webhook_cfg(await get_all_settings(db))
    url = (form.get("webhook.url") or "").strip() or saved["url"]
    template = form.get("webhook.template") or saved["template"]
    alert = _sample_alert()
    try:
        payload = render_webhook(template, alert)
    except ValueError as exc:
        return await _notify_test_result("webhook", False, str(exc))
    if not url:
        return await _notify_test_result(
            "webhook", False, "No webhook URL is saved or entered. This is the body it would get.",
            payload)
    sent = await _send_webhook({"url": url, "template": template}, alert)
    return await _notify_test_result(
        "webhook", sent["sent"], "Delivered." if sent["sent"] else sent["error"], payload)


@settings_bp.post("/notifications/test-email")
@require_role(UserRole.admin)
async def test_email():
    """Send a sample alert email using the SMTP settings in the form (unsaved):
    to you, in your zone, and to the Also send to addresses in the form, in
    the zone chosen for them. A blank password uses the saved one."""
    from steward.core.formatting import zone
    from steward.core.mail import Recipient, alert_email, is_real_address, pick_recipients
    from steward.core import outbox
    from steward.core.notifications import smtp_mode
    from steward.models.users import User
    form = await request.form
    async with current_app.db_sessionmaker() as db:
        saved = to_smtp_cfg(await get_all_settings(db))
        me = await db.get(User, session.get("user_id"))
    try:
        port = int(form.get("smtp.port") or 587)
    except ValueError:
        return await _notify_test_result("email", False, "The port must be a number.")
    extra = [r.strip() for r in form.get("smtp.recipients", "").split(",") if r.strip()]
    cfg = {"host": (form.get("smtp.host") or "").strip(), "port": port,
           "tls": "smtp.tls" in form, "username": form.get("smtp.username", ""),
           "password": form.get("smtp.password") or saved["password"],
           "from": (form.get("smtp.from") or "").strip(), "base_url": saved["base_url"]}
    to = ([Recipient(me.email.strip(), zone(me.time_zone), False, me.username)]
          if me is not None and is_real_address(me.email) else [])
    to += [r for r in pick_recipients([], "alerts", {}, extra, form.get("smtp.tz") or saved["tz"])
           if r.address.lower() not in {t.address.lower() for t in to}]
    if not cfg["host"]:
        return await _notify_test_result("email", False, "Enter an SMTP host.")
    if not to:
        return await _notify_test_result(
            "email", False, "Your account has no email address mail can reach, and no Also send to "
                            "addresses are entered. Set your address on the Account page.")
    alert = _sample_alert()
    # Queued and recorded like any email, tried once with the form's values.
    ok, said = outbox.said(await outbox.send_each(
        current_app, "test", to, lambda r: alert_email(cfg, alert, r), retry=False, cfg=cfg))
    how = {"ssl": "implicit TLS", "starttls": "STARTTLS", "plain": "no TLS"}[smtp_mode(port, cfg["tls"])]
    result = f"{said[:-1]} via {cfg['host']}:{port} ({how})." if ok else f"{cfg['host']}:{port} ({how}): {said}"
    return await _notify_test_result("email", ok, result, alert_email(cfg, alert, to[0])[1])


# ── Ansible Sources ───────────────────────────────────────────────────────────

async def _get_ansible_sources() -> list[dict]:
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    sources = settings.get("ansible.sources", [])
    return sources if isinstance(sources, list) else []


async def _save_ansible_sources(sources: list[dict]) -> None:
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            await set_setting(db, "ansible.sources", seal_source_tokens(sources)[0])
    await _reload_app_config()


async def _ansible_sources_partial(sources: list[dict]):
    return await render_template("settings/_ansible_sources.html", sources=sources)


@settings_bp.get("/ansible/")
@require_role(UserRole.admin)
async def ansible():
    sources = await _get_ansible_sources()
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    return await render_template(
        "settings/ansible.html",
        sources=sources,
        ssh_key_set=bool(settings.get("ansible.ssh_private_key")),
        become_set=bool(settings.get("ansible.become_password")),
        vault_set=bool(settings.get("ansible.vault_password")),
        ssh_public_key=settings.get("ansible.ssh_public_key", ""),
        ssh_user=settings.get("ansible.ssh_user", "steward"),
        host_key_checking=settings.get("ansible.host_key_checking", False),
        max_concurrent_runs=settings.get("ansible.max_concurrent_runs", 3),
    )


@settings_bp.post("/ansible/generate-key")
@require_role(UserRole.admin)
async def ansible_generate_key():
    """Mint a fresh managed ed25519 keypair (private encrypted, public shown).

    This is the reusable Steward identity: provisioning installs the public half
    on every host, and steady-state runs authenticate with the private half.
    Regenerating invalidates access to already-provisioned hosts until they are
    re-provisioned, so the UI confirms before calling this.
    """
    from steward.core.crypto import generate_ssh_keypair

    form = await request.form
    private_pem, public_line = generate_ssh_keypair()
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            await set_setting(db, "ansible.ssh_private_key", private_pem)
            await set_setting(db, "ansible.ssh_public_key", public_line)
    await _reload_app_config()
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "settings.saved", detail={"section": "ansible.managed_key"})
    # Allow the inline trigger on other pages (e.g. Host Agents) to return there.
    nxt = (form.get("next", "") or "").strip()
    if nxt.startswith("/") and not nxt.startswith("//"):
        return redirect(nxt)
    return redirect(url_for("settings.ansible"))


@settings_bp.post("/ansible/credentials")
@require_role(UserRole.admin)
async def ansible_save_credentials():
    """Save global Ansible credentials (masked-update: blank keeps the current value)."""
    form = await request.form
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            for field, key in (
                ("ssh_private_key", "ansible.ssh_private_key"),
                ("become_password", "ansible.become_password"),
                ("vault_password", "ansible.vault_password"),
            ):
                if f"clear_{field}" in form:
                    await set_setting(db, key, "")
                    continue
                raw = form.get(field, "") or ""
                # Preserve PEM whitespace for the key; trim passwords.
                val = raw.strip("\n") if field == "ssh_private_key" else raw.strip()
                if val:
                    await set_setting(db, key, val)
            ssh_user = (form.get("ssh_user", "") or "").strip()
            if ssh_user:
                await set_setting(db, "ansible.ssh_user", ssh_user)
            await set_setting(db, "ansible.host_key_checking", "host_key_checking" in form)
            try:
                mcr = max(1, min(50, int(form.get("max_concurrent_runs", 3))))
            except (TypeError, ValueError):
                mcr = 3
            await set_setting(db, "ansible.max_concurrent_runs", mcr)
    await _reload_app_config()
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "settings.saved", detail={"section": "ansible.credentials"})
    return redirect(url_for("settings.ansible"))


@settings_bp.post("/ansible/sources/add")
@require_role(UserRole.admin)
async def ansible_add_source():
    form = await request.form
    name = form.get("name", "").strip()
    src_type = form.get("type", "local").strip()
    if not name:
        sources = await _get_ansible_sources()
        return await _ansible_sources_partial(sources)
    entry: dict = {"name": name, "type": src_type}
    if src_type == "git":
        entry["url"] = form.get("url", "").strip()
        entry["branch"] = form.get("branch", "main").strip() or "main"
        try:
            entry["pull_interval_seconds"] = int(form.get("pull_interval_seconds", 3600))
        except ValueError:
            entry["pull_interval_seconds"] = 3600
        token = form.get("http_token", "").strip()
        if token:
            entry["http_token"] = token
    else:
        entry["path"] = form.get("path", "").strip()
    sources = await _get_ansible_sources()
    sources.append(entry)
    await _save_ansible_sources(sources)
    return await _ansible_sources_partial(sources)


@settings_bp.post("/ansible/sources/<int:idx>/remove")
@require_role(UserRole.admin)
async def ansible_remove_source(idx: int):
    sources = await _get_ansible_sources()
    if 0 <= idx < len(sources):
        sources.pop(idx)
        await _save_ansible_sources(sources)
    return await _ansible_sources_partial(sources)


@settings_bp.post("/ansible/sources/<int:idx>/save")
@require_role(UserRole.admin)
async def ansible_save_source(idx: int):
    sources = await _get_ansible_sources()
    if not (0 <= idx < len(sources)):
        return await _ansible_sources_partial(sources)
    form = await request.form
    src_type = form.get("type", "local").strip()
    entry: dict = {
        "name": form.get("name", sources[idx].get("name", "")).strip(),
        "type": src_type,
    }
    if src_type == "git":
        entry["url"] = form.get("url", "").strip()
        entry["branch"] = form.get("branch", "main").strip() or "main"
        try:
            entry["pull_interval_seconds"] = int(form.get("pull_interval_seconds", 3600))
        except ValueError:
            entry["pull_interval_seconds"] = 3600
        token = form.get("http_token", "").strip()
        if token:
            entry["http_token"] = token
        elif "http_token" in sources[idx]:
            # Preserve existing token — password fields don't round-trip via browser
            entry["http_token"] = sources[idx]["http_token"]
    else:
        entry["path"] = form.get("path", "").strip()
    sources[idx] = entry
    await _save_ansible_sources(sources)
    return await _ansible_sources_partial(sources)


@settings_bp.post("/ansible/sources/<int:idx>/sync")
@require_role(UserRole.admin)
async def ansible_sync_source(idx: int):
    """Trigger a git pull for a git source and return a status badge."""
    sources = await _get_ansible_sources()
    if not (0 <= idx < len(sources)):
        return ('<span style="color:var(--red);font-size:0.8rem;">Source not found</span>', 404)
    source = sources[idx]
    if source.get("type") != "git":
        return '<span style="color:var(--text-muted);font-size:0.8rem;">Not a git source</span>'
    from steward.ansible.sources import git_pull
    from steward.core.settings import to_ansible_cfg
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    ansible_cfg = to_ansible_cfg(settings)
    from steward.ansible.sources import get_sources
    resolved = next((s for s in get_sources(ansible_cfg) if s["name"] == source["name"]), None)
    if resolved is None:
        return '<span style="color:var(--red);font-size:0.8rem;">Could not resolve source path</span>'
    try:
        await git_pull(resolved)
        return '<span style="color:var(--green);font-size:0.8rem;">Synced</span>'
    except Exception as exc:
        logger.exception("git pull failed for source %r", source["name"])
        return ('<span style="color:var(--red);font-size:0.8rem;">Sync failed: '
                f'{html.escape(str(exc))}</span>')


# ── Auth (OIDC / LDAP) ────────────────────────────────────────────────────────

@settings_bp.get("/auth/")
@require_role(UserRole.admin)
async def auth_settings():
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    return await render_template("settings/auth.html", settings=settings)


@settings_bp.post("/auth/")
@require_role(UserRole.admin)
async def save_auth_settings():
    form = await request.form
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            # OIDC
            await set_setting(db, "oidc.enabled", "oidc.enabled" in form)
            await set_setting(db, "oidc.discovery_url", form.get("oidc.discovery_url", "").strip())
            await set_setting(db, "oidc.client_id", form.get("oidc.client_id", "").strip())
            if form.get("oidc.client_secret"):
                await set_setting(db, "oidc.client_secret", form.get("oidc.client_secret").strip())
            await set_setting(db, "oidc.scopes", form.get("oidc.scopes", "openid profile email").strip())
            await set_setting(db, "oidc.username_claim", form.get("oidc.username_claim", "preferred_username").strip())
            await set_setting(db, "oidc.email_claim", form.get("oidc.email_claim", "email").strip())
            await set_setting(db, "oidc.groups_claim", form.get("oidc.groups_claim", "groups").strip())
            await set_setting(db, "oidc.admin_group", form.get("oidc.admin_group", "").strip())
            await set_setting(db, "oidc.operator_group", form.get("oidc.operator_group", "").strip())
            # LDAP
            await set_setting(db, "ldap.enabled", "ldap.enabled" in form)
            await set_setting(db, "ldap.host", form.get("ldap.host", "").strip())
            await set_setting(db, "ldap.port", int(form.get("ldap.port") or 389))
            await set_setting(db, "ldap.tls", "ldap.tls" in form)
            await set_setting(db, "ldap.ca_cert", form.get("ldap.ca_cert", "").strip())
            await set_setting(db, "ldap.bind_dn", form.get("ldap.bind_dn", "").strip())
            if form.get("ldap.bind_password"):
                await set_setting(db, "ldap.bind_password", form.get("ldap.bind_password").strip())
            await set_setting(db, "ldap.base_dn", form.get("ldap.base_dn", "").strip())
            await set_setting(db, "ldap.user_filter", form.get("ldap.user_filter", "(uid={username})").strip())
            await set_setting(db, "ldap.admin_group_dn", form.get("ldap.admin_group_dn", "").strip())
            await set_setting(db, "ldap.operator_group_dn", form.get("ldap.operator_group_dn", "").strip())
            await set_setting(db, "ldap.attr_username", form.get("ldap.attr_username", "uid").strip())
            await set_setting(db, "ldap.attr_email", form.get("ldap.attr_email", "mail").strip())
    await _reload_app_config()
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "settings.saved", detail={"section": "auth"})
    # Clear OIDC discovery cache so new settings take effect
    from steward.auth.oidc import _discovery_cache
    _discovery_cache.clear()
    return redirect(url_for("settings.auth_settings"))


# ── Reports ───────────────────────────────────────────────────────────────────

async def _reports_page(settings: dict, flash):
    """Settings → Reports, with the schedule and the last send in the reader's
    zone: the hour is kept in the zone of whoever saved it (#5378)."""
    from steward.core.formatting import zone
    from steward.core.reports import schedule_in
    shown_day, shown_hour = schedule_in(
        int(settings.get("reports.schedule_day", 6)), int(settings.get("reports.schedule_hour", 8)),
        zone(settings.get("reports.schedule_tz")), zone(session.get("tz")))
    raw = settings.get("reports.last_sent_at") or ""
    try:
        last_sent = datetime.fromisoformat(raw) if raw else None
    except ValueError:
        last_sent = None
    return await render_template("settings/reports.html", settings=settings, flash=flash,
                                 shown_day=shown_day, shown_hour=shown_hour, last_sent=last_sent)


@settings_bp.get("/system/")
@require_role(UserRole.admin)
async def system():
    """Steward's own health (#5467): its process metrics now and over the day,
    each one an alert rule can watch (source "Steward itself"); then its
    tables' sizes and what the last hourly cleanup removed (#5469); and
    the email not yet sent (#5509), whose failures and slow retries join the
    page's verdict."""
    from steward.core import outbox
    from steward.core.health import last_cleanup, process_health, table_sizes
    from steward.core.status import sparkline_svg
    async with current_app.db_sessionmaker() as db:
        health = await process_health(db)
        # Table sizes and the last cleanup (#5469): Postgres's catalog and
        # what the cleanup kept, never a scan of the tables themselves.
        storage, cleanup = await table_sizes(db), await last_cleanup(db)
        email = await outbox.outgoing(db)
    health["problems"] = email["problems"] + health["problems"]
    for row in health["rows"]:
        # From zero to the row's own peak, with a floor (UI & Design ruling 8).
        row["spark"] = sparkline_svg(row["series"], width=120, height=20,
                                     stroke="currentColor", floor=1.0) if len(row["series"]) > 1 else ""
    return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup,
                                 email=email, kind_labels=outbox.KIND_LABELS)


@settings_bp.get("/reports/")
@require_role(UserRole.admin)
async def reports():
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    return await _reports_page(settings, None)


@settings_bp.post("/reports/")
@require_role(UserRole.admin)
async def save_reports():
    """The day and hour are what the reader picked on their own clock, so the
    schedule is kept in their zone."""
    form = await request.form
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            await set_setting(db, "reports.enabled", "reports.enabled" in form)
            await set_setting(db, "reports.schedule_day",
                              int(form.get("reports.schedule_day", 6)))
            await set_setting(db, "reports.schedule_hour",
                              int(form.get("reports.schedule_hour", 8)))
            await set_setting(db, "reports.schedule_tz", session.get("tz") or "")
    return redirect(url_for("settings.reports"))


@settings_bp.post("/reports/send-now/")
@require_role(UserRole.admin)
async def send_report_now():
    from steward.core.reports import send_report
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    ok, message = await send_report(current_app._get_current_object())
    return await _reports_page(settings, {"ok": ok, "message": message})


# ── Backups ───────────────────────────────────────────────────────────────────

async def _backups_page(flash=None):
    """Settings → Backups (#5673): how the newest backup went, the bucket and
    the daily hour (in the reader's zone), and the runs before."""
    from steward.backup import job, restoring
    from steward.core.formatting import zone
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    hour = job.shown_hour(int(settings.get("backup.schedule_hour", 3)),
                          zone(settings.get("backup.schedule_tz")), zone(session.get("tz")))
    return await render_template(
        "settings/backups.html", settings=settings, flash=flash, shown_hour=hour,
        verdict=job.verdict(settings, job.is_running()),
        history=job.history_rows(settings.get("backup.history")),
        has_secret=bool(settings.get("backup.s3_secret_key")),
        restore_failed=restoring.STATE if restoring.STATE.error else None)


@settings_bp.get("/backups/")
@require_role(UserRole.admin)
async def backups():
    return await _backups_page()


@settings_bp.post("/backups/")
@require_role(UserRole.admin)
async def save_backups():
    """The hour is what the reader picked on their own clock, so it is kept in
    their zone. A blank secret key keeps the stored one."""
    form = await request.form
    endpoint = form.get("backup.s3_endpoint", "").strip()
    if endpoint and not endpoint.startswith(("http://", "https://")):
        return await _backups_page({"ok": False, "message":
                                    "The endpoint starts with http:// or https://, e.g. http://archives:8081."})
    try:
        hour = int(form.get("backup.schedule_hour", 3))
    except ValueError:
        hour = 3
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            await set_setting(db, "backup.enabled", "backup.enabled" in form)
            await set_setting(db, "backup.s3_endpoint", endpoint)
            await set_setting(db, "backup.s3_bucket", form.get("backup.s3_bucket", "").strip())
            await set_setting(db, "backup.s3_region", form.get("backup.s3_region", "").strip() or "us-east-1")
            await set_setting(db, "backup.s3_access_key", form.get("backup.s3_access_key", "").strip())
            secret = form.get("backup.s3_secret_key", "")
            if secret.strip():
                await set_setting(db, "backup.s3_secret_key", secret.strip())
            elif "backup.clear_secret" in form:
                await set_setting(db, "backup.s3_secret_key", "")
            await set_setting(db, "backup.s3_verify_tls", "backup.s3_verify_tls" in form)
            await set_setting(db, "backup.schedule_hour", min(max(hour, 0), 23))
            await set_setting(db, "backup.schedule_tz", session.get("tz") or "")
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "settings.saved", detail={"section": "backups"})
    return redirect(url_for("settings.backups"))


@settings_bp.post("/backups/test/")
@require_role(UserRole.admin)
async def test_backups():
    """Reach the saved bucket with the saved keys (the operator asked, so the
    target is contacted)."""
    from steward.backup import job
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    target = job.target_from(settings)
    if target is None:
        return await _backups_page({"ok": False, "message":
                                    "Save the endpoint, bucket and both keys first."})
    ok, message = await job.test_target(target)
    return await _backups_page({"ok": ok, "message": message})


@settings_bp.post("/backups/now/")
@require_role(UserRole.admin)
async def backup_now():
    from steward.backup import job
    started = job.start_backup(current_app._get_current_object(),
                               by=session.get("username", "") or "an admin",
                               user_id=session.get("user_id"))
    message = ("Making a backup now; it shows under Recent backups when it is sent."
               if started else "A backup is already being made.")
    return await _backups_page({"ok": started, "message": message})


async def _check_page(report=None, error: str = "", name: str = ""):
    return await render_template("settings/backup_check.html", report=report, error=error,
                                 name=name)


@settings_bp.post("/backups/check/")
@require_role(UserRole.admin)
async def check_stored_backup():
    """Check one of the backups on the bucket against what is in place
    (#5823). Read-only: nothing here changes."""
    from steward.backup import job
    from steward.backup.dump import BackupError
    from steward.backup.s3 import S3Error
    name = (await request.form).get("name", "")
    try:
        report = await job.check_stored(current_app._get_current_object(), name)
    except (BackupError, S3Error) as exc:
        error = str(exc)
        if "AccessDenied" in error:
            error += (" The bucket's key may not read backups back; download the file from the "
                      "bucket and check it with Check a file instead.")
        return await _check_page(error=error, name=name)
    return await _check_page(report=report, name=name)


@settings_bp.post("/backups/check/file/")
@require_role(UserRole.admin)
async def check_uploaded_backup():
    """Check a backup file sent from the browser. Its size limit is raised
    before the form is read (auth/csrf.py UPLOAD_LIMITS)."""
    import tempfile

    from steward.backup import job
    from steward.backup.dump import BackupError
    upload = (await request.files).get("file")
    if upload is None or not upload.filename:
        return await _check_page(error="Choose a backup file to check.")
    name = Path(upload.filename).name
    with tempfile.TemporaryDirectory(prefix="steward-check-") as tmp:
        path = Path(tmp) / "upload.tar"
        await upload.save(path)
        try:
            report = await job.check_file(current_app._get_current_object(), path, name)
        except BackupError as exc:
            return await _check_page(error=str(exc), name=name)
    return await _check_page(report=report, name=name)



# Restore (#5817): choose a backup, see what it would change (the same check as
# above), then type to confirm. The chosen file is held server-side between
# the two (restoring.stage), so the confirmation names it by a token.

RESTORE_WORD = "restore"


async def _restore_page(report=None, error: str = "", name: str = "", held=None):
    return await render_template("settings/backup_restore.html", report=report, error=error,
                                 name=name, held=held, word=RESTORE_WORD)


async def _hold_and_check(held, name: str):
    from steward.backup import job
    from steward.backup.dump import BackupError
    try:
        report = await job.check_file(current_app._get_current_object(), held.path, name)
    except BackupError as exc:
        import shutil
        shutil.rmtree(held.dir, ignore_errors=True)
        return await _restore_page(error=str(exc), name=name)
    held.name, held.schema, held.ok = name, report.schema, report.ok
    held.save()
    return await _restore_page(report=report, name=name, held=held)


@settings_bp.post("/backups/restore/")
@require_role(UserRole.admin)
async def restore_stored_backup():
    """Fetch one of the bucket's backups, check it, and ask to confirm."""
    import shutil

    from steward.backup import job, restoring
    from steward.backup.dump import BackupError
    from steward.backup.s3 import S3Error, get_file
    name = (await request.form).get("name", "")
    if not job.is_backup_name(name):
        return await _restore_page(error=f"{name!r} isn't the name of a Steward backup.", name=name)
    async with current_app.db_sessionmaker() as db:
        target = job.target_from(await get_all_settings(db))
    if target is None:
        return await _restore_page(error="The bucket isn't set up: fill in the endpoint, bucket and keys.",
                                   name=name)
    held = restoring.stage()
    try:
        await get_file(target, name, held.path)
    except (BackupError, S3Error) as exc:
        shutil.rmtree(held.dir, ignore_errors=True)
        error = str(exc)
        if "AccessDenied" in error:
            error += (" The bucket's key may not read backups back; download the file from the "
                      "bucket and restore it with Restore a file instead.")
        return await _restore_page(error=error, name=name)
    return await _hold_and_check(held, name)


@settings_bp.post("/backups/restore/file/")
@require_role(UserRole.admin)
async def restore_uploaded_backup():
    """A backup file sent from the browser, checked and held for the
    confirmation. Its size limit is raised in auth/csrf.py UPLOAD_LIMITS."""
    from steward.backup import restoring
    upload = (await request.files).get("file")
    if upload is None or not upload.filename:
        return await _restore_page(error="Choose a backup file to restore.")
    held = restoring.stage()
    await upload.save(held.path)
    return await _hold_and_check(held, Path(upload.filename).name)


@settings_bp.post("/backups/restore/start/")
@require_role(UserRole.admin)
async def start_restore():
    """The confirmation: the held backup goes back in (restoring.py). Every
    page shows the restore until Steward restarts."""
    import shutil

    from steward.backup import job, restoring
    form = await request.form
    held = restoring.staged(form.get("token", ""))
    if held is None:
        return await _restore_page(error="That backup is no longer held (it is removed an hour after "
                                         "it was checked). Choose it again.")
    if held.schema == "newer" or not held.ok:
        shutil.rmtree(held.dir, ignore_errors=True)
        return await _restore_page(error="This backup can't be restored here; check it again to see why.",
                                   name=held.name)
    replace_all = held.schema == "older"
    if form.get("confirm", "").strip().lower() != RESTORE_WORD or (
            replace_all and "replace_all" not in form):
        report = await job.check_file(current_app._get_current_object(), held.path, held.name)
        return await _restore_page(report=report, name=held.name, held=held,
                                   error=f"Type {RESTORE_WORD} to confirm"
                                         + (", and tick Replace everything." if replace_all else "."))
    started = restoring.start_restore(current_app._get_current_object(), held.path, held.name,
                                      username=session.get("username", "") or "an admin",
                                      replace_all=replace_all, owned_dir=held.dir)
    if not started:
        return await _restore_page(error="A restore is already under way.", name=held.name)
    return redirect(url_for("settings.backups"))

def _build_plugin_cfg_from_form(plugin: dict, form) -> dict:
    """Extract config values for one plugin from a form submission."""
    name = plugin["_dir"]
    enabled = name in ALWAYS_ON_PLUGINS or f"plugin.{name}.enabled" in form
    plugin_cfg: dict = {"enabled": enabled}
    for cfg_key in plugin.get("config", {}).keys():
        field_name = f"plugin.{name}.{cfg_key}"
        default_val = plugin["config"][cfg_key]
        if isinstance(default_val, dict):
            sub_dict = {}
            for sub_key, sub_default in default_val.items():
                sub_field = f"plugin.{name}.{cfg_key}.{sub_key}"
                if isinstance(sub_default, bool):
                    sub_dict[sub_key] = sub_field in form
                elif sub_field in form:
                    raw_sub = form[sub_field]
                    if isinstance(sub_default, int):
                        try:
                            sub_dict[sub_key] = int(raw_sub)
                        except ValueError:
                            sub_dict[sub_key] = sub_default
                    else:
                        sub_dict[sub_key] = raw_sub
                else:
                    sub_dict[sub_key] = sub_default
            plugin_cfg[cfg_key] = sub_dict
        elif field_name in form:
            raw_val = form[field_name]
            if isinstance(default_val, bool):
                plugin_cfg[cfg_key] = raw_val.lower() in ("1", "true", "yes", "on")
            elif isinstance(default_val, int):
                try:
                    plugin_cfg[cfg_key] = int(raw_val)
                except ValueError:
                    plugin_cfg[cfg_key] = default_val
            else:
                plugin_cfg[cfg_key] = raw_val
    return plugin_cfg


# ── Plugins list ──────────────────────────────────────────────────────────────

# Bundled plugins that are really built-in *host/monitoring capabilities* (facets
# of a host), not discrete vendor integrations. Presentation-only split — they
# still load through the normal plugin mechanism. A plugin.yaml may set
# kind: capability|integration to override.
CAPABILITY_PLUGINS = {"host_agent", "snmp", "docker", "syslog"}


@settings_bp.get("/plugins/")
@require_role(UserRole.admin)
async def plugins():
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
        stored_plugin_names = await get_stored_plugin_names(db)
    discovered = _discover_plugins()
    _merge_plugin_config(discovered, to_plugins_cfg(settings))
    for p in discovered:
        p["_kind"] = p.get("kind") or (
            "capability" if p["_dir"] in CAPABILITY_PLUGINS else "integration")
    # Config left behind by plugins that are no longer installed. Surfaced for an
    # explicit operator decision rather than cleaned up automatically: an external
    # plugin can be missing because /data/plugins isn't mounted, and silently
    # dropping its row would destroy stored credentials.
    #
    # Load failures are folded in so every plugin counted by the admin banner has
    # a row here. A plugin that failed BECAUSE it has no directory is not in
    # `discovered`, so without this it would be counted in the banner and shown
    # nowhere on the page that banner links to.
    from steward.core.plugin_manager import get_plugin_failures
    orphans = find_orphaned_plugins(
        stored_plugin_names,
        [p["_dir"] for p in discovered],
        get_plugin_failures(),
    )
    return await render_template(
        "settings/plugins.html",
        capabilities=[p for p in discovered if p["_kind"] == "capability"],
        integrations=[p for p in discovered if p["_kind"] != "capability"],
        discovered_plugins=discovered,
        orphaned_plugins=orphans,
        settings=settings,
    )


@settings_bp.post("/plugins/orphans/<name>/remove/")
@require_role(UserRole.admin)
async def plugin_orphan_remove(name: str):
    """Delete the stored settings row for a plugin that is no longer installed.

    Refuses if the plugin IS installed — that would silently wipe a live
    plugin's config, and disabling it is what the operator wants there instead.
    """
    installed = {p["_dir"] for p in _discover_plugins()}
    if name in installed:
        return redirect(url_for("settings.plugins"))

    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            removed = await delete_setting(db, f"plugin.{name}")

    if removed:
        await _reload_app_config()
        await log_audit(
            current_app, session.get("user_id"), session.get("username", ""),
            "plugin.settings_removed", entity_type="plugin", entity_id=name)
    return redirect(url_for("settings.plugins"))


# ── Per-plugin detail (settings) ──────────────────────────────────────────────

@settings_bp.get("/plugins/<name>/")
@require_role(UserRole.admin)
async def plugin_detail(name: str):
    async with current_app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
    discovered = _discover_plugins()
    plugin = next((p for p in discovered if p["_dir"] == name), None)
    if plugin is None:
        return redirect(url_for("settings.plugins"))
    _merge_plugin_config([plugin], to_plugins_cfg(settings))
    from steward.core.plugin_manager import _LOADED_PLUGINS, get_plugin_failures
    return await render_template(
        "settings/plugin_detail.html",
        plugin=plugin,
        is_loaded=name in _LOADED_PLUGINS,
        fail_reason=get_plugin_failures().get(name),
    )


@settings_bp.post("/plugins/<name>/")
@require_role(UserRole.admin)
async def save_plugin_detail(name: str):
    form = await request.form
    discovered = _discover_plugins()
    plugin = next((p for p in discovered if p["_dir"] == name), None)
    if plugin is None:
        return redirect(url_for("settings.plugins"))

    async with current_app.db_sessionmaker() as db:
        old_settings = await get_all_settings(db)
    old_enabled = to_plugins_cfg(old_settings).get(name, {}).get("enabled", False)

    plugin_cfg = _build_plugin_cfg_from_form(plugin, form)
    # A secret is never sent back to the browser, so its box arrives blank
    # unless someone typed a new one: blank keeps the saved value.
    old_cfg = to_plugins_cfg(old_settings).get(name, {})
    for field, value in list(plugin_cfg.items()):
        if is_plugin_secret(field) and value == "" and old_cfg.get(field):
            plugin_cfg[field] = old_cfg[field]

    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            await set_setting(db, f"plugin.{name}", plugin_cfg)

    await _reload_app_config()

    new_enabled = plugin_cfg.get("enabled", False)
    if old_enabled != new_enabled:
        action = "plugin.enabled" if new_enabled else "plugin.disabled"
        await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                        action, entity_type="plugin", entity_id=name)
    # Any save of an enabled plugin that isn't running tries to start it; a
    # failure is recorded with its reason, which the page shows beside a
    # restart (#6087).
    from steward.core.plugin_manager import _LOADED_PLUGINS, hot_reload_plugin
    if new_enabled and name not in _LOADED_PLUGINS:
        hot_reload_plugin(current_app._get_current_object(), name)

    return redirect(url_for("settings.plugin_detail", name=name))


# ── App restart ───────────────────────────────────────────────────────────────

@settings_bp.post("/plugins/restart/")
@require_role(UserRole.admin)
async def restart_app_route():
    """Trigger an in-app restart (replaces the process via os.execv)."""
    import asyncio
    from steward.core.plugin_manager import restart_app

    # Schedule the restart slightly after we return the response
    async def _delayed_restart():
        await asyncio.sleep(0.5)
        restart_app()

    asyncio.create_task(_delayed_restart())
    return await render_template("settings/_restart_pending.html")