diff --git a/compose.deploy.yml b/compose.deploy.yml index bb08e55..c5bd8fb 100644 --- a/compose.deploy.yml +++ b/compose.deploy.yml @@ -22,12 +22,22 @@ services: # /data holds the auto-generated secret key, third-party plugins, and the # Ansible playbook cache — all of which must survive image updates. No # source bind-mounts: core + first-party plugins live inside the image. + # + # The container runs as the unprivileged 'app' user (uid 1000), so /data + # MUST be writable by uid 1000. A named docker volume (below) gets that + # automatically. If you bind-mount a host/NFS path instead, chown it to + # 1000:1000 (and mind NFS root_squash) — otherwise the app can't persist + # its secret key, mints a fresh one each boot, and every encrypted secret + # (managed SSH key, SMTP/OIDC/LDAP creds) becomes unrecoverable. The app + # now refuses to start in that state rather than silently degrade. - app_data:/data environment: - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:${POSTGRES_PASSWORD:-steward}@db/steward # STEWARD_SECRET_KEY is intentionally absent — the app generates a random # key on first boot and persists it to /data/secret.key, which the app_data - # volume keeps across image updates. + # volume keeps across image updates. On multi-node Swarm with a shared bind + # mount, pin STEWARD_SECRET_KEY (env or a docker secret) instead, so the key + # never depends on filesystem ownership being correct on every node. depends_on: db: condition: service_healthy diff --git a/steward/app.py b/steward/app.py index 29b26f4..8e5915e 100644 --- a/steward/app.py +++ b/steward/app.py @@ -1,11 +1,14 @@ # steward/app.py from __future__ import annotations import asyncio +import logging from pathlib import Path from quart import Quart, render_template from .config import load_bootstrap from .database import init_db +logger = logging.getLogger(__name__) + def create_app( config_path: Path | str | None = None, @@ -50,9 +53,17 @@ def create_app( # ── 3b. Encryption-at-rest: bind the encryptor, then encrypt legacy secrets ─ if not testing: from .core.crypto import init_crypto - from .core.settings import migrate_plaintext_secrets + from .core.settings import migrate_plaintext_secrets, scan_undecryptable_secrets init_crypto(app.config["SECRET_KEY"]) migrate_plaintext_secrets(app.config["DATABASE_URL"]) + # Flag any secrets sealed under a now-lost key (see the admin banner). + undecryptable = scan_undecryptable_secrets(app.config["DATABASE_URL"]) + if undecryptable: + logger.warning( + "%d stored secret(s) cannot be decrypted with the current app key " + "(re-enter them in Settings): %s", + len(undecryptable), ", ".join(undecryptable), + ) # ── 4. Load all settings from DB → populate app.config ──────────────────── if not testing: @@ -167,6 +178,7 @@ def create_app( @app.context_processor def _inject_plugin_failures(): from .core.plugin_manager import get_plugin_failures + from .core.settings import get_undecryptable_secrets # enabled_plugins lets templates gate cross-plugin embeds (e.g. the # Hosts hub only fetches the docker fragment when docker is enabled), # avoiding a 404 to a route whose blueprint isn't registered. @@ -177,6 +189,9 @@ def create_app( return { "plugin_failures": get_plugin_failures(), "enabled_plugins": enabled_plugins, + # Read from an in-memory cache (no DB hit per render); kept current by + # the startup scan + set_setting's discard-on-write. + "undecryptable_secrets": get_undecryptable_secrets(), } # ── 11. Share-token middleware ───────────────────────────────────────────── diff --git a/steward/config.py b/steward/config.py index 9c8f1fd..678359e 100644 --- a/steward/config.py +++ b/steward/config.py @@ -75,13 +75,27 @@ def load_bootstrap(config_path: Path | str | None = None) -> dict[str, Any]: def _resolve_secret_key(raw: dict) -> str: - """Resolve secret_key: env var → file → auto-generate.""" + """Resolve secret_key: env var → file → auto-generate. + + Refuses to start if a new key must be generated but cannot be persisted: an + ephemeral key changes on every restart, which silently renders every + encrypted secret (managed SSH key, SMTP/OIDC/LDAP credentials) unrecoverable. + Failing loudly with a fix beats limping along and losing data on the next + boot — exactly the footgun that bit the vdnt-docker02 deployment. + """ from_env = _env("SECRET_KEY") or raw.get("secret_key") if from_env: return from_env if _SECRET_KEY_FILE.exists(): - key = _SECRET_KEY_FILE.read_text().strip() + try: + key = _SECRET_KEY_FILE.read_text().strip() + except OSError as exc: + raise RuntimeError( + f"App secret key file {_SECRET_KEY_FILE} exists but cannot be read " + f"({exc}). Make it readable by the container user (uid 1000), or set " + f"STEWARD_SECRET_KEY." + ) from exc if key: return key @@ -89,11 +103,16 @@ def _resolve_secret_key(raw: dict) -> str: try: _SECRET_KEY_FILE.parent.mkdir(parents=True, exist_ok=True) _SECRET_KEY_FILE.write_text(key) - logger.info("Generated new secret key and saved to %s", _SECRET_KEY_FILE) except OSError as exc: - logger.warning( - "Could not write secret key to %s (%s). " - "Key will not persist across restarts.", - _SECRET_KEY_FILE, exc, - ) + raise RuntimeError( + f"Generated a new app secret key but could not persist it to " + f"{_SECRET_KEY_FILE} ({exc}). An ephemeral key changes on every restart, " + f"which makes all encrypted secrets (managed SSH key, SMTP/OIDC/LDAP " + f"credentials) unrecoverable. Fix one of:\n" + f" • set STEWARD_SECRET_KEY to a stable value " + f"(recommended for Swarm / multi-node), or\n" + f" • make {_SECRET_KEY_FILE.parent} writable by the container user " + f"(uid 1000)." + ) from exc + logger.info("Generated new secret key and saved to %s", _SECRET_KEY_FILE) return key diff --git a/steward/core/settings.py b/steward/core/settings.py index 23bece8..339dc75 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -150,6 +150,34 @@ def _decode(value: Any, key: str = "") -> Any: return decrypt_secret(value, context=key) if is_encrypted(value) else value +# Secret settings that are stored as ciphertext but won't decrypt with the +# current app key (key rotated or lost). Surfaced as an admin banner so the +# operator knows precisely which secrets to re-enter — instead of finding out +# only when something that uses one fails. Refreshed at startup +# (scan_undecryptable_secrets) and kept live: re-entering a secret clears it +# without a restart (set_setting discards it on a fresh write). +_undecryptable_secrets: set[str] = set() + + +def get_undecryptable_secrets() -> list[str]: + """Sorted keys whose stored ciphertext won't decrypt (for the UI banner).""" + return sorted(_undecryptable_secrets) + + +def _is_undecryptable(stored: Any, key: str = "") -> bool: + """True iff `stored` is an encrypted token that fails to decrypt. + + A failed decrypt returns the ciphertext unchanged (still enc-prefixed), so a + value that is still encrypted after a decrypt attempt is undecryptable. + """ + from steward.core.crypto import decrypt_secret, is_encrypted + return ( + isinstance(stored, str) + and is_encrypted(stored) + and is_encrypted(decrypt_secret(stored, context=key)) + ) + + # ───────────────────────────────────────────────────────────────────────────── # Async helpers (use inside request handlers / scheduled tasks) # ───────────────────────────────────────────────────────────────────────────── @@ -182,6 +210,12 @@ async def set_setting(session: AsyncSession, key: str, value: Any) -> None: row.value_json = json.dumps(to_store) row.updated_at = now + # A fresh write of a secret is encrypted with the current key (or cleared to + # plaintext), so it's decryptable now — drop any stale "undecryptable" flag + # so the banner clears without a restart. + if key in SECRET_KEYS: + _undecryptable_secrets.discard(key) + async def get_all_settings(session: AsyncSession) -> dict[str, Any]: """Return flat key→value dict with defaults filled in for missing keys.""" @@ -359,6 +393,37 @@ def migrate_plaintext_secrets(db_url: str) -> int: return asyncio.run(_run()) +def scan_undecryptable_secrets(db_url: str) -> list[str]: + """Populate the undecryptable-secrets cache from the DB; return the keys found. + + Run once at startup, AFTER migrate_plaintext_secrets and init_crypto: any row + that's encrypted but won't decrypt with the current key was sealed under a + different (lost/rotated) key and must be re-entered. Surfaced via the admin + banner (get_undecryptable_secrets). + """ + async def _run() -> list[str]: + from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker + engine = create_async_engine(db_url, echo=False) + factory = async_sessionmaker(engine, expire_on_commit=False) + bad: list[str] = [] + try: + async with factory() as session: + result = await session.execute( + select(AppSetting).where(AppSetting.key.in_(SECRET_KEYS)) + ) + for row in result.scalars(): + if _is_undecryptable(json.loads(row.value_json), row.key): + bad.append(row.key) + finally: + await engine.dispose() + return bad + + global _undecryptable_secrets + found = asyncio.run(_run()) + _undecryptable_secrets = set(found) + return sorted(found) + + # ───────────────────────────────────────────────────────────────────────────── # External URL helper # ───────────────────────────────────────────────────────────────────────────── diff --git a/steward/templates/base.html b/steward/templates/base.html index 3a03dfa..7b7a95d 100644 --- a/steward/templates/base.html +++ b/steward/templates/base.html @@ -280,6 +280,34 @@ function setTimeRange(val) { {% endif %} + {% if undecryptable_secrets and session.user_role == 'admin' %} + {% set _sec_tab = { + 'smtp.password': '/settings/notifications/', + 'oidc.client_secret': '/settings/auth/', + 'ldap.bind_password': '/settings/auth/', + 'ansible.ssh_private_key': '/settings/ansible/', + 'ansible.become_password': '/settings/ansible/', + 'ansible.vault_password': '/settings/ansible/', + } %} +
+ ⚠ + + + {{ undecryptable_secrets | length }} stored secret{{ 's' if undecryptable_secrets | length != 1 }} + can’t be decrypted + + — the app secret key changed, so {{ 'they' if undecryptable_secrets | length != 1 else 'it' }} + must be re-entered: + {% for k in undecryptable_secrets -%} + {%- if _sec_tab.get(k) %}{{ k }} + {%- else %}{{ k }}{% endif %}{% if not loop.last %}, {% endif %} + {%- endfor %}. + +
+ {% endif %} {% if error %}
{{ error }}
{% endif %} diff --git a/tests/core/test_undecryptable_secrets.py b/tests/core/test_undecryptable_secrets.py new file mode 100644 index 0000000..c515ca0 --- /dev/null +++ b/tests/core/test_undecryptable_secrets.py @@ -0,0 +1,29 @@ +"""Unit tests for undecryptable-secret detection (the admin-banner source).""" +from steward.core.crypto import encrypt_secret, init_crypto +from steward.core.settings import _is_undecryptable, get_undecryptable_secrets + + +def test_is_undecryptable_false_for_current_key(): + init_crypto("key-A") + tok = encrypt_secret("hunter2") + assert _is_undecryptable(tok, "smtp.password") is False + + +def test_is_undecryptable_true_after_key_rotation(): + init_crypto("key-A") + tok = encrypt_secret("hunter2") # sealed under key-A + init_crypto("key-B") # key changed/lost + assert _is_undecryptable(tok, "smtp.password") is True + + +def test_is_undecryptable_ignores_plaintext_and_empty(): + init_crypto("key-A") + assert _is_undecryptable("", "smtp.password") is False + assert _is_undecryptable("plain-value", "smtp.password") is False + assert _is_undecryptable(None, "smtp.password") is False + + +def test_get_undecryptable_secrets_returns_sorted_list(): + # Default (nothing scanned) is an empty list; the accessor is always safe to + # call from the template context processor. + assert isinstance(get_undecryptable_secrets(), list) diff --git a/tests/test_config.py b/tests/test_config.py index 40e9203..93d482a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,6 @@ import textwrap import pytest -from steward.config import load_bootstrap +from steward.config import load_bootstrap, _resolve_secret_key def test_load_bootstrap_from_yaml(tmp_path): @@ -36,3 +36,32 @@ def test_missing_database_url_raises(tmp_path): cfg_file.write_text("secret_key: s\n") with pytest.raises(ValueError, match="Database URL is required"): load_bootstrap(cfg_file) + + +# ── secret key resolution ────────────────────────────────────────────────────── + +def test_secret_key_existing_file_is_reused(tmp_path, monkeypatch): + key_file = tmp_path / "secret.key" + key_file.write_text("persisted-key\n") + monkeypatch.setattr("steward.config._SECRET_KEY_FILE", key_file) + monkeypatch.delenv("STEWARD_SECRET_KEY", raising=False) + assert _resolve_secret_key({}) == "persisted-key" + + +def test_secret_key_generated_and_persisted_when_writable(tmp_path, monkeypatch): + key_file = tmp_path / "sub" / "secret.key" # parent doesn't exist yet + monkeypatch.setattr("steward.config._SECRET_KEY_FILE", key_file) + monkeypatch.delenv("STEWARD_SECRET_KEY", raising=False) + key = _resolve_secret_key({}) + assert key and key_file.read_text().strip() == key + + +def test_secret_key_unpersistable_raises_instead_of_ephemeral(tmp_path, monkeypatch): + # A file sits where a directory is needed, so mkdir/write fails → must raise + # (an ephemeral key would silently orphan every encrypted secret). + blocker = tmp_path / "blocker" + blocker.write_text("not a dir") + monkeypatch.setattr("steward.config._SECRET_KEY_FILE", blocker / "secret.key") + monkeypatch.delenv("STEWARD_SECRET_KEY", raising=False) + with pytest.raises(RuntimeError, match="could not persist"): + _resolve_secret_key({})