feat(platform): tini as PID 1, Hypercorn, a log-level setting, the image on CI's Python, an airgapped boot gate before push (#5159)

- Dockerfile: python:3.14-slim, the Python CI tests on (was 3.13). tini from
  apt is PID 1 (ENTRYPOINT tini -- entrypoint.sh): it reaps the zombies ssh
  and ansible leave and forwards SIGTERM.
- cli: Hypercorn's serve() with one worker (run, stream and scheduler state is
  in-process), a 10 s graceful timeout, and SIGTERM/SIGINT as the shutdown
  trigger. Access and error lines go through Steward's logging. --debug keeps
  Quart's reloading dev server.
- Logging: steward/core/logging_setup.py installs one root handler before
  anything logs (until now a library's basicConfig on import was the only
  configuration). Its level is the `log.level` setting (DEBUG/INFO/WARNING/
  ERROR, default INFO), applied at boot and on save in Settings → General.
- CI publish: build, then "The image boots with no internet" (rule 164,
  snippet #3552), then push, so a broken image never reaches a tag (rule 177).
  On an --internal network with its own Postgres, the image must migrate and
  answer /health, run tini as PID 1, serve first-run setup with its assets
  from /static and no third-party host, and exit 0 on SIGTERM. Logs and
  clean-up run always.

Tests: tests/core/test_logging_setup.py; the smoke step itself.
Not in this commit: the dependency lockfile (it conflicts with rule 154 and
is waiting on the operator).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/app.py

@@ -93,6 +93,8 @@ def create_app(
             to_thresholds_cfg, to_access_cfg,
         )
         _settings = load_settings_sync(app.config["DATABASE_URL"])
+        from .core.logging_setup import set_log_level
+        set_log_level(_settings.get("log.level"))
         app.config.update(to_access_cfg(_settings))
         app.config.update(
             DATA_RETENTION_DAYS=_settings.get("data.retention_days", 90),

steward/cli.py

@@ -1,14 +1,56 @@
+import asyncio
+import logging
+import signal
 from pathlib import Path
+
 import click
-from .app import create_app
+
+from .core.logging_setup import configure_logging
+
+# How long a stop waits for requests in flight (and SSE streams) to finish.
+GRACEFUL_TIMEOUT_SECONDS = 10.0
+
+
+def hypercorn_config(host: str, port: int):
+    """Hypercorn as the production server (#5159): one worker, because Steward
+    keeps run, stream and scheduler state in-process; its access and error
+    lines go through Steward's logging, so the log.level setting governs them."""
+    from hypercorn.config import Config
+
+    config = Config()
+    config.bind = [f"{host}:{port}"]
+    config.workers = 1
+    config.graceful_timeout = GRACEFUL_TIMEOUT_SECONDS
+    config.accesslog = logging.getLogger("hypercorn.access")
+    config.errorlog = logging.getLogger("hypercorn.error")
+    config.access_log_format = "%(h)s %(r)s %(s)s %(b)s %(D)s"
+    return config
+
+
+async def _serve(app, config) -> None:
+    from hypercorn.asyncio import serve
+
+    stop = asyncio.Event()
+    loop = asyncio.get_running_loop()
+    for sig in (signal.SIGTERM, signal.SIGINT):
+        loop.add_signal_handler(sig, stop.set)
+    await serve(app, config, shutdown_trigger=stop.wait)
 
 
 @click.command()
 @click.option("--config", default="config.yaml", help="Path to config.yaml")
 @click.option("--host", default="0.0.0.0")
 @click.option("--port", default=5000, type=int)
-@click.option("--debug", is_flag=True, default=False)
+@click.option("--debug", is_flag=True, default=False,
+              help="Quart's development server with the reloader, not Hypercorn.")
 def main(config: str, host: str, port: int, debug: bool) -> None:
     """Start the Steward server."""
+    # Before anything logs; create_app applies the log.level setting once
+    # settings load.
+    configure_logging()
+    from .app import create_app
     app = create_app(config_path=Path(config))
-    app.run(host=host, port=port, debug=debug)
+    if debug:
+        app.run(host=host, port=port, debug=True)
+        return
+    asyncio.run(_serve(app, hypercorn_config(host, port)))

steward/core/logging_setup.py

@@ -0,0 +1,37 @@
+"""Steward's logging: one handler on the root logger, its level a setting (#5159).
+
+Until this, nothing in Steward configured logging. Lines reached the container
+log only because a library called basicConfig on import, so the level and the
+format were whatever that library chose. configure_logging runs first thing in
+the CLI, so a later basicConfig anywhere is a no-op. The level is the
+`log.level` setting, applied once settings load and again when it is saved.
+"""
+from __future__ import annotations
+
+import logging
+import sys
+
+LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR")
+DEFAULT_LEVEL = "INFO"
+FORMAT = "%(asctime)s %(levelname)s [%(name)s] %(message)s"
+
+
+def set_log_level(level: object) -> str:
+    """Set the root level; an unknown value falls back to INFO. Returns the
+    level applied."""
+    name = str(level or "").strip().upper()
+    if name not in LEVELS:
+        name = DEFAULT_LEVEL
+    logging.getLogger().setLevel(name)
+    return name
+
+
+def configure_logging(level: object = DEFAULT_LEVEL) -> str:
+    """Install Steward's handler on the root logger (once) and set its level."""
+    root = logging.getLogger()
+    if not any(getattr(h, "_steward", False) for h in root.handlers):
+        handler = logging.StreamHandler(sys.stderr)
+        handler.setFormatter(logging.Formatter(FORMAT))
+        handler._steward = True  # type: ignore[attr-defined]
+        root.addHandler(handler)
+    return set_log_level(level)

steward/core/settings.py

@@ -53,6 +53,9 @@ DEFAULTS: dict[str, Any] = {
     # The audit log is a security record: kept longer than operational data.
     "audit.retention_days": 365,
     "monitors.poll_interval_seconds": 60,
+    # Root log level: DEBUG, INFO, WARNING or ERROR (steward/core/logging_setup.py).
+    # WARNING also hides the per-request access lines.
+    "log.level": "INFO",
     "smtp.host": "",
     "smtp.port": 587,
     "smtp.tls": True,

steward/settings/routes.py

@@ -7,6 +7,7 @@ from quart import Blueprint, current_app, render_template, request, redirect, ur
 
 from steward.auth.middleware import require_role
 from steward.core.audit import audit_request, log_audit
+from steward.core.logging_setup import LEVELS as LOG_LEVELS, set_log_level
 from steward.models.users import UserRole
 from steward.core.settings import (
     get_all_settings, set_setting, delete_setting,
@@ -93,7 +94,8 @@ async def index():
 async def general():
     async with current_app.db_sessionmaker() as db:
         settings = await get_all_settings(db)
-    return await render_template("settings/general.html", settings=settings)
+    return await render_template("settings/general.html", settings=settings,
+                                 log_levels=LOG_LEVELS)
 
 
 @settings_bp.post("/general/")
@@ -117,6 +119,7 @@ async def save_general():
                               int(form.get("data.retention_days", 90)))
             await set_setting(db, "monitors.poll_interval_seconds",
                               int(form.get("monitors.poll_interval_seconds", 60)))
+            await set_setting(db, "log.level", set_log_level(form.get("log.level")))
     await _reload_app_config()
     await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                     "settings.saved", detail={"section": "general"})

steward/templates/settings/general.html

@@ -74,6 +74,16 @@
            value="{{ settings['monitors.poll_interval_seconds'] }}" style="max-width:120px;">
     <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.25rem;">Requires restart to take effect.</div>
   </div>
+
+  <div class="form-group">
+    <label>Log level</label>
+    <select name="log.level" style="max-width:160px;">
+      {% for level in log_levels %}
+      <option value="{{ level }}" {% if settings['log.level'] == level %}selected{% endif %}>{{ level | capitalize }}</option>
+      {% endfor %}
+    </select>
+    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.25rem;">What reaches the container log. Applies when saved. Warning hides the line logged for each request.</div>
+  </div>
 </div>
 
 <div style="margin-top:1rem;display:flex;align-items:center;gap:1rem;">