feat: two-phase migrations, DB settings loading in create_app

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

fablednetmon/app.py

@@ -1,8 +1,9 @@
+# fablednetmon/app.py
 from __future__ import annotations
 import asyncio
 from pathlib import Path
 from quart import Quart
-from .config import load_config
+from .config import load_bootstrap
 from .database import init_db
 
 
@@ -12,72 +13,107 @@ def create_app(
 ) -> Quart:
     app = Quart(__name__, template_folder="templates")
 
-    # Load config
-    cfg = load_config(config_path)
+    # ── 1. Bootstrap: db_url + secret_key only ────────────────────────────────
+    if not testing:
+        bootstrap = load_bootstrap(config_path)
+    else:
+        bootstrap = {
+            "database_url": "postgresql+asyncpg://test/test",
+            "secret_key": "test-secret-key",
+            "plugin_dir": "plugins",
+        }
+
     app.config.update(
-        SECRET_KEY=cfg["secret_key"],
-        DATABASE_URL=cfg["database"]["url"],
-        SESSION_LIFETIME_HOURS=cfg.get("session", {}).get("lifetime_hours", 8),
-        DATA_RETENTION_DAYS=cfg.get("data", {}).get("retention_days", 90),
-        PLUGIN_DIR=cfg.get("plugin_dir", "plugins"),
-        PLUGINS=cfg.get("plugins", {}),
-        SMTP=cfg.get("smtp", {}),
-        WEBHOOK=cfg.get("webhook", {}),
-        ANSIBLE=cfg.get("ansible", {}),
+        SECRET_KEY=bootstrap["secret_key"],
+        DATABASE_URL=bootstrap["database_url"],
+        PLUGIN_DIR=bootstrap["plugin_dir"],
         TESTING=testing,
-        _RAW_CFG=cfg,
     )
 
-    # Database
+    # ── 2. Init DB engine ──────────────────────────────────────────────────────
     if not testing:
         init_db(app)
     else:
         from unittest.mock import MagicMock
         app.db_sessionmaker = MagicMock()
 
-    # Run migrations (core + plugin) synchronously before event loop starts
+    # ── 3. Core migrations only (creates app_settings table) ──────────────────
     if not testing:
-        from .core.migration_runner import run_migrations, get_plugin_migration_dirs
+        from .core.migration_runner import run_core_migrations
+        run_core_migrations(app.config["DATABASE_URL"])
+
+    # ── 4. Load all settings from DB → populate app.config ────────────────────
+    if not testing:
+        from .core.settings import (
+            load_settings_sync, to_smtp_cfg, to_webhook_cfg,
+            to_ansible_cfg, to_plugins_cfg,
+        )
+        _settings = load_settings_sync(app.config["DATABASE_URL"])
+        app.config.update(
+            SESSION_LIFETIME_HOURS=_settings.get("session.lifetime_hours", 8),
+            DATA_RETENTION_DAYS=_settings.get("data.retention_days", 90),
+            MONITORS_POLL_INTERVAL=_settings.get("monitors.poll_interval_seconds", 60),
+            SMTP=to_smtp_cfg(_settings),
+            WEBHOOK=to_webhook_cfg(_settings),
+            ANSIBLE=to_ansible_cfg(_settings),
+            PLUGINS=to_plugins_cfg(_settings),
+        )
+    else:
+        app.config.update(
+            SESSION_LIFETIME_HOURS=8,
+            DATA_RETENTION_DAYS=90,
+            MONITORS_POLL_INTERVAL=60,
+            SMTP={},
+            WEBHOOK={},
+            ANSIBLE={},
+            PLUGINS={},
+        )
+
+    # ── 5. Plugin migrations (depends on knowing which plugins are enabled) ────
+    if not testing:
+        from .core.migration_runner import run_plugin_migrations, get_plugin_migration_dirs
         _plugin_dir = Path(app.config["PLUGIN_DIR"]).resolve()
         _plugin_migration_dirs = get_plugin_migration_dirs(
             _plugin_dir, app.config["PLUGINS"]
         )
-        run_migrations(app.config["DATABASE_URL"], _plugin_migration_dirs)
+        run_plugin_migrations(app.config["DATABASE_URL"], _plugin_migration_dirs)
 
-    # Alert pipeline
+    # ── 6. Alert pipeline ──────────────────────────────────────────────────────
     from .core.alerts import init_alerts
     init_alerts(app)
 
-    # Register core blueprints
+    # ── 7. Register core blueprints ────────────────────────────────────────────
     from .auth.routes import auth_bp
     from .dashboard.routes import dashboard_bp
     from .hosts.routes import hosts_bp
     from .alerts.routes import alerts_bp
     from .ansible.routes import ansible_bp
+    from .settings.routes import settings_bp
 
     app.register_blueprint(auth_bp)
     app.register_blueprint(dashboard_bp)
     app.register_blueprint(hosts_bp)
     app.register_blueprint(alerts_bp)
     app.register_blueprint(ansible_bp)
+    app.register_blueprint(settings_bp)
 
-    # Build task registry
+    # ── 8. Build task registry ─────────────────────────────────────────────────
     app._task_registry = []
 
     if not testing:
-        _register_core_tasks(app, cfg)
+        _register_core_tasks(app)
 
-    # Load plugins
+    # ── 9. Load plugins ────────────────────────────────────────────────────────
     if not testing:
         from .core.plugin_manager import load_plugins
         load_plugins(app)
 
-    # Health check
+    # ── 10. Health check ───────────────────────────────────────────────────────
     @app.get("/health")
     async def health():
         return {"status": "ok"}
 
-    # Start scheduler on first request
+    # ── 11. Startup hook ───────────────────────────────────────────────────────
     @app.before_serving
     async def startup():
         if not testing:
@@ -87,10 +123,10 @@ def create_app(
     return app
 
 
-def _register_core_tasks(app: Quart, cfg: dict) -> None:
+def _register_core_tasks(app: Quart) -> None:
     from .core.scheduler import ScheduledTask
 
-    poll_interval = cfg.get("monitors", {}).get("poll_interval_seconds", 60)
+    poll_interval = app.config.get("MONITORS_POLL_INTERVAL", 60)
     cleanup_interval = 3600  # hourly
 
     async def run_ping_monitors():
@@ -148,13 +184,13 @@ def _register_core_tasks(app: Quart, cfg: dict) -> None:
         ),
     ])
 
-    # Register git-pull tasks for each git source
-    ansible_cfg = cfg.get("ansible", {})
+    # Register git-pull tasks for each ansible git source
+    ansible_cfg = app.config.get("ANSIBLE", {})
     from .ansible.sources import get_sources
     for source in get_sources(ansible_cfg):
         if source["type"] != "git":
             continue
-        _source = source  # capture for closure
+        _source = source
 
         async def run_git_pull(src=_source):
             from .ansible.sources import git_pull
@@ -171,7 +207,6 @@ def _register_core_tasks(app: Quart, cfg: dict) -> None:
 
 
 async def _mark_interrupted_runs(app: Quart) -> None:
-    """Mark any runs still in 'running' state as 'interrupted' (process restart)."""
     from sqlalchemy import update
     from .models.ansible import AnsibleRun, AnsibleRunStatus
     from datetime import datetime, timezone

fablednetmon/core/migration_runner.py

@@ -1,3 +1,4 @@
+# fablednetmon/core/migration_runner.py
 from __future__ import annotations
 import logging
 from pathlib import Path
@@ -5,13 +6,26 @@ from pathlib import Path
 logger = logging.getLogger(__name__)
 
 
-def run_migrations(db_url: str, plugin_migration_dirs: list[Path] | None = None) -> None:
-    """Run core + plugin Alembic migrations synchronously.
+def run_core_migrations(db_url: str) -> None:
+    """Run only core Alembic migrations (no plugin dirs).
 
-    Called during create_app() before the event loop starts.
-    Applies all pending migrations in dependency order.
-    Plugin migration dirs are included via Alembic's version_locations.
+    Called first so the app_settings table exists before loading settings.
     """
+    _run(db_url, plugin_migration_dirs=[])
+
+
+def run_plugin_migrations(db_url: str, plugin_migration_dirs: list[Path]) -> None:
+    """Run plugin Alembic migrations after settings are loaded."""
+    if plugin_migration_dirs:
+        _run(db_url, plugin_migration_dirs=plugin_migration_dirs)
+
+
+def run_migrations(db_url: str, plugin_migration_dirs: list[Path] | None = None) -> None:
+    """Run core + plugin Alembic migrations (single-phase convenience wrapper)."""
+    _run(db_url, plugin_migration_dirs=plugin_migration_dirs or [])
+
+
+def _run(db_url: str, plugin_migration_dirs: list[Path]) -> None:
     from alembic import command
     from alembic.config import Config
 
@@ -22,9 +36,8 @@ def run_migrations(db_url: str, plugin_migration_dirs: list[Path] | None = None)
     cfg.set_main_option("script_location", str(core_migrations))
     cfg.set_main_option("sqlalchemy.url", db_url)
 
-    # Build version_locations: core + any plugin version directories
     version_locs = [str(core_versions)]
-    for plugin_dir in (plugin_migration_dirs or []):
+    for plugin_dir in plugin_migration_dirs:
         pv = plugin_dir / "versions"
         if pv.exists():
             version_locs.append(str(pv))
@@ -37,12 +50,7 @@ def run_migrations(db_url: str, plugin_migration_dirs: list[Path] | None = None)
 
 
 def get_plugin_migration_dirs(plugin_dir: Path, plugins_cfg: dict) -> list[Path]:
-    """Return migration directories for all enabled plugins that have one.
-
-    Does not import any Python code — filesystem scan only.
-    Used to collect migration dirs before importing plugins, so migrations
-    can run before plugin Python code is loaded.
-    """
+    """Return migration directories for enabled plugins (filesystem scan only)."""
     dirs = []
     for name, cfg in plugins_cfg.items():
         if not cfg.get("enabled", False):