FabledSteward/tests/integration/test_docker_service_page.py

"""Integration: a swarm service's own page (#5302) and the Docker overview's
service table (#5299, #5300) against real Postgres. The page reads the logs
and samples of every task the service ran, the live one and one Docker has
already removed (known only by its events), and a failure's "Logs →" picks
that task. Requires STEWARD_DATABASE_URL."""
from __future__ import annotations

import asyncio
import importlib
import json
import os
import time
import uuid
from datetime import datetime, timedelta, timezone

import pytest

pytestmark = pytest.mark.integration


@pytest.fixture
def app():
    if not os.environ.get("STEWARD_DATABASE_URL"):
        pytest.skip("needs Postgres")
    from steward.app import create_app
    return create_app(testing=False)


def _docker(sub):
    """The app's loaded docker plugin module (see test_docker_recent_usage)."""
    from steward.core.plugin_manager import get_loaded_plugin_modules
    loaded = get_loaded_plugin_modules().get("docker")
    base = loaded.__name__ if loaded is not None else "plugins.docker"
    return importlib.import_module(f"{base}.{sub}")


def test_a_service_page_reads_every_task(app):
    from steward.models.hosts import Host
    from steward.models.metrics import PluginMetric
    from steward.models.users import User, UserRole
    m = _docker("models")

    tag = uuid.uuid4().hex[:8]
    name = f"it{tag}_web"
    live_task, dead_task = "a" * 17 + tag, "b" * 17 + tag          # 25 characters, as Swarm's
    live, dead = f"{name}.1.{live_task}", f"{name}.1.{dead_task}"
    hid, uid, node = str(uuid.uuid4()), str(uuid.uuid4()), f"n{tag}"
    now = datetime.now(timezone.utc)

    def iso(dt):
        return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")

    tasks = [{"task_id": live_task, "slot": 1, "node_id": node, "desired_state": "running",
              "state": "running", "message": "started", "error": "", "exit_code": None,
              "at": iso(now - timedelta(minutes=9))},
             {"task_id": dead_task, "slot": 1, "node_id": node, "desired_state": "shutdown",
              "state": "failed", "message": "started", "error": "task: non-zero exit (1)",
              "exit_code": 1, "at": iso(now - timedelta(minutes=10))}]

    def sample(cname, mins, cpu, mem):
        when = now - timedelta(minutes=mins)
        return [PluginMetric(source_module="docker", resource_name=f"swarm/{cname.split('.')[0]}",
                             metric_name=metric, value=value, recorded_at=when)
                for metric, value in (("cpu_pct", cpu), ("mem_used_bytes", mem))]

    async def go():
        async with app.db_sessionmaker() as s:
            async with s.begin():
                s.add(User(id=uid, username=f"sp-{tag}", email=f"sp-{tag}@test",
                           password_hash="x", role=UserRole.viewer, is_active=True))
                s.add(Host(id=hid, name=f"dock-{tag}", address="10.9.7.1"))
            async with s.begin():
                s.add_all([
                    m.DockerSwarmNode(host_id=hid, node_id=node, hostname=f"node-{tag}", role="manager",
                                      availability="active", status="ready", leader=True),
                    m.DockerSwarmService(host_id=hid, service_name=name, mode="replicated", desired=1,
                                         running=1, image="nginx:1.27",
                                         placement_json=json.dumps([{"node_id": node, "running": 1}]),
                                         tasks_json=json.dumps(tasks), updated_at=now),
                    m.DockerContainer(host_id=hid, name=live, image="nginx:1.27", status="running",
                                      cpu_pct=50.0, mem_used_bytes=100 << 20, service_name=name,
                                      task_id=live_task, node_id=node, started_at=now - timedelta(minutes=9),
                                      scraped_at=now),
                    # Docker has removed the failed task's container: only its events are left.
                    m.DockerEvent(host_id=hid, container_name=dead, event="start", at=now - timedelta(minutes=11)),
                    m.DockerEvent(host_id=hid, container_name=dead, event="die", detail="exit 1",
                                  at=now - timedelta(minutes=10)),
                    m.DockerLog(host_id=hid, container_name=live, stream="stdout", line=f"serving {tag}",
                                ts=now - timedelta(minutes=5)),
                    m.DockerLog(host_id=hid, container_name=dead, stream="stderr", line=f"panic {tag}",
                                ts=now - timedelta(minutes=10)),
                ])
                s.add_all(sample(dead, 40, 200.0, 300 << 20) + sample(live, 30, 40.0, 90 << 20)
                          + sample(live, 20, 60.0, 100 << 20) + sample(live, 1, 50.0, 100 << 20))

        client = app.test_client()
        async with client.session_transaction() as sess:
            sess.update(user_id=uid, user_role="viewer", username=f"sp-{tag}", login_at=time.time())

        async def get(url):
            resp = await client.get(url)
            return resp.status_code, await resp.get_data(as_text=True)

        out = {"page": await get(f"/plugins/docker/service/{name}"),
               "focus": await get(f"/plugins/docker/service/{name}?task={dead_task}"),
               "missing": await get(f"/plugins/docker/service/it{tag}_nothing"),
               "logs": await get(f"/plugins/docker/service/{name}/logs/lines"),
               "one": await get(f"/plugins/docker/service/{name}/logs/lines?task={dead_task}"),
               "history": await get(f"/plugins/docker/service/{name}/history?range=1h"),
               "rows": await get(f"/plugins/docker/rows?q=it{tag}"),
               "other": await get(f"/plugins/docker/rows?q=it{tag}&stack=someotherstack")}
        async with app.db_sessionmaker() as s:
            async with s.begin():
                await s.delete(await s.get(Host, hid))
                await s.delete(await s.get(User, uid))
        return out

    got = asyncio.run(go())

    status, page = got["page"]
    assert status == 200
    assert "nginx:1.27" in page and "1 of 1 running" in page
    assert "task: non-zero exit (1)" in page                          # Swarm's task record
    # One table (#5310): the node reads as the Steward host on it, on the running
    # task and the failed one alike; the running task links its container.
    assert page.count(f'<a href="/hosts/{hid}">dock-{tag}</a>') == 2 and f"node-{tag}" not in page
    assert f'href="/plugins/docker/container/{hid}/{live}"' in page
    assert "in trouble" not in page and "Last failed task" not in page   # it is healthy again
    assert f'<option value="{live_task}">1 · {live_task[:10]}</option>' in page
    assert f'<option value="{dead_task}">1 · {dead_task[:10]}</option>' in page
    assert f'href="/plugins/docker/?stack=it{tag}"' in page
    assert f'<option value="{dead_task}" selected>' in got["focus"][1]
    assert got["missing"][0] == 404 and "Service not found" in got["missing"][1]

    logs = got["logs"][1]
    assert f"serving {tag}" in logs and f"panic {tag}" in logs        # the removed task's lines too
    assert logs.index(f"serving {tag}") < logs.index(f"panic {tag}")  # newest first
    assert f"1 · {dead_task[:10]}" in logs
    one = got["one"][1]
    assert f"panic {tag}" in one and f"serving {tag}" not in one

    history = got["history"][1]
    assert "<polyline" in history and "peak 2.0 cores" in history and "now · peak" in history

    rows = got["rows"][1]
    assert f'href="/plugins/docker/service/{name}"' in rows
    # A stack chip keeps the search it was clicked under.
    assert f'<a class="chip" href="/plugins/docker/?q=it{tag}&amp;stack=it{tag}"' in rows
    assert f'/plugins/docker/service/{name}?task={dead_task}#logs' in rows   # the failure's logs
    # Another stack's filter leaves it out of the table; the failures list is never filtered.
    assert f'<td class="cell-name"><a href="/plugins/docker/service/{name}"' in rows
    assert f'<td class="cell-name"><a href="/plugins/docker/service/{name}"' not in got["other"][1]