FabledSteward/plugins/docker/timeline.py

# plugins/docker/timeline.py
"""What Docker puts on the timeline, judged from each snapshot as it is
ingested: a deploy (a container or swarm service running a new image, or a
container recreated), and a crash-loop as one entry for the whole episode.
A single restart or death is recorded but hidden. As on the Docker pages, a
crash loop is Settings → Thresholds' number of deaths within CRASH_WINDOW:
the one that reaches it opens a crash-loop entry, which each later one
extends until the container has been quiet for CRASH_WINDOW.

A container belongs to its swarm service when it has one, so a service's
tasks, replaced on every deploy and every crash, read as one subject.
"""
from __future__ import annotations

import logging
from datetime import timedelta

from sqlalchemy import func, select

from steward.models.timeline import TimelineEntry
from steward.timeline import catalog
from steward.timeline.record import record

logger = logging.getLogger(__name__)

TYPES = (
    ("docker.deployed", "Deployed", "change", "service", True,
     "A container or swarm service started on a new image, or a container was recreated."),
    ("docker.crash_loop", "Crash-looping", "observed", "service", True,
     "A container kept restarting or dying: one entry for the whole episode."),
    ("docker.restarted", "Container restarted or died", "observed", "service", False,
     "A single restart, death or out-of-memory kill."),
)

CRASH_WINDOW = timedelta(hours=1)
_TROUBLE = ("restart", "die", "oom")


def register() -> None:
    for t in TYPES:
        catalog.register_type(*t)


def unit(c: dict) -> str:
    """What a container is part of: its swarm service, else itself."""
    return c.get("service_name") or c.get("name")


def short_image(image: str | None) -> str:
    """An image as a person reads it: the name and tag, and a few characters
    of the digest when there is one."""
    if not image:
        return "an unknown image"
    name, _, digest = image.partition("@")
    name = name.rsplit("/", 1)[-1]
    if digest:
        return f"{name} ({digest.split(':', 1)[-1][:12]})"
    return name


def deploys(old_rows: list, latest: list[dict]) -> list[dict]:
    """Each unit whose containers now run an image they didn't, or a
    container recreated under the same name: {unit, name, image, was,
    recreated, swarm}. A swarm task landing on a host that ran none of its
    service before is the scheduler moving it, not a deploy."""
    images: dict[str, set] = {}
    ids = {}
    for r in old_rows:
        images.setdefault(r.service_name or r.name, set()).add(r.image)
        ids[r.name] = (r.container_id, r.image)
    out, seen = [], set()
    for c in latest:
        if (c.get("status") or "").lower() != "running" or not c.get("name"):
            continue
        key, image = unit(c), c.get("image") or ""
        if key in seen:
            continue
        before = images.get(key)
        old_id, old_image = ids.get(c["name"], (None, None))
        if before and image not in before:
            was = old_image if old_image and old_image != image else sorted(before)[0]
            out.append({"unit": key, "name": c["name"], "image": image, "was": was,
                        "recreated": False, "swarm": bool(c.get("service_name"))})
        elif old_id and c.get("container_id") and old_id != c["container_id"] and not c.get("service_name"):
            out.append({"unit": key, "name": c["name"], "image": image, "was": old_image,
                        "recreated": True, "swarm": False})
        elif before is None and not c.get("service_name"):
            out.append({"unit": key, "name": c["name"], "image": image, "was": None,
                        "recreated": False, "swarm": False})
        else:
            continue
        seen.add(key)
    return out


def _deploy_summary(d: dict, host_name: str) -> str:
    what = f"{d['unit']} on {host_name}"
    if d["recreated"]:
        return f"{what}: recreated on {short_image(d['image'])}"
    if d["was"] is None:
        return f"{what}: started on {short_image(d['image'])}"
    new, old = short_image(d["image"]), short_image(d["was"])
    if d["image"].split("@")[0] == (d["was"] or "").split("@")[0]:
        return f"{what}: new build of {new}"
    return f"{what}: {old} → {new}"


async def _trouble(session, host, key: str, name: str, at, said: str, deaths: int) -> None:
    """One restart, death or OOM kill of `key`: extends its crash-loop, opens
    one when it is the `deaths`-th within CRASH_WINDOW, else a hidden entry."""
    subject = f"{host.name}/{key}"
    loop = (await session.execute(select(TimelineEntry).where(
        TimelineEntry.type == "docker.crash_loop", TimelineEntry.subject == subject,
        TimelineEntry.until >= at - CRASH_WINDOW).order_by(TimelineEntry.at.desc()).limit(1))
    ).scalar_one_or_none()
    if loop is not None:
        count = (loop.detail or {}).get("count", 0) + 1
        loop.until = max(loop.until, at)
        loop.detail = {**(loop.detail or {}), "count": count, "last": said}
        loop.summary = f"{key} on {host.name} crash-looping: {count} restarts or deaths"
        return
    recent = (await session.execute(select(func.count(), func.min(TimelineEntry.at)).where(
        TimelineEntry.type == "docker.restarted", TimelineEntry.subject == subject,
        TimelineEntry.at >= at - CRASH_WINDOW))).one()
    url = f"/plugins/docker/container/{host.id}/{name}"
    if recent[0] + 1 >= deaths:
        count = recent[0] + 1
        record(session, type="docker.crash_loop", at=recent[1] or at, until=at,
               summary=f"{key} on {host.name} crash-looping: {count} restarts or deaths",
               hosts=[host.name], subject=subject, actor="Docker",
               detail={"count": count, "last": said, "url": url})
        return
    record(session, type="docker.restarted", at=at, summary=f"{key} on {host.name}: {said}",
           hosts=[host.name], subject=subject, actor="Docker", detail={"url": url})


_SAID = {"restart": "restarted", "die": "died", "oom": "killed, out of memory"}


async def from_snapshot(session, host, old_rows: list, latest: list[dict], events: list, at,
                        deaths: int) -> None:
    """The entries one snapshot makes: its deploys, and its restarts and
    deaths (`events`: (name, event, detail, when)), in a savepoint so a
    failure here never loses the snapshot. `deaths` makes a crash loop."""
    try:
        async with session.begin_nested():
            for d in deploys(old_rows, latest):
                url = "/plugins/docker/swarm" if d["swarm"] else f"/plugins/docker/container/{host.id}/{d['name']}"
                record(session, type="docker.deployed", at=at, summary=_deploy_summary(d, host.name),
                       hosts=[host.name], subject=f"{host.name}/{d['unit']}", actor="Docker",
                       detail={"image": d["image"], "was": d["was"], "url": url})
            units = {c.get("name"): unit(c) for c in latest if c.get("name")}
            units.update({r.name: r.service_name or r.name for r in old_rows if r.name not in units})
            for name, event, detail, when in events:
                if event in _TROUBLE:
                    said = _SAID[event] + (f" ({detail})" if detail else "")
                    await _trouble(session, host, units.get(name, name), name, when or at, said, deaths)
                    await session.flush()
    except Exception:
        logger.exception("Docker on %s: timeline entries could not be written", host.name)