# plugins/docker/timeline.py """What Docker puts on the timeline, judged from each snapshot as it is ingested: a deploy (a container or swarm service running a new image, or a container recreated), and a crash-loop as one entry for the whole episode. A single restart or death is recorded but hidden. As on the Docker pages, a crash loop is Settings → Thresholds' number of deaths within CRASH_WINDOW: the one that reaches it opens a crash-loop entry, which each later one extends until the container has been quiet for CRASH_WINDOW. A container belongs to its swarm service when it has one, so a service's tasks, replaced on every deploy and every crash, read as one subject. """ from __future__ import annotations import logging from datetime import timedelta from sqlalchemy import func, select from steward.models.timeline import TimelineEntry from steward.timeline import catalog from steward.timeline.record import record logger = logging.getLogger(__name__) TYPES = ( ("docker.deployed", "Deployed", "change", "service", True, "A container or swarm service started on a new image, or a container was recreated."), ("docker.crash_loop", "Crash-looping", "observed", "service", True, "A container kept restarting or dying: one entry for the whole episode."), ("docker.restarted", "Container restarted or died", "observed", "service", False, "A single restart, death or out-of-memory kill."), ) CRASH_WINDOW = timedelta(hours=1) _TROUBLE = ("restart", "die", "oom") def register() -> None: for t in TYPES: catalog.register_type(*t) def unit(c: dict) -> str: """What a container is part of: its swarm service, else itself.""" return c.get("service_name") or c.get("name") def short_image(image: str | None) -> str: """An image as a person reads it: the name and tag, and a few characters of the digest when there is one.""" if not image: return "an unknown image" name, _, digest = image.partition("@") name = name.rsplit("/", 1)[-1] if digest: return f"{name} ({digest.split(':', 1)[-1][:12]})" return name def deploys(old_rows: list, latest: list[dict]) -> list[dict]: """Each unit whose containers now run an image they didn't, or a container recreated under the same name: {unit, name, image, was, recreated, swarm}. A swarm task landing on a host that ran none of its service before is the scheduler moving it, not a deploy.""" images: dict[str, set] = {} ids = {} for r in old_rows: images.setdefault(r.service_name or r.name, set()).add(r.image) ids[r.name] = (r.container_id, r.image) out, seen = [], set() for c in latest: if (c.get("status") or "").lower() != "running" or not c.get("name"): continue key, image = unit(c), c.get("image") or "" if key in seen: continue before = images.get(key) old_id, old_image = ids.get(c["name"], (None, None)) if before and image not in before: was = old_image if old_image and old_image != image else sorted(before)[0] out.append({"unit": key, "name": c["name"], "image": image, "was": was, "recreated": False, "swarm": bool(c.get("service_name"))}) elif old_id and c.get("container_id") and old_id != c["container_id"] and not c.get("service_name"): out.append({"unit": key, "name": c["name"], "image": image, "was": old_image, "recreated": True, "swarm": False}) elif before is None and not c.get("service_name"): out.append({"unit": key, "name": c["name"], "image": image, "was": None, "recreated": False, "swarm": False}) else: continue seen.add(key) return out def _deploy_summary(d: dict, host_name: str) -> str: what = f"{d['unit']} on {host_name}" if d["recreated"]: return f"{what}: recreated on {short_image(d['image'])}" if d["was"] is None: return f"{what}: started on {short_image(d['image'])}" new, old = short_image(d["image"]), short_image(d["was"]) if d["image"].split("@")[0] == (d["was"] or "").split("@")[0]: return f"{what}: new build of {new}" return f"{what}: {old} → {new}" async def _trouble(session, host, key: str, name: str, at, said: str, deaths: int) -> None: """One restart, death or OOM kill of `key`: extends its crash-loop, opens one when it is the `deaths`-th within CRASH_WINDOW, else a hidden entry.""" subject = f"{host.name}/{key}" loop = (await session.execute(select(TimelineEntry).where( TimelineEntry.type == "docker.crash_loop", TimelineEntry.subject == subject, TimelineEntry.until >= at - CRASH_WINDOW).order_by(TimelineEntry.at.desc()).limit(1)) ).scalar_one_or_none() if loop is not None: count = (loop.detail or {}).get("count", 0) + 1 loop.until = max(loop.until, at) loop.detail = {**(loop.detail or {}), "count": count, "last": said} loop.summary = f"{key} on {host.name} crash-looping: {count} restarts or deaths" return recent = (await session.execute(select(func.count(), func.min(TimelineEntry.at)).where( TimelineEntry.type == "docker.restarted", TimelineEntry.subject == subject, TimelineEntry.at >= at - CRASH_WINDOW))).one() url = f"/plugins/docker/container/{host.id}/{name}" if recent[0] + 1 >= deaths: count = recent[0] + 1 record(session, type="docker.crash_loop", at=recent[1] or at, until=at, summary=f"{key} on {host.name} crash-looping: {count} restarts or deaths", hosts=[host.name], subject=subject, actor="Docker", detail={"count": count, "last": said, "url": url}) return record(session, type="docker.restarted", at=at, summary=f"{key} on {host.name}: {said}", hosts=[host.name], subject=subject, actor="Docker", detail={"url": url}) _SAID = {"restart": "restarted", "die": "died", "oom": "killed, out of memory"} async def from_snapshot(session, host, old_rows: list, latest: list[dict], events: list, at, deaths: int) -> None: """The entries one snapshot makes: its deploys, and its restarts and deaths (`events`: (name, event, detail, when)), in a savepoint so a failure here never loses the snapshot. `deaths` makes a crash loop.""" try: async with session.begin_nested(): for d in deploys(old_rows, latest): url = "/plugins/docker/swarm" if d["swarm"] else f"/plugins/docker/container/{host.id}/{d['name']}" record(session, type="docker.deployed", at=at, summary=_deploy_summary(d, host.name), hosts=[host.name], subject=f"{host.name}/{d['unit']}", actor="Docker", detail={"image": d["image"], "was": d["was"], "url": url}) units = {c.get("name"): unit(c) for c in latest if c.get("name")} units.update({r.name: r.service_name or r.name for r in old_rows if r.name not in units}) for name, event, detail, when in events: if event in _TROUBLE: said = _SAID[event] + (f" ({detail})" if detail else "") await _trouble(session, host, units.get(name, name), name, when or at, said, deaths) await session.flush() except Exception: logger.exception("Docker on %s: timeline entries could not be written", host.name)