feat(widgets): a troubled container on the Docker widget says since when (#5810)

Its problems and since come from the same container_problems and
problem_since the Docker page uses, read from the day's deaths and
health changes; troubled containers sort worst problem first, then name.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/ingest.py

@@ -147,12 +147,36 @@ async def _persist_logs(session, host, log_batches) -> None:
     if not await get_setting(session, "docker.logs.enabled"):
         return
     exclude = set(await get_setting(session, "docker.logs.exclude") or ())
-    for row in _log_rows(log_batches, host.id):
-        if row["container_name"] in exclude:
+    rows = [r for r in _log_rows(log_batches, host.id) if r["container_name"] not in exclude]
+    stored = await _stored_lines(session, host.id, rows)
+    for row in rows:
+        if (row["container_name"], row["ts"], row["stream"], row["line"]) in stored:
             continue
         session.add(DockerLog(**row))
 
 
+async def _stored_lines(session, host_id: str, rows: list) -> set:
+    """The lines of `rows` this host already has, as (container, ts, stream,
+    line). A restarted agent has lost its cursors and re-sends each
+    container's last lines (#5809); a push of new lines overlaps nothing, so
+    the one query, bounded per container to the batch's own span, finds none."""
+    from sqlalchemy import and_, or_, select
+    from .models import DockerLog
+
+    spans: dict[str, list] = {}
+    for r in rows:
+        span = spans.setdefault(r["container_name"], [r["ts"], r["ts"]])
+        span[0], span[1] = min(span[0], r["ts"]), max(span[1], r["ts"])
+    if not spans:
+        return set()
+    found = await session.execute(
+        select(DockerLog.container_name, DockerLog.ts, DockerLog.stream, DockerLog.line)
+        .where(DockerLog.host_id == host_id, or_(*(
+            and_(DockerLog.container_name == name, DockerLog.ts.between(lo, hi))
+            for name, (lo, hi) in spans.items()))))
+    return {tuple(r) for r in found}
+
+
 async def _persist_swarm(session, host, swarm: dict) -> None:
     """Upsert this manager's swarm topology; drop rows no longer reported.
 

plugins/docker/routes.py

@@ -237,6 +237,10 @@ async def widget():
             {c.host_id for c in all_containers}
             | {n.host_id for n in nodes} | {s.host_id for s in services},
         )
+        deaths = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
+        sick = await docker_svc.unhealthy_since(
+            db, [(c.host_id, c.name) for c in all_containers
+                 if not c.service_name and (c.health or "") == "unhealthy"])
 
     # Swarm tasks collapse into per-service rows (each replica tagged with its
     # host); non-swarm containers stay grouped by host.
@@ -260,7 +264,19 @@ async def widget():
 
     # The widget-shape order (UI & Design ruling): what is wrong first, then
     # the healthy rest one line each, collapsing past HEALTHY_SHOWN.
-    bad_containers = sorted((c for c in standalone if troubled(c)), key=lambda c: c.name)
+    # Worst problem first, each with since when, as the Docker page has them.
+    hour_ago, crash = now - timedelta(hours=1), docker_svc.crash_loop_deaths(current_app.config)
+    trouble = {}
+    for c in standalone:
+        if troubled(c):
+            died = deaths.get((c.host_id, c.name), [])
+            problems = docker_svc.container_problems(
+                c, sum(1 for e in died if e.at >= hour_ago), bool(died), crash)
+            trouble[(c.host_id, c.name)] = (
+                docker_svc.PROBLEMS.index(problems[0]) if problems else len(docker_svc.PROBLEMS),
+                docker_svc.problem_since(problems, died, hour_ago, sick.get((c.host_id, c.name))))
+    bad_containers = sorted((c for c in standalone if troubled(c)),
+                            key=lambda c: (trouble[(c.host_id, c.name)][0], c.name))
     healthy = ([{"kind": "service", "s": s} for s in swarm_services if s["state"] == "healthy"]
                + [{"kind": "container", "c": c} for c in sorted(running, key=lambda c: c.name)
                   if not troubled(c)]
@@ -271,6 +287,7 @@ async def widget():
         "docker/widget.html",
         bad_services=[s for s in swarm_services if s["state"] != "healthy"],
         bad_containers=bad_containers,
+        bad_since={key: since for key, (_rank, since) in trouble.items()},
         healthy=healthy[:HEALTHY_SHOWN], healthy_more=max(0, len(healthy) - HEALTHY_SHOWN),
         services_total=len(swarm_services),
         services_healthy=sum(s["state"] == "healthy" for s in swarm_services),

plugins/docker/templates/docker/widget.html

@@ -35,7 +35,8 @@
   <span class="w-label" title="{{ c.name }}">
     {%- if session.user_id %}<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>{% else %}{{ c.name }}{% endif %}
     <span class="w-sub tone-crit">
-      {%- if c.health == 'unhealthy' %}unhealthy{% elif c.status == 'restarting' %}restarting{% else %}exited {{ c.exit_code }}{% if c.oom_killed %}, out of memory{% endif %}{% endif %} · {{ host_name.get(c.host_id, c.host_id) }}
+      {%- if c.health == 'unhealthy' %}unhealthy{% elif c.status == 'restarting' %}restarting{% else %}exited {{ c.exit_code }}{% if c.oom_killed %}, out of memory{% endif %}{% endif %}
+      {%- set since = (bad_since or {}).get((c.host_id, c.name)) %}{% if since %} since {{ since|ago }}{% endif %} · {{ host_name.get(c.host_id, c.host_id) }}
     </span>
   </span>
 </div>

plugins/traefik/routes.py

@@ -496,10 +496,21 @@ async def widget():
             r["cert"]["grade"] = grade
             if _GRADE_ORDER.get(grade, 2) < _GRADE_ORDER[r["state"]]:
                 left = cert_left(r["cert"]["days"])     # "12 days", "expires today", "expired …"
+                r["by_cert"] = True
                 r["state"], r["why"] = grade, (
                     f"certificate expires in {left}" if left[0].isdigit() else f"certificate {left}")
     problems, healthy = split_routers(current)
     healthy.sort(key=lambda r: -(r["latest"].request_rate or 0))
+    # Since when, from the day's history, for problems the traffic shows (a
+    # certificate's says its own date). Only the routers in trouble are read.
+    traffic = [r for r in problems if not r.get("by_cert")]
+    if traffic:
+        since = now - WIDGET_SINCE
+        async with current_app.db_sessionmaker() as db:
+            history = await router_history(db, [r["name"] for r in traffic], since,
+                                           bucket_seconds(since))
+        for r in traffic:
+            r["since"], r["since_before"] = problem_since(history[r["name"]], thresholds)
     # With no router domains to go on, certificates can't sit on their
     # routers: the expiring ones are listed after the routers instead.
     loose = [] if domains is not None else sorted(
@@ -516,6 +527,8 @@ async def widget():
 
 # Healthy routers on the dashboard widget (the busiest); the page lists all.
 ROUTERS_SHOWN = 6
+# How far back the widget looks for when a router's problem began.
+WIDGET_SINCE = timedelta(hours=24)
 
 
 @traefik_bp.get("/widget/access_log")

plugins/traefik/templates/traefik/widget.html

@@ -24,7 +24,8 @@
     <td class="w-name" title="{{ r.name }}">
       {{- status_dot(r.state) -}}
       {{ r.name.split('@')[0] }}
-      {%- if r.why and r.state != "ok" %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}</span>{% endif %}
+      {%- if r.why and r.state != "ok" %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}
+        {%- if r.since_before %}, all of the last 24 h{% elif r.since %}, started {{ r.since|ago }}{% endif %}</span>{% endif %}
     </td>
     <td>{% if m.request_rate is none %}—{% else %}{{ "%.2f"|format(m.request_rate) }}{% endif %}</td>
     <td class="w-wide-only">{% if m.response_bytes_rate is none %}—{% else %}{{ m.response_bytes_rate|rate }}{% endif %}</td>

steward/hosts/service.py

@@ -373,7 +373,7 @@ GLANCE_ORDER = {"down": 0, "crit": 1, "stale": 2, "warn": 3, "up": 4, "unknown":
 def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict]:
     """The Hosts — Overview rows, worst first (#5251).
 
-    Each is {host, state, worst, agent}. State is "down" when a linked monitor
+    Each is {host, state, worst, agent, down_since}. State is "down" when a linked monitor
     is down, "stale" when the agent has stopped reporting, else the grade of
     the reading furthest past its cutoff ("crit" / "warn": CPU, memory, disk /
     or load per core, against Settings → Thresholds; CPU, memory and load
@@ -406,5 +406,6 @@ def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict
             state = "up"
         else:
             state = "unknown"
-        rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}, "grades": grades})
+        rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}, "grades": grades,
+                     "down_since": st.get("down_since") if state == "down" else None})
     return sorted(rows, key=lambda r: (GLANCE_ORDER[r["state"]], r["host"].name.lower()))

steward/templates/hosts/overview_widget.html

@@ -27,7 +27,7 @@
     <td class="w-name" title="{{ r.host.name }}">
       {{- status_dot(r.state) -}}
       {% if session.user_id %}<a href="/hosts/{{ r.host.id }}">{{ r.host.name }}</a>{% else %}{{ r.host.name }}{% endif %}
-      {%- if r.state == 'down' %}<span class="w-sub tone-crit">a monitor is down</span>
+      {%- if r.state == 'down' %}<span class="w-sub tone-crit">a monitor is down{% if r.down_since %} for {{ r.down_since|ago|replace(' ago', '') }}{% endif %}</span>
       {%- elif r.state == 'stale' %}<span class="w-sub tone-warn">agent last reported {{ a.last_at|ago }}; figures are from then</span>
       {%- elif not a %}<span class="w-sub">no agent</span>{% endif %}
     </td>

tests/core/test_dashboard_defaults.py

@@ -120,7 +120,8 @@ def test_hosts_sort_worst_first():
     th = to_thresholds_cfg({})
     hosts = [_host("a", "alpha"), _host("b", "busy"), _host("d", "dead"), _host("q", "quiet"),
              _host("w", "warm"), _host("n", "nothing")]
-    status = {"d": {"state": "down"}, "a": {"state": "up"}}
+    went = datetime(2026, 10, 15, 9, 0, tzinfo=timezone.utc)
+    status = {"d": {"state": "down", "down_since": went}, "a": {"state": "up", "down_since": None}}
     agent = {"alpha": {"cpu_pct": 10.0, "fresh": True}, "busy": {"cpu_pct": 99.0, "fresh": True},
              "quiet": {"cpu_pct": 99.0, "fresh": False},
              "warm": {"cpu_pct": 10.0, "load_per_core": 85, "fresh": True}}
@@ -130,6 +131,17 @@ def test_hosts_sort_worst_first():
         ("alpha", "up"), ("nothing", "unknown")]
     assert rows[1]["worst"][0] == "CPU" and rows[3]["worst"][0] == "Load"
     assert rows[2]["worst"] is None          # a quiet agent's old figures aren't graded
+    assert rows[0]["down_since"] == went and rows[4]["down_since"] is None   # #5810
+
+
+async def test_a_down_host_says_for_how_long(app):
+    """#5810: the problem row carries its since, as the Hosts list does."""
+    rows = [{"host": _host("d", "dead"), "state": "down", "worst": None, "agent": {},
+             "down_since": datetime.now(timezone.utc) - timedelta(hours=3)}]
+    async with app.test_request_context("/hosts/overview/widget"):
+        page = await render_template("hosts/overview_widget.html", rows=rows)
+    page = " ".join(page.split())
+    assert "a monitor is down for" in page and ">3h<" in page and " ago" not in page
 
 
 async def test_the_hosts_widget_gives_a_verdict_and_load_against_cores(app):

tests/integration/test_docker.py

@@ -221,6 +221,55 @@ def test_persist_logs_stores_lines(app):
     ]
 
 
+@_NEEDS_DB
+def test_persist_logs_skips_lines_a_restarted_agent_sends_again(app):
+    """#5809: a restarted agent has lost its cursors and re-sends each
+    container's last lines. Those are stored once; a new line, or another line
+    at the same instant, is still kept."""
+    from types import SimpleNamespace
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+
+    persist = _persist_fn(app)
+    now = datetime.now(timezone.utc)
+
+    def line(container, at, text_, stream="stdout"):
+        return {"container": container, "stream": stream,
+                "ts": f"2026-01-01T00:00:0{at}+00:00", "line": text_}
+
+    first = [(now, [line("web", 1, "started"), line("web", 2, "listening"),
+                    line("db", 1, "ready")])]
+    again = [(now, [line("web", 1, "started"), line("web", 2, "listening"),
+                    line("web", 2, "worker 2 up"), line("web", 3, "GET /"),
+                    line("db", 1, "ready"), line("db", 1, "ready", stream="stderr")])]
+
+    hid = str(uuid.uuid4())
+    host = SimpleNamespace(id=hid, name="loghost-again")  # as below: no expiry across commits
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add(Host(id=hid, name="loghost-again", address="10.0.0.19"))
+                await s.flush()
+                await persist(s, host, [], None, None, first)
+            async with s.begin():
+                await persist(s, host, [], None, None, again)
+            rows = (await s.execute(text(
+                "SELECT container_name, stream, line FROM docker_logs "
+                "WHERE host_id = :h ORDER BY container_name, ts, stream, line"),
+                {"h": hid})).all()
+            return [tuple(r) for r in rows]
+
+    assert asyncio.run(_go()) == [
+        ("db", "stderr", "ready"),
+        ("db", "stdout", "ready"),
+        ("web", "stdout", "started"),
+        ("web", "stdout", "listening"),
+        ("web", "stdout", "worker 2 up"),
+        ("web", "stdout", "GET /"),
+    ]
+
+
 @_NEEDS_DB
 def test_persist_logs_respects_toggle_and_exclude(app):
     """Server-side controls (Settings): an excluded container's lines are dropped

tests/plugins/docker/test_docker_page.py

@@ -299,6 +299,16 @@ async def test_the_services_widget_lists_what_is_wrong_then_packs_the_healthy(ap
     assert not hex_colours(page), hex_colours(page)
 
 
+async def test_a_troubled_container_says_since_when_on_the_widget(app):
+    """#5810: a container in trouble carries its since, as the Docker page does."""
+    sick = _c("backup_cron", host_id="h1", status="running", health="unhealthy")
+    page = await _services_widget(
+        app, bad_containers=[sick],
+        bad_since={("h1", "backup_cron"): datetime.now(timezone.utc) - timedelta(minutes=40)})
+    page = " ".join(page.split())
+    assert "unhealthy since <time" in page and "40m ago" in page and "· VDNT-Docker02" in page
+
+
 async def test_the_services_widget_says_when_all_is_well(app):
     page = await _services_widget(app, services_total=2, services_healthy=2, running_count=1,
                                   healthy=[{"kind": "service", "s": _widget_svc("a")}])

tests/plugins/traefik/test_traefik_page.py

@@ -209,6 +209,21 @@ async def test_the_widget_gives_a_verdict_then_routers_under_one_header(app):
     assert "+4 more healthy" in page
 
 
+async def test_a_router_in_trouble_says_since_when_on_the_widget(app):
+    """#5810: the widget's problem rows carry the since the page shows."""
+    slow = _router("api@docker", _latest(p95=450.0))
+    slow.update(since=datetime.now(timezone.utc) - timedelta(hours=2), since_before=False)
+    failing = _router("pay@docker", _latest(e5=4.0))
+    failing.update(since=datetime.now(timezone.utc) - timedelta(days=1), since_before=True)
+    async with app.test_request_context("/plugins/traefik/widget"):
+        page = await render_plugin_template(
+            app, "traefik", "traefik/widget.html", problems=[failing, slow], healthy=[],
+            healthy_more=0, total=2, loose_certs=[], domains_known=True)
+    page = " ".join(page.split())
+    assert "all of the last 24 h" in page
+    assert page.count(", started ") == 1 and page.index("all of the last 24 h") < page.index(", started ")
+
+
 async def test_without_router_domains_expiring_certs_follow_the_routers(app):
     fine = _router("web@docker", _latest())
     async with app.test_request_context("/plugins/traefik/widget"):