feat(widgets): Routers and Hosts problem rows say since when (#5810)

Routers: a router in trouble reads 'started 2h ago' or 'all of the last
24 h', from problem_since over the day's history, read for the troubled
routers only (a certificate's problem carries its own date). Hosts: a
down host reads 'a monitor is down for 3h'; host_glance keeps the
down_since host_status already had.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/routes.py

@@ -496,10 +496,21 @@ async def widget():
             r["cert"]["grade"] = grade
             if _GRADE_ORDER.get(grade, 2) < _GRADE_ORDER[r["state"]]:
                 left = cert_left(r["cert"]["days"])     # "12 days", "expires today", "expired …"
+                r["by_cert"] = True
                 r["state"], r["why"] = grade, (
                     f"certificate expires in {left}" if left[0].isdigit() else f"certificate {left}")
     problems, healthy = split_routers(current)
     healthy.sort(key=lambda r: -(r["latest"].request_rate or 0))
+    # Since when, from the day's history, for problems the traffic shows (a
+    # certificate's says its own date). Only the routers in trouble are read.
+    traffic = [r for r in problems if not r.get("by_cert")]
+    if traffic:
+        since = now - WIDGET_SINCE
+        async with current_app.db_sessionmaker() as db:
+            history = await router_history(db, [r["name"] for r in traffic], since,
+                                           bucket_seconds(since))
+        for r in traffic:
+            r["since"], r["since_before"] = problem_since(history[r["name"]], thresholds)
     # With no router domains to go on, certificates can't sit on their
     # routers: the expiring ones are listed after the routers instead.
     loose = [] if domains is not None else sorted(
@@ -516,6 +527,8 @@ async def widget():
 
 # Healthy routers on the dashboard widget (the busiest); the page lists all.
 ROUTERS_SHOWN = 6
+# How far back the widget looks for when a router's problem began.
+WIDGET_SINCE = timedelta(hours=24)
 
 
 @traefik_bp.get("/widget/access_log")

plugins/traefik/templates/traefik/widget.html

@@ -24,7 +24,8 @@
     <td class="w-name" title="{{ r.name }}">
       {{- status_dot(r.state) -}}
       {{ r.name.split('@')[0] }}
-      {%- if r.why and r.state != "ok" %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}</span>{% endif %}
+      {%- if r.why and r.state != "ok" %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}
+        {%- if r.since_before %}, all of the last 24 h{% elif r.since %}, started {{ r.since|ago }}{% endif %}</span>{% endif %}
     </td>
     <td>{% if m.request_rate is none %}—{% else %}{{ "%.2f"|format(m.request_rate) }}{% endif %}</td>
     <td class="w-wide-only">{% if m.response_bytes_rate is none %}—{% else %}{{ m.response_bytes_rate|rate }}{% endif %}</td>

steward/hosts/service.py

@@ -373,7 +373,7 @@ GLANCE_ORDER = {"down": 0, "crit": 1, "stale": 2, "warn": 3, "up": 4, "unknown":
 def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict]:
     """The Hosts — Overview rows, worst first (#5251).
 
-    Each is {host, state, worst, agent}. State is "down" when a linked monitor
+    Each is {host, state, worst, agent, down_since}. State is "down" when a linked monitor
     is down, "stale" when the agent has stopped reporting, else the grade of
     the reading furthest past its cutoff ("crit" / "warn": CPU, memory, disk /
     or load per core, against Settings → Thresholds; CPU, memory and load
@@ -406,5 +406,6 @@ def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict
             state = "up"
         else:
             state = "unknown"
-        rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}, "grades": grades})
+        rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}, "grades": grades,
+                     "down_since": st.get("down_since") if state == "down" else None})
     return sorted(rows, key=lambda r: (GLANCE_ORDER[r["state"]], r["host"].name.lower()))

steward/templates/hosts/overview_widget.html

@@ -27,7 +27,7 @@
     <td class="w-name" title="{{ r.host.name }}">
       {{- status_dot(r.state) -}}
       {% if session.user_id %}<a href="/hosts/{{ r.host.id }}">{{ r.host.name }}</a>{% else %}{{ r.host.name }}{% endif %}
-      {%- if r.state == 'down' %}<span class="w-sub tone-crit">a monitor is down</span>
+      {%- if r.state == 'down' %}<span class="w-sub tone-crit">a monitor is down{% if r.down_since %} for {{ r.down_since|ago|replace(' ago', '') }}{% endif %}</span>
       {%- elif r.state == 'stale' %}<span class="w-sub tone-warn">agent last reported {{ a.last_at|ago }}; figures are from then</span>
       {%- elif not a %}<span class="w-sub">no agent</span>{% endif %}
     </td>

tests/core/test_dashboard_defaults.py

@@ -120,7 +120,8 @@ def test_hosts_sort_worst_first():
     th = to_thresholds_cfg({})
     hosts = [_host("a", "alpha"), _host("b", "busy"), _host("d", "dead"), _host("q", "quiet"),
              _host("w", "warm"), _host("n", "nothing")]
-    status = {"d": {"state": "down"}, "a": {"state": "up"}}
+    went = datetime(2026, 10, 15, 9, 0, tzinfo=timezone.utc)
+    status = {"d": {"state": "down", "down_since": went}, "a": {"state": "up", "down_since": None}}
     agent = {"alpha": {"cpu_pct": 10.0, "fresh": True}, "busy": {"cpu_pct": 99.0, "fresh": True},
              "quiet": {"cpu_pct": 99.0, "fresh": False},
              "warm": {"cpu_pct": 10.0, "load_per_core": 85, "fresh": True}}
@@ -130,6 +131,17 @@ def test_hosts_sort_worst_first():
         ("alpha", "up"), ("nothing", "unknown")]
     assert rows[1]["worst"][0] == "CPU" and rows[3]["worst"][0] == "Load"
     assert rows[2]["worst"] is None          # a quiet agent's old figures aren't graded
+    assert rows[0]["down_since"] == went and rows[4]["down_since"] is None   # #5810
+
+
+async def test_a_down_host_says_for_how_long(app):
+    """#5810: the problem row carries its since, as the Hosts list does."""
+    rows = [{"host": _host("d", "dead"), "state": "down", "worst": None, "agent": {},
+             "down_since": datetime.now(timezone.utc) - timedelta(hours=3)}]
+    async with app.test_request_context("/hosts/overview/widget"):
+        page = await render_template("hosts/overview_widget.html", rows=rows)
+    page = " ".join(page.split())
+    assert "a monitor is down for" in page and ">3h<" in page and " ago" not in page
 
 
 async def test_the_hosts_widget_gives_a_verdict_and_load_against_cores(app):

tests/plugins/traefik/test_traefik_page.py

@@ -209,6 +209,21 @@ async def test_the_widget_gives_a_verdict_then_routers_under_one_header(app):
     assert "+4 more healthy" in page
 
 
+async def test_a_router_in_trouble_says_since_when_on_the_widget(app):
+    """#5810: the widget's problem rows carry the since the page shows."""
+    slow = _router("api@docker", _latest(p95=450.0))
+    slow.update(since=datetime.now(timezone.utc) - timedelta(hours=2), since_before=False)
+    failing = _router("pay@docker", _latest(e5=4.0))
+    failing.update(since=datetime.now(timezone.utc) - timedelta(days=1), since_before=True)
+    async with app.test_request_context("/plugins/traefik/widget"):
+        page = await render_plugin_template(
+            app, "traefik", "traefik/widget.html", problems=[failing, slow], healthy=[],
+            healthy_more=0, total=2, loose_certs=[], domains_known=True)
+    page = " ".join(page.split())
+    assert "all of the last 24 h" in page
+    assert page.count(", started ") == 1 and page.index("all of the last 24 h") < page.index(", started ")
+
+
 async def test_without_router_domains_expiring_certs_follow_the_routers(app):
     fine = _router("web@docker", _latest())
     async with app.test_request_context("/plugins/traefik/widget"):