"""steward/core/formatting.py
How Steward writes a size, a rate, a span of time and a moment, and how it
grades a reading against Settings → Thresholds — once, for every page and
plugin (#5093). Registered as Jinja filters (|bytes, |rate, |duration, |ago,
|when) and the status_class global in app.py; Python callers import them from
here. A moment is written in the reader's own zone, on a 24-hour clock
(milestone 490); app.py hands the filters the zone saved on their account.
"""
from __future__ import annotations
from datetime import datetime, timezone, tzinfo
from functools import lru_cache
from typing import Any
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError, available_timezones
from markupsafe import Markup, escape
NONE = "—"
_UNITS = ("B", "KiB", "MiB", "GiB", "TiB", "PiB")
_SPANS = (("d", 86400), ("h", 3600), ("m", 60), ("s", 1))
def human_bytes(n: float | None) -> str:
"""Binary units: '0 B', '512 B', '1.5 KiB', '1.4 GiB'."""
if n is None:
return NONE
size = float(n)
for unit in _UNITS:
if abs(size) < 1024 or unit == _UNITS[-1]:
return f"{int(size)} B" if unit == "B" else f"{size:.1f} {unit}"
size /= 1024
raise AssertionError("unreachable")
def human_rate(bytes_per_s: float | None) -> str:
"""Bytes per second in binary units: '12.3 MiB/s'."""
return NONE if bytes_per_s is None else f"{human_bytes(bytes_per_s)}/s"
def human_duration(seconds: float | None) -> str:
"""The two largest units of a span: '45s', '8m', '1m 30s', '2h 5m', '3d 4h'.
The second unit is dropped when it is zero. Negative spans read as None."""
if seconds is None or seconds < 0:
return NONE
left = int(seconds)
if left == 0:
return "0s"
parts = []
for i, (unit, size) in enumerate(_SPANS):
count, left = divmod(left, size)
if count:
parts.append(f"{count}{unit}")
nxt = _SPANS[i + 1] if i + 1 < len(_SPANS) else None
if nxt and left // nxt[1]:
parts.append(f"{left // nxt[1]}{nxt[0]}")
break
return " ".join(parts)
def _utc(when: datetime) -> datetime:
return when.replace(tzinfo=timezone.utc) if when.tzinfo is None else when.astimezone(timezone.utc)
def seconds_since(when: datetime | None, now: datetime | None = None) -> float | None:
"""Seconds from `when` to now; a naive datetime is taken as UTC."""
if when is None:
return None
return ((now or datetime.now(timezone.utc)) - _utc(when)).total_seconds()
def zone(name: str | None) -> tzinfo:
"""The zone a reader asked for by its IANA name, or UTC when none was
given or it is not one this machine knows."""
if not name:
return timezone.utc
try:
return ZoneInfo(name)
except (ZoneInfoNotFoundError, ValueError):
return timezone.utc
@lru_cache(maxsize=1)
def zone_names() -> tuple[str, ...]:
"""Every IANA zone this machine knows, sorted: the Account page's choices,
and what a saved zone is checked against."""
return tuple(sorted(n for n in available_timezones() if "/" in n or n == "UTC"))
def zones_by_region() -> list[tuple[str, list[str]]]:
"""A zone picker's choices (the zone_select macro): UTC alone, then every
zone under its region ('America', 'Europe', …)."""
from itertools import groupby
named = [z for z in zone_names() if "/" in z]
return [("", ["UTC"])] + [(region, list(zones)) for region, zones
in groupby(named, key=lambda z: z.split("/", 1)[0])]
def tz_label(tz: tzinfo | None = None, now: datetime | None = None) -> str:
"""The zone's short name as it stands now ('EDT', 'CET', 'UTC'), for a
column header that says which zone its times are in."""
return (now or datetime.now(timezone.utc)).astimezone(tz or timezone.utc).tzname() or "UTC"
def _exact(local: datetime) -> str:
"""'Thu 8 Oct 2026 06:58:12 EDT': the full moment, for a hover."""
return f"{local:%a} {local.day} {local:%b %Y %H:%M:%S} {local.tzname()}"
def _time(when: datetime, local: datetime, text: str) -> Markup:
return Markup('{} ').format(
when.strftime("%Y-%m-%dT%H:%M:%SZ"), _exact(local), escape(text))
def ago(when: datetime | None, now: datetime | None = None, tz: tzinfo | None = None) -> Markup:
"""'3m ago' (or 'in 3m' for a future moment) in a element whose
title is the exact moment in the reader's zone, so a hover gives it."""
if when is None:
return Markup(NONE)
when = _utc(when)
secs = seconds_since(when, now)
if abs(secs) < 1:
text = "just now"
elif secs > 0:
text = f"{human_duration(secs)} ago"
else:
text = f"in {human_duration(-secs)}"
return _time(when, when.astimezone(tz or timezone.utc), text)
def when(moment: datetime | None, tz: tzinfo | None = None, now: datetime | None = None,
seconds: bool = False) -> Markup:
"""The exact moment in the reader's zone on a 24-hour clock, always with
its date: 'Thu 8 Oct 06:58' this year, '6 Oct 2025 14:12' before it. A
list that groups by recency says Today or Yesterday in a divider
(day_group), never in place of the date (#5214). `seconds` adds them
('Thu 8 Oct 06:58:12'), for log lines a second apart."""
if moment is None:
return Markup(NONE)
tz = tz or timezone.utc
moment = _utc(moment)
local = moment.astimezone(tz)
this_year = (now or datetime.now(timezone.utc)).astimezone(tz).year
day, hm = local.date(), f"{local:%H:%M:%S}" if seconds else f"{local:%H:%M}"
if day.year == this_year:
text = f"{local:%a} {day.day} {local:%b} {hm}"
else:
text = f"{day.day} {local:%b} {day.year} {hm}"
return _time(moment, local, text)
DAY_GROUPS = ("Today", "Yesterday", "Last 7 days", "Older")
def day_group(moment: datetime, tz: tzinfo | None = None, now: datetime | None = None) -> str:
"""Which of DAY_GROUPS a past moment falls in, by calendar day in the
reader's zone: the dividers of a newest-first log."""
tz = tz or timezone.utc
day = _utc(moment).astimezone(tz).date()
back = ((now or datetime.now(timezone.utc)).astimezone(tz).date() - day).days
if back <= 0:
return DAY_GROUPS[0]
if back == 1:
return DAY_GROUPS[1]
return DAY_GROUPS[2] if back < 7 else DAY_GROUPS[3]
def status_class(value: Any, kind: str, thresholds: dict) -> str:
"""Grade a reading against Settings → Thresholds (to_thresholds_cfg):
'ok', 'warn' or 'crit', or 'unknown' when there is no reading.
A kind the thresholds don't name is a mistake in the caller, not a
reading to colour as healthy, so it raises."""
if value is None:
return "unknown"
th = thresholds[kind]
warn, crit = th["warn"], th["crit"]
if th.get("dir", "high") == "low":
return "crit" if value < crit else "warn" if value < warn else "ok"
return "crit" if value >= crit else "warn" if value >= warn else "ok"
# Every state a page shows, and the tone it is drawn in: ok (green), warn
# (amber), crit (red), info (blue) or dim. The status_badge and status_dot
# macros read this, so one state looks the same on every page.
STATES: dict[str, tuple[str, str]] = {
"up": ("ok", "Up"),
"ok": ("ok", "OK"),
"resolved": ("ok", "Resolved"),
"down": ("crit", "Down"),
"firing": ("crit", "Firing"),
"crit": ("crit", "Critical"),
"warn": ("warn", "Warning"),
"pending": ("warn", "Pending"),
"degraded": ("warn", "Degraded"),
"stale": ("warn", "Stale"),
"no_data": ("warn", "No data"),
"acknowledged": ("info", "Acknowledged"),
"data_resumed": ("ok", "Reporting again"),
"test": ("info", "Test"),
"inactive": ("dim", "Inactive"),
"paused": ("dim", "Paused"),
"unknown": ("dim", "Unknown"),
# An Ansible run (AnsibleRunStatus): working is blue, not amber, so a run
# in progress doesn't read as a problem.
"queued": ("dim", "Queued"),
"running": ("info", "Running"),
"success": ("ok", "Succeeded"),
"failed": ("crit", "Failed"),
"interrupted": ("warn", "Interrupted"),
"cancelled": ("dim", "Cancelled"),
# A metric anomaly (#5740), and where its review stands.
"anomaly": ("warn", "Metric anomaly"),
"to_review": ("warn", "To review"),
"normal": ("ok", "Marked normal"),
"real": ("crit", "Real problem"),
"lapsed": ("dim", "Not reviewed"),
}
def status_info(state: Any) -> tuple[str, str]:
"""(tone, label) for a state name or enum. A state the map doesn't know
is drawn dim under its own name rather than breaking the page."""
key = str(getattr(state, "value", state) or "unknown").lower()
return STATES.get(key, ("dim", key.replace("_", " ").capitalize()))
def required(value: Any, what: str) -> Any:
"""For a macro argument that must be given: raise if it is empty."""
if value is None or not str(value).strip():
raise ValueError(f"{what} is required")
return value
_SEVERITY = {"crit": 2, "warn": 1}
def worst_reading(readings, thresholds: dict, normals: dict | None = None):
"""The reading furthest past its cutoff, as (label, value, grade), or None
when every reading is within its cutoffs or missing.
`readings` is (label, value, kind) triples, graded like status_class; the
first of equally bad readings wins, so list them in the order a reader
should hear about them. `normals` ({kind: (Norm, floor)}) are the
resource's own normals for this hour: a reading past its cutoff but
within its normal is not past it (#5742)."""
from steward.core.anomalies import graded
normals = normals or {}
worst, rank = None, 0
for label, value, kind in readings:
if kind in normals:
grade = graded(value, kind, thresholds, *normals[kind], "%")[0]
else:
grade = status_class(value, kind, thresholds)
if _SEVERITY.get(grade, 0) > rank:
worst, rank = (label, value, grade), _SEVERITY[grade]
return worst
def metric_value(name: str, value: Any) -> str:
"""A metric reading in its unit, from the name's suffix: _pct → 94%,
_ms → 120 ms, _bytes → 1.2 GiB, _bps → its rate, _secs / seconds_ → a
span (Prometheus's _seconds too), _per_sec → 1.5/s, _c → 71°C, _days →
12 days; is_up / is_healthy read as yes or no.
None (no reading) is "—"."""
if value is None:
return "—"
n = name or ""
if n in ("is_up", "is_healthy"):
return {"is_up": ("up", "down"), "is_healthy": ("healthy", "unhealthy")}[n][0 if value >= 0.5 else 1]
if n.endswith("_pct") or "_pct_" in n:
return f"{value:.1f}%" if abs(value) < 10 else f"{value:.0f}%"
if n.endswith("_ms"):
return f"{value:.0f} ms"
if n.endswith("_bytes"):
return human_bytes(value)
if n.endswith(("_bps", "bytes_rate")):
return human_rate(value)
if n.endswith("_per_sec"):
return f"{value:.3g}/s"
if n.endswith("_secs") or n.startswith("seconds_") or (
n.endswith("_seconds") and not n.endswith("_timestamp_seconds")):
return human_duration(value)
if n.endswith("_c") or n.startswith("temp_c"): # temp_c_max too
return f"{value:.0f}°C"
if n.endswith("_days"):
return f"{value:.0f} days"
return f"{value:g}"
# What each alertable metric is called on a page, and what it measures. The
# key is the name a source records; a name not listed reads as itself with
# its underscores spaced (SNMP's OID labels are the operator's own words).
METRICS: dict[str, tuple[str, str]] = {
"is_up": ("Up or down", "whether the check passed"),
"is_healthy": ("Health check", "the container's own HEALTHCHECK result"),
"response_ms": ("Response time", "how long the check took"),
"ingest_posts": ("Agent posts per minute", "host-agent reports Steward took in the last minute"),
"ingest_failed": ("Failed agent posts", "reports in the last minute that didn't land"),
"ingest_slowest_ms": ("Slowest agent post", "the longest report in the last minute; agents give up at 10 s"),
"db_roundtrip_ms": ("Database round trip", "a trivial query to Steward's database, connection included"),
"unwatched_secs": ("Time not watching", "how long Steward wasn't recording before this minute; 0 when unbroken"),
"restart_gap_secs": ("Restarted after", "recorded when Steward starts: how long since it last recorded"),
"cleanup_secs": ("Hourly cleanup took", "how long the last retention and rollup run took"),
"backup_age_secs": ("Newest backup's age", "time since Steward last sent a backup to its bucket"),
"backup_bytes": ("Backup size", "the size of the last backup sent"),
"open_fds": ("Open files", "files Steward itself has open"),
"open_fds_pct": ("Open files (% of limit)", "files Steward has open, against its limit"),
"scrape_ms": ("Scrape time", "how long the target took to hand over its metrics"),
"series_kept": ("Series stored", "series from the target's last scrape that were stored"),
"series_capped": ("Series over the cap", "series the last scrape left out: past the target's series cap"),
"request_rate": ("Requests per second", "requests through this router"),
"error_rate_4xx_pct": ("4xx responses", "share of responses that were client errors"),
"error_rate_5xx_pct": ("5xx responses", "share of responses that were server errors"),
"latency_p50_ms": ("Response time (median)", "half of requests were faster than this"),
"latency_p95_ms": ("Response time (95th percentile)", "95% of requests were faster than this"),
"latency_p99_ms": ("Response time (99th percentile)", "99% of requests were faster than this"),
"response_bytes_rate": ("Response throughput", "bytes per second sent back"),
"cert_expiry_days": ("Certificate expires in", "days until the TLS certificate expires"),
"latency_ms": ("WAN latency", "round trip from the gateway"),
"total_clients": ("Connected clients", "devices on the network"),
"cpu_pct": ("CPU", "CPU in use; for a container, 100% is one core"),
"mem_used_bytes": ("Memory used", "what a container's processes hold, in bytes; file cache not counted"),
"mem_cache_bytes": ("File cache", "files a container read or wrote, kept in memory until the host needs it back"),
"restart_count": ("Restarts", "times the container has restarted, in total"),
"disk_total_bytes": ("Docker disk use", "everything Docker keeps on the host (resource: the host)"),
"disk_images_bytes": ("Docker images", "space Docker's images take on the host"),
"disk_containers_bytes": ("Container files", "what containers wrote on top of their images"),
"disk_volumes_bytes": ("Docker volumes", "space Docker's volumes take on the host"),
"disk_build_cache_bytes": ("Docker build cache", "left by docker build; safe to clear"),
"disk_reclaimable_bytes": ("Reclaimable by Docker", "held by images no container uses"),
"mem_used_pct": ("Memory used", "memory in use, file cache not counted; a container's is of its limit, or of the host's RAM when it has none"),
"mem_available_bytes": ("Memory available", "memory free for new work"),
"swap_used_bytes": ("Swap used", "memory paged out to disk"),
"disk_used_pct_worst": ("Fullest disk", "the most-full mount on the host, by space or inodes"),
"disk_used_pct": ("Disk used", "one mount (resource host:mount)"),
"inodes_used_pct": ("Inodes used", "one mount's file slots in use (resource host:mount); at 100% no file can be created"),
"load_1m": ("Load (1 min)", "runnable processes, averaged over a minute"),
"load_5m": ("Load (5 min)", "runnable processes, averaged over 5 minutes"),
"load_15m": ("Load (15 min)", "runnable processes, averaged over 15 minutes"),
"load_pct": ("Load per core", "1-minute load as a % of the host's cores; 100% keeps every core busy"),
"uptime_secs": ("Uptime", "time since the host booted"),
"seconds_since_seen": ("Time since the agent reported", "over 3 minutes means the agent is offline"),
"clock_off_secs": ("Clock off by", "how far the host's clock is from Steward's, either way"),
"mounts_stalled": ("Mounts not answering", "mounts the agent couldn't read in 5 s, e.g. an NFS share whose server stopped"),
"nfs_exe_max_ms": ("NFS delay (slowest mount)", "the host's slowest NFS mount: how long a request took, queueing included"),
"nfs_exe_ms": ("NFS delay", "one NFS mount (resource host:mount): how long a request took on average, queueing included"),
"nfs_rtt_ms": ("NFS round trip", "one NFS mount: how long the server took to answer a request"),
"nfs_read_ms": ("NFS read delay", "one NFS mount: how long a read took on average"),
"nfs_write_ms": ("NFS write delay", "one NFS mount: how long a write took on average"),
"nfs_ops_per_sec": ("NFS requests", "one NFS mount: requests finished a second"),
"nfs_tx_bps": ("NFS sent", "one NFS mount: bytes a second sent to its server, writes included"),
"nfs_rx_bps": ("NFS received", "one NFS mount: bytes a second received from its server, reads included"),
"nfs_retrans": ("NFS retransmissions", "one NFS mount: requests sent again since the last sample"),
"nfs_timeouts": ("NFS timeouts", "one NFS mount: requests that timed out since the last sample"),
"agent_rss_bytes": ("Agent memory", "the host agent's own resident memory; tens of MB is normal"),
"agent_cpu_pct": ("Agent CPU", "the host agent's own CPU, % of one core; a few % is normal"),
"updates_pending": ("Updates waiting", "packages the host's package manager would upgrade, checked hourly"),
"updates_security": ("Security updates waiting", "of the updates waiting, those from a security source"),
"reboot_required": ("Reboot required", "1 when the host needs a reboot to finish an update, 0 when not"),
"maintenance_failed": ("Maintenance failed", "1 when the host's latest scheduled or manual update failed or timed out, or the host didn't come back into service in time and hasn't been back since"),
"temp_c_max": ("Hottest sensor", "the host's hottest real thermometer; inputs with nothing wired to them are left out"),
"temp_c": ("Sensor temperature", "one temperature sensor (resource host:temp:label)"),
"net_rx_bps": ("Network in", "bytes per second received, by the whole host, one interface or one container"),
"net_tx_bps": ("Network out", "bytes per second sent, by the whole host, one interface or one container"),
"disk_read_bps": ("Disk reads", "bytes per second read, from all disks or one"),
"disk_write_bps": ("Disk writes", "bytes per second written, to all disks or one"),
"drives_failing": ("Drives failing", "drives on the host failing their own health check, raising a critical warning or out of spare blocks"),
"drive_failing": ("Drive failing", "1 when this drive fails its own health check, raises a critical warning or is out of spare blocks"),
"drive_life_used_max_pct": ("Most worn drive", "the most life any of the host's SSDs has used of its rating"),
"drive_life_used_pct": ("Drive life used", "how much of its rated write life this drive has used"),
"drive_bad_sectors": ("Bad sectors", "media errors, or reallocated, pending and uncorrectable sectors, as the drive counts them"),
"drive_temp_c": ("Drive temperature", "the drive's own temperature sensor"),
}
# Where a rule's metric comes from, as the picker and the rules list say it.
SOURCES: dict[str, str] = {
"icmp": "Ping monitors", "tcp": "TCP monitors", "dns": "DNS monitors",
"http": "HTTP monitors", "steward": "Steward itself", "host_agent": "Host agent",
"docker": "Docker containers", "traefik": "Traefik routers", "unifi": "UniFi",
"snmp": "SNMP devices", "prometheus": "Prometheus targets", "syslog": "Syslog patterns",
}
def cert_left(days: float | None) -> str:
"""Time left on a certificate: "12 days", "1 day", "today", or "expired
3 days ago" rather than "-3 days"."""
if days is None:
return NONE
whole = int(days)
if days < 0:
gone = int(-days)
return "expired today" if gone == 0 else f"expired {gone} day{'s' if gone != 1 else ''} ago"
if whole == 0:
return "expires today"
return f"{whole} day{'s' if whole != 1 else ''}"
# Words a settings key spells in lower case that read in capitals.
_ACRONYMS = {"url": "URL", "urls": "URLs", "ssl": "SSL", "tls": "TLS", "ip": "IP", "api": "API",
"dns": "DNS", "http": "HTTP", "https": "HTTPS", "snmp": "SNMP", "oid": "OID",
"oids": "OIDs", "id": "ID", "geoip": "GeoIP", "db": "database", "mmdb": "MMDB",
"wan": "WAN", "dpi": "DPI", "smtp": "SMTP"}
def field_label(key: str) -> str:
"""A config key as a form label: "metrics_url" → "Metrics URL",
"scrape_interval_seconds" → "Scrape interval seconds"."""
words = [_ACRONYMS.get(w.lower(), w) for w in str(key or "").replace("-", "_").split("_") if w]
if not words:
return ""
words[0] = words[0][:1].upper() + words[0][1:]
return " ".join(words)
def metric_label(name: str) -> str:
entry = METRICS.get(name or "")
return entry[0] if entry else (name or "").replace("_", " ")
def metric_about(name: str) -> str:
entry = METRICS.get(name or "")
return entry[1] if entry else ""
def source_label(source: str) -> str:
return SOURCES.get(source or "", (source or "").replace("_", " "))
def metric_unit(name: str) -> str:
"""The unit a threshold for this metric is written in, from the same
suffixes metric_value reads; "" for a count or a plain number."""
n = name or ""
if n in ("is_up", "is_healthy"):
return ""
if n.endswith("_pct") or "_pct_" in n:
return "%"
if n.endswith("_ms"):
return "ms"
if n.endswith("_bytes"):
return "bytes"
if n.endswith(("_bps", "bytes_rate")):
return "bytes/s"
if n.endswith("_secs") or n.startswith("seconds_"):
return "seconds"
if n.endswith("_c") or n.startswith("temp_c"): # temp_c_max too
return "°C"
if n.endswith("_days"):
return "days"
if n == "request_rate":
return "req/s"
return ""