feat(monitors): a DNS monitor can name a resolver; the host page puts Ansible above Docker (#5214)

DNS monitors take an optional Resolver, a server's IP. Set, the check asks
that server directly for A and AAAA over UDP and records how long it took
to answer; empty keeps today's lookup through the container's resolver.
"Does my local DNS still answer for public names" and "does 1.1.1.1" are
now separate monitors. No new dependency.

On a host's page the Ansible card moves above the Docker list, which can
run to dozens of containers and hid it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/mcp/tools/monitors.py

@@ -174,7 +174,9 @@ async def create_monitor(name: str, type: str, target: str, host: str = "",
       free-standing destination.
     config, by type (every key optional, defaults shown):
       tcp:  {"port": 80}
-      dns:  {"expected_ip": null}  (set to fail when the name resolves elsewhere)
+      dns:  {"expected_ip": null, "resolver": null}  (expected_ip: fail when the
+             name resolves elsewhere; resolver: a DNS server's IP to ask
+             directly, timing its answer; null asks the container's own)
       http: {"method": "GET" | "HEAD" | "POST", "expected_status": 200,
              "content_match": "", "headers": {}, "timeout_seconds": 10 (1-60),
              "follow_redirects": true, "verify_ssl": true}

steward/models/monitors.py

@@ -53,7 +53,7 @@ class Monitor(Base):
     # Type-specific config as a JSON object. Keys by type:
     #   icmp -> {}
     #   tcp  -> {"port": int}
-    #   dns  -> {"expected_ip": str | None}
+    #   dns  -> {"expected_ip": str | None, "resolver": str | None (a server IP; None = the container's)}
     #   http -> {"method", "expected_status", "content_match", "headers",
     #            "timeout_seconds", "follow_redirects", "verify_ssl"}
     config_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")

steward/monitors/dns.py

@@ -1,30 +1,53 @@
-"""DNS resolution check — pure probe returning a result dict."""
+"""DNS resolution check — pure probe returning a result dict.
+
+With no resolver, the name is looked up the way the container looks names
+up (getaddrinfo: /etc/hosts, then the resolvers in resolv.conf). With one,
+an A and an AAAA query go straight to that server over UDP, and the time
+it took to answer is the check's response time — so "does my local DNS
+still answer for public names" and "does 1.1.1.1" are separate monitors.
+"""
 from __future__ import annotations
 import asyncio
+import ipaddress
 import logging
+import os
 import socket
+import struct
+import time
 
 logger = logging.getLogger(__name__)
 
+QUERY_TIMEOUT_S = 5.0
+_RCODES = {1: "FORMERR", 2: "SERVFAIL", 3: "NXDOMAIN", 4: "NOTIMP", 5: "REFUSED"}
+_A, _AAAA = 1, 28
 
-async def dns_check(address: str, expected_ip: str | None = None) -> dict:
-    """Resolve `address`. Returns {is_up, resolved_ip, error_msg}.
+
+async def dns_check(address: str, expected_ip: str | None = None,
+                    resolver: str | None = None) -> dict:
+    """Resolve `address`. Returns {is_up, resolved_ip, error_msg} and, when a
+    resolver is given, response_ms.
 
     Up = resolution succeeded AND (no expected_ip, or at least one returned
     record matches expected_ip). resolved_ip is the first A/AAAA record.
     """
-    result = await _resolve(address)
+    if resolver:
+        result = await _query_server(address, resolver)
+    else:
+        result = await _resolve(address)
+    out = {"response_ms": result["ms"]} if result.get("ms") is not None else {}
     if not result["resolved"]:
-        return {"is_up": False, "resolved_ip": None, "error_msg": "Resolution failed"}
+        return {**out, "is_up": False, "resolved_ip": None,
+                "error_msg": result.get("error") or "Resolution failed"}
 
     resolved_ip = result["ip"]
     if expected_ip and expected_ip not in result["all_ips"]:
         return {
+            **out,
             "is_up": False,
             "resolved_ip": resolved_ip,
             "error_msg": f"Expected {expected_ip} not in {', '.join(result['all_ips'])}",
         }
-    return {"is_up": True, "resolved_ip": resolved_ip, "error_msg": None}
+    return {**out, "is_up": True, "resolved_ip": resolved_ip, "error_msg": None}
 
 
 async def _resolve(address: str) -> dict:
@@ -41,3 +64,97 @@ async def _resolve(address: str) -> dict:
     except Exception as exc:
         logger.debug("DNS resolution failed for %s: %s", address, exc)
         return {"resolved": False, "ip": None, "all_ips": []}
+
+
+async def _query_server(address: str, resolver: str) -> dict:
+    """Ask `resolver` directly for A and AAAA, together; the slower answer is the time."""
+    started = time.monotonic()
+    answers = await asyncio.gather(_ask(address, resolver, _A), _ask(address, resolver, _AAAA),
+                                   return_exceptions=True)
+    ms = round((time.monotonic() - started) * 1000, 1)
+    ips, errors = [], []
+    for got in answers:
+        (errors if isinstance(got, Exception) else ips).append(got)
+    all_ips = [ip for batch in ips for ip in batch]
+    if all_ips:
+        return {"resolved": True, "ip": all_ips[0], "all_ips": all_ips, "ms": ms}
+    if errors:
+        # A timeout on both queries leaves no time worth recording.
+        timed_out = all(isinstance(e, TimeoutError) for e in errors) and not ips
+        return {"resolved": False, "ip": None, "all_ips": [], "ms": None if timed_out else ms,
+                "error": f"{resolver}: {errors[0]}"}
+    return {"resolved": False, "ip": None, "all_ips": [], "ms": ms,
+            "error": f"{resolver} has no A or AAAA record for {address}"}
+
+
+class _Reply(asyncio.DatagramProtocol):
+    def __init__(self, qid: int):
+        self.qid = qid
+        self.done = asyncio.get_running_loop().create_future()
+
+    def datagram_received(self, data, addr):
+        if len(data) >= 2 and struct.unpack("!H", data[:2])[0] == self.qid and not self.done.done():
+            self.done.set_result(data)
+
+    def error_received(self, exc):
+        if not self.done.done():
+            self.done.set_exception(exc)
+
+
+async def _ask(name: str, resolver: str, qtype: int) -> list[str]:
+    """One UDP query; the addresses of `qtype` in the answer, or an exception saying why not."""
+    qid = struct.unpack("!H", os.urandom(2))[0]
+    loop = asyncio.get_running_loop()
+    transport, proto = await loop.create_datagram_endpoint(
+        lambda: _Reply(qid), remote_addr=(resolver, 53))
+    try:
+        transport.sendto(build_query(qid, name, qtype))
+        try:
+            data = await asyncio.wait_for(proto.done, QUERY_TIMEOUT_S)
+        except asyncio.TimeoutError:
+            raise TimeoutError(f"no answer within {QUERY_TIMEOUT_S:.0f}s") from None
+    finally:
+        transport.close()
+    return parse_answer(data, qtype)
+
+
+def build_query(qid: int, name: str, qtype: int) -> bytes:
+    """A standard recursive query for `name`."""
+    header = struct.pack("!HHHHHH", qid, 0x0100, 1, 0, 0, 0)   # RD set, one question
+    labels = b"".join(bytes([len(p)]) + p.encode("idna") for p in name.strip(".").split(".") if p)
+    return header + labels + b"\x00" + struct.pack("!HH", qtype, 1)
+
+
+def parse_answer(data: bytes, qtype: int) -> list[str]:
+    """The `qtype` addresses in a reply; ValueError naming the server's error code."""
+    if len(data) < 12:
+        raise ValueError("short reply")
+    _, flags, qd, an, _, _ = struct.unpack("!HHHHHH", data[:12])
+    rcode = flags & 0x000F
+    if rcode:
+        raise ValueError(_RCODES.get(rcode, f"error code {rcode}"))
+    at = 12
+    for _ in range(qd):
+        at = _skip_name(data, at) + 4
+    ips = []
+    for _ in range(an):
+        at = _skip_name(data, at)
+        rtype, _, _, length = struct.unpack("!HHIH", data[at:at + 10])
+        at += 10
+        rdata = data[at:at + length]
+        at += length
+        if rtype == qtype == _A and length == 4:
+            ips.append(str(ipaddress.IPv4Address(rdata)))
+        elif rtype == qtype == _AAAA and length == 16:
+            ips.append(str(ipaddress.IPv6Address(rdata)))
+    return ips
+
+
+def _skip_name(data: bytes, at: int) -> int:
+    while True:
+        n = data[at]
+        if n == 0:
+            return at + 1
+        if n & 0xC0 == 0xC0:   # compression pointer ends the name
+            return at + 2
+        at += n + 1

steward/monitors/routes.py

@@ -28,7 +28,7 @@ def _form_opts(form, mtype: str) -> dict:
     if mtype == MonitorType.tcp.value:
         return {"port": form.get("port")}
     if mtype == MonitorType.dns.value:
-        return {"expected_ip": form.get("expected_ip")}
+        return {"expected_ip": form.get("expected_ip"), "resolver": form.get("resolver")}
     if mtype == MonitorType.http.value:
         return {
             "method": form.get("method"),

steward/monitors/runner.py

@@ -36,7 +36,8 @@ async def run_monitor(monitor: Monitor) -> dict:
         elif monitor.type == MonitorType.tcp.value:
             result.update(await tcp_check(monitor.target, int(cfg.get("port") or DEFAULT_TCP_PORT)))
         elif monitor.type == MonitorType.dns.value:
-            result.update(await dns_check(monitor.target, cfg.get("expected_ip") or None))
+            result.update(await dns_check(monitor.target, cfg.get("expected_ip") or None,
+                                         cfg.get("resolver") or None))
         elif monitor.type == MonitorType.http.value:
             result.update(await http_check(
                 url=monitor.target,

steward/monitors/service.py

@@ -6,6 +6,7 @@ pipeline) call the same code and refuse the same input for the same reasons.
 """
 from __future__ import annotations
 
+import ipaddress
 import json
 from collections.abc import Mapping
 from datetime import datetime, timedelta, timezone
@@ -21,7 +22,7 @@ HTTP_METHODS = ("GET", "HEAD", "POST")
 CONFIG_KEYS: dict[str, tuple[str, ...]] = {
     MonitorType.icmp.value: (),
     MonitorType.tcp.value: ("port",),
-    MonitorType.dns.value: ("expected_ip",),
+    MonitorType.dns.value: ("expected_ip", "resolver"),
     MonitorType.http.value: ("method", "expected_status", "content_match", "headers",
                              "timeout_seconds", "follow_redirects", "verify_ssl"),
 }
@@ -56,6 +57,17 @@ def _headers(value) -> dict:
     return value
 
 
+def _resolver(value) -> str | None:
+    """A DNS server's IP address, or None for the container's own resolver."""
+    value = str(value or "").strip()
+    if not value:
+        return None
+    try:
+        return str(ipaddress.ip_address(value))
+    except ValueError:
+        raise ValueError(f"resolver must be a DNS server's IP address (e.g. 1.1.1.1), got {value!r}.") from None
+
+
 def normalise_target(mtype: str, target: str) -> str:
     """For http, default a bare host to https://; others pass through."""
     target = (target or "").strip()
@@ -77,7 +89,8 @@ def build_config(mtype: str, opts: Mapping) -> dict:
     if mtype == MonitorType.tcp.value:
         return {"port": _whole("port", opts.get("port"), 80, 1, 65535)}
     if mtype == MonitorType.dns.value:
-        return {"expected_ip": (str(opts.get("expected_ip") or "").strip() or None)}
+        return {"expected_ip": (str(opts.get("expected_ip") or "").strip() or None),
+                "resolver": _resolver(opts.get("resolver"))}
     if mtype == MonitorType.http.value:
         method = str(opts.get("method") or "GET").strip().upper()
         if method not in HTTP_METHODS:

steward/templates/hosts/detail.html

@@ -150,13 +150,7 @@
 </div>
 </div>{# Monitors + Agent grid #}
 
-{# ── Docker (docker plugin fragment; renders nothing if the host has none);
-   above Ansible: containers are state to read, Ansible is something to do ──── #}
-{% if "docker" in enabled_plugins %}
-<div hx-get="/plugins/docker/host/{{ host.id }}" hx-trigger="load" hx-swap="innerHTML"></div>
-{% endif %}
-
-{# ── Ansible (full width) ─────────────────────────────────────────────────── #}
+{# ── Ansible (full width), before Docker ───────────────────────────────────── #}
 <div class="card">
   <h3 class="section-title">Ansible</h3>
   {% if linked_target %}
@@ -243,6 +237,13 @@
   {% endif %}
 </div>{# Ansible card #}
 
+{# ── Docker (docker plugin fragment; renders nothing if the host has none).
+   After Ansible: the container list can run long, and Ansible beneath it was
+   easy to miss (#5214). ─────────────────────────────────────────────────────── #}
+{% if "docker" in enabled_plugins %}
+<div hx-get="/plugins/docker/host/{{ host.id }}" hx-trigger="load" hx-swap="innerHTML"></div>
+{% endif %}
+
 {# ── SNMP (renders nothing unless a configured SNMP device maps to this host) ── #}
 {% if "snmp" in enabled_plugins %}
 <div hx-get="/plugins/snmp/host/{{ host.id }}" hx-trigger="load" hx-swap="innerHTML"></div>

steward/templates/monitors/_fields.html

@@ -10,7 +10,7 @@
 <p class="field-hint mfield" data-types="http" style="grid-column:1/-1;margin:0;">
   <strong>HTTP</strong> requests the target URL (http:// or https://). Up when the status matches, and the page contains the text below if you set one. For https, the certificate's expiry is tracked too.</p>
 <p class="field-hint mfield" data-types="dns" style="grid-column:1/-1;margin:0;">
-  <strong>DNS</strong> looks the target name up (google.com, nas.lan, …) through the resolver Steward's container uses. Up when it resolves, and to the expected IP if you set one. No response time is recorded.</p>
+  <strong>DNS</strong> looks the target name up (google.com, nas.lan, …). Up when it resolves, and to the expected IP if you set one. With no resolver it asks the way Steward's container does (its resolv.conf, so usually your local DNS) and records no response time; name a resolver (1.1.1.1, your router, …) to ask that server directly and time its answer.</p>
 
 {# tcp #}
 <div class="mfield" data-types="tcp">
@@ -24,6 +24,12 @@
   <input type="text" name="expected_ip" value="{{ config.get('expected_ip') or '' }}"
          placeholder="e.g. 192.0.2.10" autocomplete="off" class="input">
 </div>
+<div class="mfield" data-types="dns">
+  <label class="field-label">Resolver <span style="color:var(--text-dim);">(optional)</span></label>
+  <input type="text" name="resolver" value="{{ config.get('resolver') or '' }}"
+         placeholder="the container's own" autocomplete="off" class="input"
+         title="A DNS server's IP address to ask directly. Empty asks the way Steward's container does.">
+</div>
 
 {# http #}
 <div class="mfield" data-types="http">

steward/templates/monitors/detail.html

@@ -134,6 +134,9 @@
     <dd style="overflow-wrap:anywhere;">{{ value|join(", ") if value is mapping else ("yes" if value is sameas true else ("no" if value is sameas false else value)) }}</dd>
     {% endif %}
     {% endfor %}
+    {% if monitor.type == "dns" and not monitor.config.get("resolver") %}
+    <dt>Resolver</dt><dd>the container's own (its resolv.conf)</dd>
+    {% endif %}
   </dl>
 </section>
 {% endblock %}