FabledSteward/steward/monitors/ping.py

"""ICMP / TCP reachability checks — pure probes returning a result dict.

A failed check says why in `error_msg` (#5430): a down monitor that can't
say whether nothing answered or something refused leaves the operator to
guess.

`response_ms` is the network's own time, not Steward's (#6105): ping's
reported round trip, and a TCP connect timed in a thread of its own. Timed
from the event loop it carried starting a process and whatever else the loop
was busy with: about 15 ms on a LAN whose round trip is under 1, and every
host "slow" together whenever Steward was.
"""
from __future__ import annotations
import asyncio
import logging
import re
import socket
import time

logger = logging.getLogger(__name__)

TCP_TIMEOUT = 5.0
DEFAULT_TCP_PORT = 80
PING_WAIT_S = 2
PING_TIMEOUT = 5.0


def tcp_failure(exc: BaseException, address: str, port: int) -> str:
    """Why a TCP connect failed, in a phrase."""
    if isinstance(exc, (asyncio.TimeoutError, TimeoutError)):
        return f"No answer on port {port} within {TCP_TIMEOUT:g} s"
    if isinstance(exc, ConnectionRefusedError):
        return f"Connection refused on port {port}"
    if isinstance(exc, socket.gaierror):
        return f"Can't resolve {address}: {exc.strerror or exc}"
    if isinstance(exc, OSError):
        return f"Can't connect to port {port}: {exc.strerror or exc}"
    return str(exc) or type(exc).__name__


def _connect_ms(address: str, port: int) -> float:
    """Milliseconds a TCP connect to address:port takes, the name resolved
    first and not counted; each address tried in turn, as a client would.
    Blocking: run it in a thread."""
    last: OSError | None = None
    for family, kind, proto, _, sockaddr in socket.getaddrinfo(address, port, type=socket.SOCK_STREAM):
        with socket.socket(family, kind, proto) as sock:
            sock.settimeout(TCP_TIMEOUT)
            start = time.perf_counter()
            try:
                sock.connect(sockaddr)
            except OSError as exc:
                last = exc
                continue
            return (time.perf_counter() - start) * 1000
    raise last or OSError(f"no address for {address}")


async def tcp_check(address: str, port: int) -> dict:
    """TCP connect probe. Returns {is_up, response_ms, error_msg}."""
    try:
        ms = await asyncio.wait_for(asyncio.to_thread(_connect_ms, address, port),
                                    timeout=TCP_TIMEOUT * 2)       # resolving, then the connect's own 5 s
        return {"is_up": True, "response_ms": round(ms, 2), "error_msg": None}
    except Exception as exc:
        return {"is_up": False, "response_ms": None, "error_msg": tcp_failure(exc, address, port)}


_RTT = re.compile(rb"time[=<]([0-9.]+) ?ms")


def ping_rtt(stdout: bytes) -> float | None:
    """The round trip ping reports ("time=0.412 ms", or "time<1 ms"), or None."""
    m = _RTT.search(stdout or b"")
    try:
        return float(m.group(1)) if m else None
    except ValueError:
        return None


def ping_failure(returncode: int, stderr: str) -> str:
    """Why `ping -c 1` failed: exit 1 is no reply; anything else is ping's
    own complaint (an unknown name, no route), first line of it."""
    if returncode == 1:
        return f"No reply within {PING_WAIT_S} s"
    said = stderr.strip().splitlines()[0] if stderr.strip() else ""
    return said or f"ping failed (exit {returncode})"


async def icmp_check(address: str) -> dict:
    """ICMP echo via the system ping binary (setuid; no raw-socket privilege).

    Falls back to a TCP connect on the default port if the ping binary is
    unavailable or errors, so a host with ICMP filtered still gets a signal;
    a failure then says the TCP check stood in.
    """
    start = time.monotonic()
    try:
        proc = await asyncio.create_subprocess_exec(
            "ping", "-c", "1", "-W", str(PING_WAIT_S), address,
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE,
        )
    except Exception as exc:
        logger.warning("ICMP ping failed for %s: %s, falling back to TCP", address, exc)
        result = await tcp_check(address, DEFAULT_TCP_PORT)
        if not result["is_up"]:
            result["error_msg"] = f"ping unavailable, TCP check instead: {result['error_msg']}"
        return result
    try:
        stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=PING_TIMEOUT)
    except asyncio.TimeoutError:
        proc.kill()
        try:
            await asyncio.wait_for(proc.wait(), timeout=PING_TIMEOUT)
        except asyncio.TimeoutError:
            logger.warning("ping for %s did not exit after kill", address)
        return {"is_up": False, "response_ms": None,
                "error_msg": f"No reply within {PING_TIMEOUT:g} s"}
    if proc.returncode == 0:
        rtt = ping_rtt(stdout)
        ms = rtt if rtt is not None else (time.monotonic() - start) * 1000   # a ping that doesn't say
        return {"is_up": True, "response_ms": round(ms, 2), "error_msg": None}
    return {"is_up": False, "response_ms": None,
            "error_msg": ping_failure(proc.returncode, (stderr or b"").decode(errors="replace"))}