FabledSteward/steward/core/metric_series.py

"""Reads over the generic metric store (plugin_metrics + its hourly rollup).

Any source that records into PluginMetric — host_agent, traefik, unifi, snmp,
the monitor types, Steward's self-monitor — is readable through these without
importing the plugin that wrote it.
"""
from __future__ import annotations

from datetime import datetime, timedelta, timezone

from sqlalchemy import func, or_, select, text

from steward.core.like import escape_like
from steward.core.time_range import bucket_seconds
from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest

# Loose index scans: step through the distinct leading values of the
# (source_module, resource_name, …) index one probe at a time instead of
# scanning every sample. Postgres has no native skip scan before 18.
_SOURCES_SQL = text("""
    WITH RECURSIVE s(v) AS (
        SELECT min(source_module) FROM plugin_metrics
        UNION ALL
        SELECT (SELECT min(source_module) FROM plugin_metrics WHERE source_module > s.v)
        FROM s WHERE s.v IS NOT NULL
    )
    SELECT v FROM s WHERE v IS NOT NULL
""")

_RESOURCES_SQL = text("""
    WITH RECURSIVE r(v) AS (
        SELECT min(resource_name) FROM plugin_metrics WHERE source_module = :src
        UNION ALL
        SELECT (SELECT min(resource_name) FROM plugin_metrics
                WHERE source_module = :src AND resource_name > r.v)
        FROM r WHERE r.v IS NOT NULL
    )
    SELECT v FROM r WHERE v IS NOT NULL
""")


async def sources(db) -> list[str]:
    """Every source_module with raw samples on record."""
    return [v for (v,) in await db.execute(_SOURCES_SQL)]


async def resources(db, source_module: str) -> list[str]:
    """Every resource a source has raw samples for, sorted."""
    return [v for (v,) in await db.execute(_RESOURCES_SQL, {"src": source_module})]


def _resource_match(resource: str, include_children: bool):
    match = PluginMetricLatest.resource_name == resource
    if include_children:
        like = escape_like(resource) + ":%"
        match = or_(match, PluginMetricLatest.resource_name.like(like, escape="\\"))
    return match


async def latest_values(db, source_module: str, resource: str, *,
                        include_children: bool = False) -> list[PluginMetricLatest]:
    """The newest sample of each metric for a resource, sorted by (resource, metric).

    include_children also returns the resource's sub-resources, named
    "<resource>:<part>" (a host's mounts, interfaces, sensors). Read from
    plugin_metric_latest, so the cost is the resource's metric count, not its
    history.
    """
    return list((await db.execute(
        select(PluginMetricLatest)
        .where(PluginMetricLatest.source_module == source_module,
               _resource_match(resource, include_children))
        .order_by(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name)
    )).scalars())


async def latest_by_resource(db, source_module: str, resources, metrics=None,
                             ) -> dict[str, dict[str, PluginMetricLatest]]:
    """{resource: {metric: newest sample}} for several resources of a source,
    optionally only the named metrics. A resource with no samples is absent."""
    resources = list(resources)
    if not resources:
        return {}
    q = select(PluginMetricLatest).where(
        PluginMetricLatest.source_module == source_module,
        PluginMetricLatest.resource_name.in_(resources))
    if metrics is not None:
        q = q.where(PluginMetricLatest.metric_name.in_(list(metrics)))
    out: dict[str, dict[str, PluginMetricLatest]] = {}
    for r in (await db.execute(q)).scalars():
        out.setdefault(r.resource_name, {})[r.metric_name] = r
    return out


_BACKFILL_SQL = text("""
    INSERT INTO plugin_metric_latest
        (source_module, resource_name, metric_name, value, recorded_at)
    SELECT DISTINCT ON (metric_name) source_module, resource_name, metric_name,
           value, recorded_at
    FROM plugin_metrics
    WHERE source_module = :src AND resource_name = :res
      AND recorded_at > :since AND recorded_at <= :last
    ORDER BY metric_name, recorded_at DESC
    ON CONFLICT DO NOTHING
""")


async def backfill_latest(sessionmaker, log) -> int:
    """Fill plugin_metric_latest for resources that have history but no latest
    row yet: the ones that stopped reporting before the table existed (#5208).
    A live resource fills itself on its next sample.

    Runs in the background after startup so it never holds the boot. Per
    resource it costs one backward probe for the last sample and a range over
    the hour before it; a resource that already has latest rows is skipped, so
    later boots only step through the resource names. Returns rows written.
    """
    import time
    started, written, filled = time.monotonic(), 0, 0
    async with sessionmaker() as db:
        todo = []
        for src in await sources(db):
            have = set((await db.execute(
                select(PluginMetricLatest.resource_name)
                .where(PluginMetricLatest.source_module == src).distinct()
            )).scalars())
            todo += [(src, res) for res in await resources(db, src) if res not in have]
    for src, res in todo:
        async with sessionmaker() as db, db.begin():
            last = (await db.execute(
                select(func.max(PluginMetric.recorded_at)).where(
                    PluginMetric.source_module == src, PluginMetric.resource_name == res)
            )).scalar()
            if last is None:
                continue
            result = await db.execute(_BACKFILL_SQL, {
                "src": src, "res": res, "last": last, "since": last - timedelta(hours=1)})
            written += result.rowcount or 0
            filled += 1
    if filled:
        log.info("latest metrics: backfilled %d values for %d quiet resources in %.1fs",
                 written, filled, time.monotonic() - started)
    return written


async def points(db, source_module: str, resource: str, metric: str, since: datetime,
                 *, target: int = 120, until: datetime | None = None) -> list[tuple[datetime, float]]:
    """(time, average) points for one metric since `since` (up to `until`,
    else now), oldest first.

    The part of the range older than the raw window comes from the hourly
    rollup; the rest is bucket-averaged in SQL (buckets no wider than an hour
    when both are present, so the series is evenly grained across the seam).
    The raw window is metrics.retention.raw_days, the setting the rollup in
    core/metrics_retention.py reads, so the seam is where the rollup put it.
    """
    from steward.core.settings import get_setting

    raw_days = int(await get_setting(db, "metrics.retention.raw_days") or 7)
    now = datetime.now(timezone.utc)
    raw_cutoff = (now - timedelta(days=raw_days)).replace(minute=0, second=0, microsecond=0)
    out: list[tuple[datetime, float]] = []

    if since < raw_cutoff:
        hourly = await db.execute(
            select(PluginMetricHourly.bucket, PluginMetricHourly.value_avg)
            .where(
                PluginMetricHourly.source_module == source_module,
                PluginMetricHourly.resource_name == resource,
                PluginMetricHourly.metric_name == metric,
                PluginMetricHourly.bucket >= since,
                PluginMetricHourly.bucket < (min(raw_cutoff, until) if until else raw_cutoff),
            )
            .order_by(PluginMetricHourly.bucket)
        )
        out.extend((b, float(v)) for b, v in hourly)

    raw_since = max(since, raw_cutoff)
    if until is not None and raw_since >= until:
        return out
    width_s = (bucket_seconds(raw_since, target) if until is None
               else max(60, int((until - raw_since).total_seconds() / target)))
    if since < raw_cutoff:
        width_s = min(width_s, 3600)
    bucket = func.date_bin(
        func.make_interval(0, 0, 0, 0, 0, 0, width_s),
        PluginMetric.recorded_at,
        func.to_timestamp(0),
    ).label("bucket")
    raw = await db.execute(
        select(bucket, func.avg(PluginMetric.value))
        .where(
            PluginMetric.source_module == source_module,
            PluginMetric.resource_name == resource,
            PluginMetric.metric_name == metric,
            PluginMetric.recorded_at >= raw_since,
            *((PluginMetric.recorded_at < until,) if until is not None else ()),
        )
        .group_by(bucket)
        .order_by(bucket)
    )
    out.extend((b, float(v)) for b, v in raw)
    return out