FabledSteward/steward/core/cleanup.py

from __future__ import annotations
import asyncio
import logging
import os
import time
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING

from sqlalchemy import delete

from steward.core.anomalies import prune as prune_anomalies
from steward.models.alerts import AlertEvent
from steward.models.ansible import AnsibleRun
from steward.models.audit import AuditEvent
from steward.models.monitors import MonitorResult
from steward.models.outbox import EmailOutbox

if TYPE_CHECKING:
    from quart import Quart

logger = logging.getLogger(__name__)


# Time-series columns pruned at data.retention_days (#5155). A plugin adds its
# own with prune_by_age() in setup(), so the core never imports plugin models.
# Tables kept another way are not listed: plugin_metrics and the docker tables
# are rolled up (below), and the audit log has its own, longer window.
_AGED: list = [MonitorResult.checked_at, AnsibleRun.started_at, AlertEvent.transitioned_at,
               EmailOutbox.created_at]


def prune_by_age(column) -> None:
    """Have the hourly cleanup delete rows whose `column` (a mapped DateTime
    attribute, e.g. TraefikMetric.scraped_at) is older than data.retention_days."""
    if not any(c is column for c in _AGED):
        _AGED.append(column)


# What each rollup count means, as Settings → System lists it (#5469).
_METRIC_COUNTS = [("raw_rows_rolled", "plugin_metrics: raw samples rolled into hourly"),
                  ("rollup_pruned", "plugin_metrics_hourly: past the rollup window"),
                  ("latest_pruned", "plugin_metric_latest: stopped reporting")]
_DOCKER_COUNTS = [("raw_rows_rolled", "docker_metrics: raw samples rolled into hourly"),
                  ("rollup_pruned", "docker_metrics_hourly: past the rollup window"),
                  ("events_pruned", "docker_events: past the events window"),
                  ("logs_age_pruned", "docker_logs: past the age window"),
                  ("logs_size_pruned", "docker_logs: over a container's size cap")]


async def run_cleanup(app: "Quart") -> dict:
    """Delete rows older than their retention window from every growing table,
    and run logs older than the data window, then run the metric and Docker
    rollups. Windows are read from Settings each run (rule 25: no restart).

    Returns what it did, {"at", "secs", "removed": [[what, count]]}, and keeps
    the same as the cleanup.last_run setting for Settings → System (#5469). A
    run that raises records nothing, so the page keeps the last one that
    finished."""
    from steward.core.settings import get_setting, set_setting
    from steward.ansible.executor import ARTIFACT_DIR

    now = datetime.now(timezone.utc)
    started = time.monotonic()
    removed: list[list] = []
    async with app.db_sessionmaker() as session:
        async with session.begin():
            data_days = int(await get_setting(session, "data.retention_days") or 90)
            audit_days = int(await get_setting(session, "audit.retention_days") or 365)
            cutoff = now - timedelta(days=data_days)
            aged = [(c, cutoff, data_days) for c in _AGED]
            aged.append((AuditEvent.timestamp, now - timedelta(days=audit_days), audit_days))
            for column, before, days in aged:
                result = await session.execute(
                    delete(column.class_).where(column < before)
                )
                table = column.class_.__tablename__
                removed.append([f"{table}: older than {days} days", result.rowcount or 0])
                if result.rowcount:
                    logger.info("Pruned %d rows from %s", result.rowcount, table)
            # Anomalies go by when their last occurrence ended, and only once
            # reviewed or lapsed: one still to review is never pruned (#5740).
            pruned = await prune_anomalies(session, cutoff)
            removed.append([f"metric_anomalies: reviewed, ended over {data_days} days ago",
                            pruned])

        files = await asyncio.to_thread(prune_run_logs, ARTIFACT_DIR, cutoff)
        removed.append([f"Ansible run log files: older than {data_days} days", files])
        if files:
            logger.info("Pruned %d Ansible run log files from %s", files, ARTIFACT_DIR)

        # plugin_metrics is NOT blanket-deleted here — it's rolled up to hourly
        # then pruned, so multi-week host history stays cheap.
        counts = await _run_metrics_retention(session, now)
        removed += [[what, counts.get(k) or 0] for k, what in _METRIC_COUNTS]
        counts = await _run_docker_retention(session, now)
        if counts is not None:
            removed += [[what, counts.get(k) or 0] for k, what in _DOCKER_COUNTS]

        summary = {"at": now.isoformat(), "secs": round(time.monotonic() - started, 2),
                   "removed": removed}
        async with session.begin():
            await set_setting(session, "cleanup.last_run", summary)
    return summary


def prune_run_logs(directory: str, before: datetime) -> int:
    """Delete the run log files in `directory` last written before `before`,
    the same window as the runs they belong to. Returns how many went."""
    removed = 0
    try:
        entries = list(os.scandir(directory))
    except OSError:          # no run has written one yet, or not readable
        return 0
    limit = before.timestamp()
    for entry in entries:
        try:
            if (entry.name.endswith(".log") and entry.is_file()
                    and entry.stat().st_mtime < limit):
                os.unlink(entry.path)
                removed += 1
        except OSError:
            logger.warning("Could not prune run log %s", entry.path)
    return removed


async def _run_metrics_retention(session, now: datetime) -> dict:
    """Roll up + prune plugin_metrics (raw → hourly → gone). Windows read fresh
    from settings each run (rule 25 — UI change takes effect next cleanup, no
    restart). get_setting's SELECT autobegins, so read inside the begin block."""
    from steward.core.metrics_retention import rollup_plugin_metrics
    from steward.core.settings import get_setting

    async with session.begin():
        raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7)
        rollup_days = int(await get_setting(session, "metrics.retention.rollup_days") or 90)
        counts = await rollup_plugin_metrics(
            session, raw_days=raw_days, rollup_days=rollup_days, now=now,
        )
    if counts and any(counts.values()):
        logger.info("Metrics retention: %s", counts)
    return counts or {}


async def _run_docker_retention(session, now: datetime) -> dict | None:
    """Drive the docker plugin's rollup + prune and its log ring via its
    capabilities, if loaded.

    Windows are read fresh from settings each run (rule 25 — a change in the
    Settings UI takes effect on the next hourly cleanup, no restart). Each runs
    in its own transaction, so a docker-side failure can't roll back the generic
    prune above, and the log ring's deletes don't wait on the metrics rollup
    (#5449). No-op when the docker plugin is disabled (capability absent).
    """
    from steward.core.capabilities import has_capability, invoke_capability
    if not has_capability("docker.run_retention"):
        return None
    from steward.core.settings import get_setting
    from steward.models.users import UserRole

    # Reads + rollup/prune share one transaction — get_setting's SELECT would
    # otherwise autobegin one, making a later session.begin() raise.
    async with session.begin():
        raw_days = int(await get_setting(session, "docker.retention.metrics_raw_days") or 7)
        rollup_days = int(await get_setting(session, "docker.retention.metrics_rollup_days") or 90)
        events_days = int(await get_setting(session, "docker.retention.events_days") or 30)
        counts = await invoke_capability(
            "docker.run_retention", UserRole.viewer, session,
            events_days=events_days, metrics_raw_days=raw_days,
            metrics_rollup_days=rollup_days, now=now,
        )
    async with session.begin():
        logs_days = int(await get_setting(session, "docker.logs.retention_days") or 3)
        logs_cap = int(
            await get_setting(session, "docker.logs.max_bytes_per_container") or 5_000_000)
        counts |= await invoke_capability(
            "docker.prune_logs", UserRole.viewer, session,
            retention_days=logs_days, max_bytes_per_container=logs_cap, now=now,
        )
    if counts and any(counts.values()):
        logger.info("Docker retention: %s", counts)
    return counts