bvandeusenandClaude Opus 5.5
ad4f7d9232
Measured on dev (385a579), agent posts cut to five samples still dropped:
flush=8.8s for 990 rows, 6.6s for 680, 8.4s for 1000 — about 110 rows/s,
~9 ms a row, with rules and evaluation near zero. One random read per
row fits a random uuid4 primary key whose index no longer fits in memory:
every new row lands on a random page of it. (Inferred from the rate; the
table and index sizes have not been read.)
- steward.core.ids.time_ordered_id: a UUIDv7-layout string (48-bit ms
timestamp, then random), same 36-char shape as the stored uuid4 ids, so
no migration. Python's uuid7() arrives in 3.14; the image runs 3.13.
- Used for plugin_metrics and its hourly rollup, monitor_results,
audit_events, the docker metrics/hourly/events/logs tables, and the
traefik and unifi time series.
- MAX_SAMPLES_PER_INGEST 5 → 2 (~400 rows) for now: it fits the agent's
10 s even at the old rate, so hosts land data whichever way the new rate
comes out. Raise it once the new rate is measured.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
58 lines
2.8 KiB
Python
58 lines
2.8 KiB
Python
from __future__ import annotations
|
||
from datetime import datetime, timezone
|
||
from sqlalchemy import DateTime, Float, Index, Integer, String, UniqueConstraint
|
||
from sqlalchemy.orm import Mapped, mapped_column
|
||
from .base import Base
|
||
from steward.core.ids import time_ordered_id
|
||
|
||
|
||
class PluginMetric(Base):
|
||
__tablename__ = "plugin_metrics"
|
||
|
||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||
source_module: Mapped[str] = mapped_column(String(64), nullable=False)
|
||
resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||
metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||
value: Mapped[float] = mapped_column(Float, nullable=False)
|
||
recorded_at: Mapped[datetime] = mapped_column(
|
||
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
|
||
)
|
||
|
||
# This time-series table grows by (sources × resources × sample cadence); every
|
||
# host-detail / full-metrics / dashboard-widget read filters by
|
||
# (source_module, resource_name) over a recorded_at range. Without these it's a
|
||
# full sequential scan on every load.
|
||
__table_args__ = (
|
||
Index("ix_plugin_metrics_module_resource_recorded",
|
||
"source_module", "resource_name", "recorded_at"),
|
||
Index("ix_plugin_metrics_module_resource_metric_recorded",
|
||
"source_module", "resource_name", "metric_name", "recorded_at"),
|
||
)
|
||
|
||
|
||
class PluginMetricHourly(Base):
|
||
"""Hourly rollup of plugin_metrics — the coarse series that retention rolls
|
||
raw samples into before pruning them, so multi-day/week history stays cheap.
|
||
|
||
One row per (source_module, resource_name, metric_name, hour bucket); the
|
||
unique constraint is the conflict target for the idempotent rollup upsert.
|
||
Charts read this for the part of a range older than the raw-retention window.
|
||
"""
|
||
__tablename__ = "plugin_metrics_hourly"
|
||
|
||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||
source_module: Mapped[str] = mapped_column(String(64), nullable=False)
|
||
resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||
metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||
bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||
value_avg: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||
value_max: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||
sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||
|
||
__table_args__ = (
|
||
UniqueConstraint("source_module", "resource_name", "metric_name", "bucket",
|
||
name="uq_plugin_metrics_hourly_bucket"),
|
||
Index("ix_plugin_metrics_hourly_lookup",
|
||
"source_module", "resource_name", "metric_name", "bucket"),
|
||
)
|