fix(metrics): time-series rows get time-ordered ids, so an insert stops costing a disk read (#5149)
Measured on dev (385a579), agent posts cut to five samples still dropped:
flush=8.8s for 990 rows, 6.6s for 680, 8.4s for 1000 — about 110 rows/s,
~9 ms a row, with rules and evaluation near zero. One random read per
row fits a random uuid4 primary key whose index no longer fits in memory:
every new row lands on a random page of it. (Inferred from the rate; the
table and index sizes have not been read.)
- steward.core.ids.time_ordered_id: a UUIDv7-layout string (48-bit ms
timestamp, then random), same 36-char shape as the stored uuid4 ids, so
no migration. Python's uuid7() arrives in 3.14; the image runs 3.13.
- Used for plugin_metrics and its hourly rollup, monitor_results,
audit_events, the docker metrics/hourly/events/logs tables, and the
traefik and unifi time series.
- MAX_SAMPLES_PER_INGEST 5 → 2 (~400 rows) for now: it fits the agent's
10 s even at the old rate, so hosts land data whichever way the new rate
comes out. Raise it once the new rate is measured.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
"""Row ids for tables that grow by time.
|
||||
|
||||
A random uuid4 primary key puts every new row on a random page of the key's
|
||||
index; once that index outgrows memory, each insert waits for a disk read.
|
||||
Measured on the live instance (#5149): plugin_metrics took ~9 ms a row, about
|
||||
110 rows/s, so a host agent's post could not finish inside its 10 s timeout.
|
||||
A time-ordered id lands every new row at the index's right edge instead.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
def time_ordered_id() -> str:
|
||||
"""A UUIDv7-layout id (RFC 9562) as a 36-character string: a 48-bit
|
||||
millisecond timestamp, then 74 random bits. Same shape and column type as
|
||||
the uuid4 ids already stored, so no table changes; Python's own uuid7()
|
||||
arrives only in 3.14."""
|
||||
ms = time.time_ns() // 1_000_000
|
||||
value = (ms & 0xFFFF_FFFF_FFFF) << 80 | int.from_bytes(os.urandom(10), "big")
|
||||
value = (value & ~(0xF << 76)) | (0x7 << 76) # version 7
|
||||
value = (value & ~(0x3 << 62)) | (0x2 << 62) # RFC 9562 variant
|
||||
return str(uuid.UUID(int=value))
|
||||
@@ -1,15 +1,15 @@
|
||||
from __future__ import annotations
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy import DateTime, ForeignKey, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from .base import Base
|
||||
from steward.core.ids import time_ordered_id
|
||||
|
||||
|
||||
class AuditEvent(Base):
|
||||
__tablename__ = "audit_events"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||||
timestamp: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
from __future__ import annotations
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy import DateTime, Float, Index, Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from .base import Base
|
||||
from steward.core.ids import time_ordered_id
|
||||
|
||||
|
||||
class PluginMetric(Base):
|
||||
__tablename__ = "plugin_metrics"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||||
source_module: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
@@ -40,7 +40,7 @@ class PluginMetricHourly(Base):
|
||||
"""
|
||||
__tablename__ = "plugin_metrics_hourly"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||||
source_module: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
|
||||
@@ -8,6 +8,7 @@ from sqlalchemy import (
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from .base import Base
|
||||
from steward.core.ids import time_ordered_id
|
||||
|
||||
|
||||
class MonitorType(str, enum.Enum):
|
||||
@@ -93,7 +94,7 @@ class MonitorResult(Base):
|
||||
__tablename__ = "monitor_results"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
|
||||
String(36), primary_key=True, default=time_ordered_id
|
||||
)
|
||||
monitor_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("monitors.id", ondelete="CASCADE"),
|
||||
|
||||
Reference in New Issue
Block a user