diff --git a/plugins/docker/models.py b/plugins/docker/models.py index f074493..99330de 100644 --- a/plugins/docker/models.py +++ b/plugins/docker/models.py @@ -1,6 +1,5 @@ # plugins/docker/models.py from __future__ import annotations -import uuid from datetime import datetime, timezone from sqlalchemy import ( BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text, @@ -8,6 +7,7 @@ from sqlalchemy import ( ) from sqlalchemy.orm import Mapped, mapped_column from steward.models.base import Base +from steward.core.ids import time_ordered_id class DockerContainer(Base): @@ -67,7 +67,7 @@ class DockerMetric(Base): __tablename__ = "docker_metrics" id: Mapped[str] = mapped_column( - String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + String(36), primary_key=True, default=time_ordered_id ) host_id: Mapped[str] = mapped_column( String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True @@ -103,7 +103,7 @@ class DockerMetricHourly(Base): __tablename__ = "docker_metrics_hourly" id: Mapped[str] = mapped_column( - String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + String(36), primary_key=True, default=time_ordered_id ) host_id: Mapped[str] = mapped_column( String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False @@ -139,7 +139,7 @@ class DockerEvent(Base): __tablename__ = "docker_events" id: Mapped[str] = mapped_column( - String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + String(36), primary_key=True, default=time_ordered_id ) host_id: Mapped[str] = mapped_column( String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False @@ -176,7 +176,7 @@ class DockerLog(Base): __tablename__ = "docker_logs" id: Mapped[str] = mapped_column( - String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + String(36), primary_key=True, default=time_ordered_id ) host_id: Mapped[str] = mapped_column( String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index bb27b68..8efc3cf 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -343,7 +343,9 @@ SLOW_INGEST_SECONDS = 2.0 # instance that did not finish inside the agent's 10 s, the rollback left the # buffer full, and every later post failed the same way. The newest samples # are kept: the older ones would be lost to the agent's ring buffer anyway. -MAX_SAMPLES_PER_INGEST = 5 +# Two (~400 rows) fits 10 s even at the ~110 rows/s measured before +# plugin_metrics got time-ordered ids; raise it once the new rate is known. +MAX_SAMPLES_PER_INGEST = 2 def newest_samples(samples: list, limit: int) -> tuple[list, int]: diff --git a/plugins/traefik/models.py b/plugins/traefik/models.py index d177d10..67c5bf7 100644 --- a/plugins/traefik/models.py +++ b/plugins/traefik/models.py @@ -1,16 +1,16 @@ # plugins/traefik/models.py from __future__ import annotations -import uuid from datetime import datetime, timezone from sqlalchemy import DateTime, Float, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column from steward.models.base import Base +from steward.core.ids import time_ordered_id class TraefikMetric(Base): __tablename__ = "traefik_metrics" - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) router_name: Mapped[str] = mapped_column(String(255), nullable=False) scraped_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) @@ -41,7 +41,7 @@ class TraefikAccessSummary(Base): """Aggregated traffic-origin summary — one row per scheduler tick.""" __tablename__ = "traefik_access_summaries" - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) period_start: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) period_end: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) total_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0) @@ -57,7 +57,7 @@ class TraefikGlobalStat(Base): """One row per scrape — process-level and config stats.""" __tablename__ = "traefik_global_stats" - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) scraped_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) ) diff --git a/plugins/unifi/models.py b/plugins/unifi/models.py index bb36410..0a2e3d1 100644 --- a/plugins/unifi/models.py +++ b/plugins/unifi/models.py @@ -1,10 +1,10 @@ # plugins/unifi/models.py from __future__ import annotations -import uuid from datetime import datetime, timezone from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column from steward.models.base import Base +from steward.core.ids import time_ordered_id # ── Time-series (one row per scrape) ────────────────────────────────────────── @@ -13,7 +13,7 @@ class UnifiWanStat(Base): """WAN interface health — one row per scrape.""" __tablename__ = "unifi_wan_stats" - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) wan_ip: Mapped[str | None] = mapped_column(String(64), nullable=True) @@ -27,7 +27,7 @@ class UnifiClientSnapshot(Base): """Aggregated client counts + top talkers — one row per scrape.""" __tablename__ = "unifi_client_snapshots" - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) total_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) wireless_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0) @@ -41,7 +41,7 @@ class UnifiWlanStat(Base): """Per-SSID/radio stats — one row per scrape per SSID+radio combination.""" __tablename__ = "unifi_wlan_stats" - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) ssid: Mapped[str] = mapped_column(String(255), nullable=False) radio: Mapped[str | None] = mapped_column(String(8), nullable=True) # ng=2.4GHz, na=5GHz, 6e=6GHz @@ -56,7 +56,7 @@ class UnifiDpiSnapshot(Base): """DPI application/category traffic snapshot — one row per scrape (JSON blob).""" __tablename__ = "unifi_dpi_snapshots" - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) # JSON: [{"cat_id", "cat_name", "rx_bytes", "tx_bytes"}] aggregated across all clients by_category_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") diff --git a/steward/core/ids.py b/steward/core/ids.py new file mode 100644 index 0000000..1cd698d --- /dev/null +++ b/steward/core/ids.py @@ -0,0 +1,25 @@ +"""Row ids for tables that grow by time. + +A random uuid4 primary key puts every new row on a random page of the key's +index; once that index outgrows memory, each insert waits for a disk read. +Measured on the live instance (#5149): plugin_metrics took ~9 ms a row, about +110 rows/s, so a host agent's post could not finish inside its 10 s timeout. +A time-ordered id lands every new row at the index's right edge instead. +""" +from __future__ import annotations + +import os +import time +import uuid + + +def time_ordered_id() -> str: + """A UUIDv7-layout id (RFC 9562) as a 36-character string: a 48-bit + millisecond timestamp, then 74 random bits. Same shape and column type as + the uuid4 ids already stored, so no table changes; Python's own uuid7() + arrives only in 3.14.""" + ms = time.time_ns() // 1_000_000 + value = (ms & 0xFFFF_FFFF_FFFF) << 80 | int.from_bytes(os.urandom(10), "big") + value = (value & ~(0xF << 76)) | (0x7 << 76) # version 7 + value = (value & ~(0x3 << 62)) | (0x2 << 62) # RFC 9562 variant + return str(uuid.UUID(int=value)) diff --git a/steward/models/audit.py b/steward/models/audit.py index 2d53d50..fe58c13 100644 --- a/steward/models/audit.py +++ b/steward/models/audit.py @@ -1,15 +1,15 @@ from __future__ import annotations -import uuid from datetime import datetime, timezone from sqlalchemy import DateTime, ForeignKey, String, Text from sqlalchemy.orm import Mapped, mapped_column from .base import Base +from steward.core.ids import time_ordered_id class AuditEvent(Base): __tablename__ = "audit_events" - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) timestamp: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) ) diff --git a/steward/models/metrics.py b/steward/models/metrics.py index 8a26996..0cd35c6 100644 --- a/steward/models/metrics.py +++ b/steward/models/metrics.py @@ -1,15 +1,15 @@ from __future__ import annotations -import uuid from datetime import datetime, timezone from sqlalchemy import DateTime, Float, Index, Integer, String, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column from .base import Base +from steward.core.ids import time_ordered_id class PluginMetric(Base): __tablename__ = "plugin_metrics" - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) source_module: Mapped[str] = mapped_column(String(64), nullable=False) resource_name: Mapped[str] = mapped_column(String(255), nullable=False) metric_name: Mapped[str] = mapped_column(String(128), nullable=False) @@ -40,7 +40,7 @@ class PluginMetricHourly(Base): """ __tablename__ = "plugin_metrics_hourly" - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) source_module: Mapped[str] = mapped_column(String(64), nullable=False) resource_name: Mapped[str] = mapped_column(String(255), nullable=False) metric_name: Mapped[str] = mapped_column(String(128), nullable=False) diff --git a/steward/models/monitors.py b/steward/models/monitors.py index 89713f2..c75ed65 100644 --- a/steward/models/monitors.py +++ b/steward/models/monitors.py @@ -8,6 +8,7 @@ from sqlalchemy import ( ) from sqlalchemy.orm import Mapped, mapped_column from .base import Base +from steward.core.ids import time_ordered_id class MonitorType(str, enum.Enum): @@ -93,7 +94,7 @@ class MonitorResult(Base): __tablename__ = "monitor_results" id: Mapped[str] = mapped_column( - String(36), primary_key=True, default=lambda: str(uuid.uuid4()) + String(36), primary_key=True, default=time_ordered_id ) monitor_id: Mapped[str] = mapped_column( String(36), ForeignKey("monitors.id", ondelete="CASCADE"), diff --git a/tests/core/test_ids.py b/tests/core/test_ids.py new file mode 100644 index 0000000..4bb0b2c --- /dev/null +++ b/tests/core/test_ids.py @@ -0,0 +1,34 @@ +"""Time-series rows get time-ordered ids, so a new row lands at the right edge +of the primary-key index rather than on a random page of it (#5149).""" +import uuid + +from steward.core import ids +from steward.core.ids import time_ordered_id +from steward.models.metrics import PluginMetric + + +def test_the_id_is_a_version_7_uuid_string(): + raw = time_ordered_id() + u = uuid.UUID(raw) + assert len(raw) == 36 and str(u) == raw + assert u.version == 7 and u.variant == uuid.RFC_4122 + + +def test_ids_sort_by_the_millisecond_they_were_made(monkeypatch): + made = [] + for ms in (1_791_327_000_000, 1_791_327_000_001, 1_791_327_060_000): + monkeypatch.setattr(ids.time, "time_ns", lambda ms=ms: ms * 1_000_000) + made.append(time_ordered_id()) + assert made == sorted(made) + assert len({m[:13] for m in made}) == 3 # the timestamp leads the string + + +def test_two_ids_in_one_millisecond_differ(monkeypatch): + monkeypatch.setattr(ids.time, "time_ns", lambda: 1_791_327_000_000 * 1_000_000) + assert time_ordered_id() != time_ordered_id() + + +def test_plugin_metrics_use_it(): + # SQLAlchemy wraps a no-argument default as fn(context); call it as it would. + made = PluginMetric.__table__.c.id.default.arg(None) + assert uuid.UUID(made).version == 7