bvandeusenandClaude Opus 5.5
656e127fd8
#5490 "A day of one metric reads fast even cold". A cold 24 h series read measured 18.3 s against 63 ms warm (#5485): every sample's row sat on its own heap page. ix_plugin_metrics_series_value carries `value` (INCLUDE), so the read is an index-only scan; it replaces the plain series index from 0023. It is built once, with CREATE INDEX CONCURRENTLY from a task started after the app is serving, never in a migration, so the boot isn't held while it builds over the live table (#5220) and ingest and pages carry on. A build cut short by a restart leaves an INVALID index; the next boot drops and rebuilds it. Later boots find it present and only check. Postgres keeps it current on each insert, as it did the index it replaces. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
61 lines
2.9 KiB
Python
61 lines
2.9 KiB
Python
"""The series index on plugin_metrics, built in the background after boot (#5490).
|
|
|
|
A read of one metric over a day (a rule's preview, query_metrics, a chart)
|
|
walks the (source, resource, metric, recorded_at) index; without the value in
|
|
the index, each sample is then fetched from its own heap page, interleaved
|
|
with every other series, so a cold day measured 18.3 s against 63 ms warm
|
|
(#5485). With `value` INCLUDEd the read is an index-only scan over adjacent
|
|
index pages.
|
|
|
|
The index is built with CREATE INDEX CONCURRENTLY from a task started after
|
|
the app is serving, never in a migration: a build over the live table takes
|
|
minutes on slow storage, and a migration would hold the boot with Steward
|
|
down for all of it (#5220). Ingest and pages keep working while it builds.
|
|
Once it is valid the index it replaces is dropped. Later boots find it
|
|
present and only check.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
|
|
from sqlalchemy import text
|
|
|
|
SERIES_INDEX = "ix_plugin_metrics_series_value"
|
|
# The plain series index the covering one replaces (migration 0023).
|
|
REPLACES = "ix_plugin_metrics_module_resource_metric_recorded"
|
|
|
|
_STATE = text("""
|
|
SELECT i.indisvalid FROM pg_class c JOIN pg_index i ON i.indexrelid = c.oid
|
|
WHERE c.relname = :name
|
|
""")
|
|
|
|
|
|
async def ensure_series_index(engine, log) -> str:
|
|
"""Make sure the covering series index exists and is valid, then drop the
|
|
one it replaces. Returns "present" when it already was, else "built".
|
|
|
|
A build a restart interrupted leaves an INVALID index behind; that one is
|
|
dropped and built again. Runs outside a transaction, as CONCURRENTLY
|
|
requires."""
|
|
async with engine.connect() as conn:
|
|
conn = await conn.execution_options(isolation_level="AUTOCOMMIT")
|
|
valid = (await conn.execute(_STATE, {"name": SERIES_INDEX})).scalar()
|
|
outcome = "present"
|
|
if valid is not True:
|
|
if valid is False:
|
|
log.warning("metric index: %s was left half-built; dropping it to build again",
|
|
SERIES_INDEX)
|
|
await conn.execute(text(f"DROP INDEX CONCURRENTLY IF EXISTS {SERIES_INDEX}"))
|
|
log.info("metric index: building %s in the background; pages and ingest carry on",
|
|
SERIES_INDEX)
|
|
started = time.monotonic()
|
|
await conn.execute(text(
|
|
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {SERIES_INDEX} ON plugin_metrics "
|
|
"(source_module, resource_name, metric_name, recorded_at) INCLUDE (value)"))
|
|
log.info("metric index: built %s in %.0fs", SERIES_INDEX, time.monotonic() - started)
|
|
outcome = "built"
|
|
if (await conn.execute(_STATE, {"name": REPLACES})).scalar() is not None:
|
|
await conn.execute(text(f"DROP INDEX CONCURRENTLY IF EXISTS {REPLACES}"))
|
|
log.info("metric index: dropped %s, which %s replaces", REPLACES, SERIES_INDEX)
|
|
return outcome
|