feat(library): measure every track's loudness in the background (M464 #4995)

The first step of loudness normalization: the server measures each track
with ffmpeg's EBU R128 filter (true peak, mono as dual mono) and stores the
integrated loudness, true peak and loudness range in track_loudness
(migration 0065).

It also keeps a histogram of the 400 ms gating blocks at 0.1 LU, so album
loudness can be computed exactly later with no second decode (#4996). The
histogram reproduces ffmpeg's own figure (-10.68 against -10.7 on the
captured fixture), and the analyzer logs a warning if the two ever drift.

- A background worker, cloned from the fingerprint backfill, measures every
  track, new ones included. Measuring inline in the scan was dropped: the
  analysis decodes the whole file, and a large import could pass the scan's
  one-hour stuck threshold. The scan only deletes a changed file's
  measurement; the worker ticks every 10 minutes.
- Timeouts, the cancel/missing-binary split and settled verdicts follow the
  fingerprint runner. Silence and undecodable files are stored as verdicts;
  stalls are retried. The deadline scales with track length.
- loudness_settings (enabled, files at once) and an admin card with the
  coverage gauge, under GET/PUT /api/admin/library/loudness-settings and
  GET /api/admin/library/loudness.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/db/dbq/loudness.sql.go

@@ -0,0 +1,208 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.31.1
+// source: loudness.sql
+
+package dbq
+
+import (
+	"context"
+
+	"github.com/jackc/pgx/v5/pgtype"
+)
+
+const deleteTrackLoudness = `-- name: DeleteTrackLoudness :exec
+DELETE FROM track_loudness WHERE track_id = $1
+`
+
+// The scan saw new bytes at this path. The stored measurement describes the old
+// ones, so it goes, and the backfill measures the file again.
+func (q *Queries) DeleteTrackLoudness(ctx context.Context, trackID pgtype.UUID) error {
+	_, err := q.db.Exec(ctx, deleteTrackLoudness, trackID)
+	return err
+}
+
+const getLoudnessCoverage = `-- name: GetLoudnessCoverage :one
+SELECT count(*)::bigint AS total,
+       count(*) FILTER (
+           WHERE l.analysis_version >= $1
+             AND l.integrated_lufs IS NOT NULL
+       )::bigint AS measured,
+       count(*) FILTER (
+           WHERE l.analysis_version >= $1
+             AND l.integrated_lufs IS NULL AND NOT l.unreadable
+       )::bigint AS silent,
+       count(*) FILTER (
+           WHERE l.analysis_version >= $1
+             AND l.unreadable
+       )::bigint AS unreadable,
+       count(*) FILTER (
+           WHERE l.track_id IS NULL OR l.analysis_version < $1
+       )::bigint AS pending
+  FROM tracks t
+  LEFT JOIN track_loudness l ON l.track_id = t.id
+ WHERE t.missing_since IS NULL
+`
+
+type GetLoudnessCoverageRow struct {
+	Total      int64
+	Measured   int64
+	Silent     int64
+	Unreadable int64
+	Pending    int64
+}
+
+// The admin gauge. measured + silent + unreadable + pending = total. Missing
+// tracks are excluded, or the gauge could never reach the end.
+func (q *Queries) GetLoudnessCoverage(ctx context.Context, currentVersion int16) (GetLoudnessCoverageRow, error) {
+	row := q.db.QueryRow(ctx, getLoudnessCoverage, currentVersion)
+	var i GetLoudnessCoverageRow
+	err := row.Scan(
+		&i.Total,
+		&i.Measured,
+		&i.Silent,
+		&i.Unreadable,
+		&i.Pending,
+	)
+	return i, err
+}
+
+const getLoudnessSettings = `-- name: GetLoudnessSettings :one
+SELECT id, enabled, backfill_concurrency, updated_at FROM loudness_settings WHERE id = true
+`
+
+func (q *Queries) GetLoudnessSettings(ctx context.Context) (LoudnessSetting, error) {
+	row := q.db.QueryRow(ctx, getLoudnessSettings)
+	var i LoudnessSetting
+	err := row.Scan(
+		&i.ID,
+		&i.Enabled,
+		&i.BackfillConcurrency,
+		&i.UpdatedAt,
+	)
+	return i, err
+}
+
+const listTracksNeedingLoudness = `-- name: ListTracksNeedingLoudness :many
+SELECT t.id, t.file_path, t.duration_ms
+  FROM tracks t
+  LEFT JOIN track_loudness l ON l.track_id = t.id
+ WHERE t.missing_since IS NULL
+   AND (l.track_id IS NULL OR l.analysis_version < $1)
+   AND t.id > $2
+ ORDER BY t.id
+ LIMIT $3
+`
+
+type ListTracksNeedingLoudnessParams struct {
+	CurrentVersion int16
+	AfterID        pgtype.UUID
+	BatchLimit     int32
+}
+
+type ListTracksNeedingLoudnessRow struct {
+	ID         pgtype.UUID
+	FilePath   string
+	DurationMs int32
+}
+
+// The backfill's work queue: tracks with no measurement, or one taken by an
+// older method. Keyset-paged on id so a pass visits each track at most once;
+// an inconclusive attempt writes no row, and without the cursor a file that
+// keeps timing out would be listed again straight away. Missing tracks are
+// skipped: there is no file to read.
+func (q *Queries) ListTracksNeedingLoudness(ctx context.Context, arg ListTracksNeedingLoudnessParams) ([]ListTracksNeedingLoudnessRow, error) {
+	rows, err := q.db.Query(ctx, listTracksNeedingLoudness, arg.CurrentVersion, arg.AfterID, arg.BatchLimit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ListTracksNeedingLoudnessRow
+	for rows.Next() {
+		var i ListTracksNeedingLoudnessRow
+		if err := rows.Scan(&i.ID, &i.FilePath, &i.DurationMs); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const updateLoudnessSettings = `-- name: UpdateLoudnessSettings :one
+UPDATE loudness_settings
+   SET enabled              = $1,
+       backfill_concurrency = $2,
+       updated_at           = now()
+ WHERE id = true
+RETURNING id, enabled, backfill_concurrency, updated_at
+`
+
+type UpdateLoudnessSettingsParams struct {
+	Enabled             bool
+	BackfillConcurrency int32
+}
+
+// Whole-row write from the admin card; migration 0065's CHECK is the backstop
+// behind the service's own validation.
+func (q *Queries) UpdateLoudnessSettings(ctx context.Context, arg UpdateLoudnessSettingsParams) (LoudnessSetting, error) {
+	row := q.db.QueryRow(ctx, updateLoudnessSettings, arg.Enabled, arg.BackfillConcurrency)
+	var i LoudnessSetting
+	err := row.Scan(
+		&i.ID,
+		&i.Enabled,
+		&i.BackfillConcurrency,
+		&i.UpdatedAt,
+	)
+	return i, err
+}
+
+const upsertTrackLoudness = `-- name: UpsertTrackLoudness :exec
+INSERT INTO track_loudness (
+    track_id, integrated_lufs, true_peak_dbtp, loudness_range_lu,
+    block_hist_start, block_hist, unreadable, analysis_version
+) VALUES (
+    $1, $2, $3,
+    $4, $5, $6,
+    $7, $8
+)
+ON CONFLICT (track_id) DO UPDATE SET
+    integrated_lufs   = EXCLUDED.integrated_lufs,
+    true_peak_dbtp    = EXCLUDED.true_peak_dbtp,
+    loudness_range_lu = EXCLUDED.loudness_range_lu,
+    block_hist_start  = EXCLUDED.block_hist_start,
+    block_hist        = EXCLUDED.block_hist,
+    unreadable        = EXCLUDED.unreadable,
+    analysis_version  = EXCLUDED.analysis_version,
+    analyzed_at       = now()
+`
+
+type UpsertTrackLoudnessParams struct {
+	TrackID         pgtype.UUID
+	IntegratedLufs  *float32
+	TruePeakDbtp    *float32
+	LoudnessRangeLu *float32
+	BlockHistStart  *int16
+	BlockHist       []int32
+	Unreadable      bool
+	AnalysisVersion int16
+}
+
+// Written when the backfill measures a track (#4995). Replaces the row
+// wholesale: a measurement of the old bytes has no standing once the file has
+// changed.
+func (q *Queries) UpsertTrackLoudness(ctx context.Context, arg UpsertTrackLoudnessParams) error {
+	_, err := q.db.Exec(ctx, upsertTrackLoudness,
+		arg.TrackID,
+		arg.IntegratedLufs,
+		arg.TruePeakDbtp,
+		arg.LoudnessRangeLu,
+		arg.BlockHistStart,
+		arg.BlockHist,
+		arg.Unreadable,
+		arg.AnalysisVersion,
+	)
+	return err
+}

internal/db/dbq/models.go

@@ -417,6 +417,13 @@ type LidarrRequest struct {
 	LidarrAddConfirmedAt pgtype.Timestamptz
 }
 
+type LoudnessSetting struct {
+	ID                  bool
+	Enabled             bool
+	BackfillConcurrency int32
+	UpdatedAt           pgtype.Timestamptz
+}
+
 type MissingReacquisition struct {
 	AlbumID       pgtype.UUID
 	Attempts      int32
@@ -713,6 +720,18 @@ type TrackFingerprint struct {
 	ChromaprintLengthSec int32
 }
 
+type TrackLoudness struct {
+	TrackID         pgtype.UUID
+	IntegratedLufs  *float32
+	TruePeakDbtp    *float32
+	LoudnessRangeLu *float32
+	BlockHistStart  *int16
+	BlockHist       []int32
+	Unreadable      bool
+	AnalysisVersion int16
+	AnalyzedAt      pgtype.Timestamptz
+}
+
 type TrackSimilarity struct {
 	TrackAID  pgtype.UUID
 	TrackBID  pgtype.UUID

internal/db/migrations/0065_track_loudness.down.sql

@@ -0,0 +1,2 @@
+DROP TABLE IF EXISTS loudness_settings;
+DROP TABLE IF EXISTS track_loudness;

internal/db/migrations/0065_track_loudness.up.sql

@@ -0,0 +1,65 @@
+-- 0065_track_loudness.up.sql — measured loudness per track, for loudness
+-- normalization (Scribe milestone #464, #4995).
+--
+-- Measured with ffmpeg's EBU R128 filter rather than read from ReplayGain tags:
+-- tags in the wild are written against four different reference levels, and
+-- most files have none. internal/library/loudness.go says how it is measured.
+--
+-- A table of its own rather than columns on tracks, for the reason
+-- track_fingerprints is (0058): tracks is read with SELECT * on the hot path,
+-- and the block histogram is a few hundred integers only album loudness reads.
+--
+-- What a row means, which the backfill depends on:
+--   no row                          never analyzed, or the file changed since
+--   analysis_version < current      measured by an older method; measure again
+--   analysis_version = current      settled until the file changes:
+--     integrated_lufs NOT NULL        measured
+--     integrated_lufs NULL, unreadable false
+--                                     read fine, but no 400 ms block was above
+--                                     the -70 LUFS gate: silence, or too short
+--     unreadable true                 ffmpeg could not decode the file
+-- A failure that says nothing about the file (a timeout, a cancelled pass, a
+-- missing ffmpeg) writes no row, so the backfill tries again.
+CREATE TABLE track_loudness (
+    track_id          uuid        PRIMARY KEY REFERENCES tracks (id) ON DELETE CASCADE,
+    -- Gated integrated loudness (ITU-R BS.1770), mono measured as dual mono.
+    integrated_lufs   real,
+    -- Highest inter-sample peak, from 4x oversampling, in dB relative to full
+    -- scale. NULL for digital silence, whose peak is -inf.
+    true_peak_dbtp    real,
+    -- Loudness range (EBU Tech 3342): how much the loudness moves within the
+    -- track. Not used for gain; kept because it costs nothing here.
+    loudness_range_lu real,
+    -- How many 400 ms gating blocks fell in each 0.1 LU bin. Bin i holds blocks
+    -- measuring -70.0 + (block_hist_start + i) / 10 LUFS; the array is trimmed
+    -- to the first and last non-empty bins. Album loudness is the gated loudness
+    -- of every block on the album, so it is computed from these exactly, with no
+    -- second decode (#4996).
+    block_hist_start  smallint,
+    block_hist        integer[],
+    unreadable        boolean     NOT NULL DEFAULT false,
+    analysis_version  smallint    NOT NULL,
+    analyzed_at       timestamptz NOT NULL DEFAULT now(),
+
+    CONSTRAINT track_loudness_hist_pair
+        CHECK ((block_hist IS NULL) = (block_hist_start IS NULL))
+);
+
+-- Loudness analysis's knobs, in admin Settings. Rule 25: an operator setting is
+-- a database row, changed without a restart. Singleton in the style of
+-- fingerprint_settings (0061).
+CREATE TABLE loudness_settings (
+    id                   boolean PRIMARY KEY DEFAULT true,
+    -- Off stops the background analysis. Tracks already measured keep their
+    -- values, so normalization keeps working for them.
+    enabled              boolean NOT NULL DEFAULT true,
+    -- Files analyzed at once. Each is a full decode, competing with playback
+    -- transcoding for CPU and with streaming for the mount.
+    backfill_concurrency integer NOT NULL DEFAULT 2,
+    updated_at           timestamptz NOT NULL DEFAULT now(),
+
+    CONSTRAINT loudness_settings_singleton CHECK (id = true),
+    CONSTRAINT loudness_settings_concurrency_range
+        CHECK (backfill_concurrency >= 1 AND backfill_concurrency <= 8)
+);
+INSERT INTO loudness_settings (id) VALUES (true) ON CONFLICT (id) DO NOTHING;

internal/db/queries/loudness.sql

@@ -0,0 +1,77 @@
+-- name: UpsertTrackLoudness :exec
+-- Written when the backfill measures a track (#4995). Replaces the row
+-- wholesale: a measurement of the old bytes has no standing once the file has
+-- changed.
+INSERT INTO track_loudness (
+    track_id, integrated_lufs, true_peak_dbtp, loudness_range_lu,
+    block_hist_start, block_hist, unreadable, analysis_version
+) VALUES (
+    sqlc.arg(track_id), sqlc.narg(integrated_lufs), sqlc.narg(true_peak_dbtp),
+    sqlc.narg(loudness_range_lu), sqlc.narg(block_hist_start), sqlc.narg(block_hist),
+    sqlc.arg(unreadable), sqlc.arg(analysis_version)
+)
+ON CONFLICT (track_id) DO UPDATE SET
+    integrated_lufs   = EXCLUDED.integrated_lufs,
+    true_peak_dbtp    = EXCLUDED.true_peak_dbtp,
+    loudness_range_lu = EXCLUDED.loudness_range_lu,
+    block_hist_start  = EXCLUDED.block_hist_start,
+    block_hist        = EXCLUDED.block_hist,
+    unreadable        = EXCLUDED.unreadable,
+    analysis_version  = EXCLUDED.analysis_version,
+    analyzed_at       = now();
+
+-- name: DeleteTrackLoudness :exec
+-- The scan saw new bytes at this path. The stored measurement describes the old
+-- ones, so it goes, and the backfill measures the file again.
+DELETE FROM track_loudness WHERE track_id = $1;
+
+-- name: ListTracksNeedingLoudness :many
+-- The backfill's work queue: tracks with no measurement, or one taken by an
+-- older method. Keyset-paged on id so a pass visits each track at most once;
+-- an inconclusive attempt writes no row, and without the cursor a file that
+-- keeps timing out would be listed again straight away. Missing tracks are
+-- skipped: there is no file to read.
+SELECT t.id, t.file_path, t.duration_ms
+  FROM tracks t
+  LEFT JOIN track_loudness l ON l.track_id = t.id
+ WHERE t.missing_since IS NULL
+   AND (l.track_id IS NULL OR l.analysis_version < sqlc.arg(current_version))
+   AND t.id > sqlc.arg(after_id)
+ ORDER BY t.id
+ LIMIT sqlc.arg(batch_limit);
+
+-- name: GetLoudnessCoverage :one
+-- The admin gauge. measured + silent + unreadable + pending = total. Missing
+-- tracks are excluded, or the gauge could never reach the end.
+SELECT count(*)::bigint AS total,
+       count(*) FILTER (
+           WHERE l.analysis_version >= sqlc.arg(current_version)
+             AND l.integrated_lufs IS NOT NULL
+       )::bigint AS measured,
+       count(*) FILTER (
+           WHERE l.analysis_version >= sqlc.arg(current_version)
+             AND l.integrated_lufs IS NULL AND NOT l.unreadable
+       )::bigint AS silent,
+       count(*) FILTER (
+           WHERE l.analysis_version >= sqlc.arg(current_version)
+             AND l.unreadable
+       )::bigint AS unreadable,
+       count(*) FILTER (
+           WHERE l.track_id IS NULL OR l.analysis_version < sqlc.arg(current_version)
+       )::bigint AS pending
+  FROM tracks t
+  LEFT JOIN track_loudness l ON l.track_id = t.id
+ WHERE t.missing_since IS NULL;
+
+-- name: GetLoudnessSettings :one
+SELECT * FROM loudness_settings WHERE id = true;
+
+-- name: UpdateLoudnessSettings :one
+-- Whole-row write from the admin card; migration 0065's CHECK is the backstop
+-- behind the service's own validation.
+UPDATE loudness_settings
+   SET enabled              = sqlc.arg(enabled),
+       backfill_concurrency = sqlc.arg(backfill_concurrency),
+       updated_at           = now()
+ WHERE id = true
+RETURNING *;