M462 security hardening and M464 loudness analysis (steps 1–3) #135

Merged
bvandeusen merged 18 commits from dev into main 2026-10-06 14:19:58 -04:00
10 changed files with 722 additions and 19 deletions
Showing only changes of commit c2f81bf8df - Show all commits
+169
View File
@@ -11,6 +11,24 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const deleteOrphanAlbumLoudness = `-- name: DeleteOrphanAlbumLoudness :execrows
DELETE FROM album_loudness a
WHERE NOT EXISTS (
SELECT 1 FROM tracks t WHERE t.album_id = a.album_id AND t.missing_since IS NULL
)
`
// Album rows whose album has no present track left (every file missing). The
// album row itself survives a missing file, so its loudness would otherwise
// stay behind describing tracks that are gone; it is recomputed if they return.
func (q *Queries) DeleteOrphanAlbumLoudness(ctx context.Context) (int64, error) {
result, err := q.db.Exec(ctx, deleteOrphanAlbumLoudness)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const deleteTrackLoudness = `-- name: DeleteTrackLoudness :exec
DELETE FROM track_loudness WHERE track_id = $1
`
@@ -83,6 +101,120 @@ func (q *Queries) GetLoudnessSettings(ctx context.Context) (LoudnessSetting, err
return i, err
}
const listAlbumLoudnessInputs = `-- name: ListAlbumLoudnessInputs :many
SELECT t.id,
(l.track_id IS NOT NULL)::boolean AS settled,
l.true_peak_dbtp,
l.block_hist_start,
l.block_hist
FROM tracks t
LEFT JOIN track_loudness l
ON l.track_id = t.id AND l.analysis_version >= $1
WHERE t.album_id = $2
AND t.missing_since IS NULL
`
type ListAlbumLoudnessInputsParams struct {
CurrentVersion int16
AlbumID pgtype.UUID
}
type ListAlbumLoudnessInputsRow struct {
ID pgtype.UUID
Settled bool
TruePeakDbtp *float32
BlockHistStart *int16
BlockHist []int32
}
// Every present track on one album with its current measurement, if any.
// settled is false for a track not yet measured at the current version.
func (q *Queries) ListAlbumLoudnessInputs(ctx context.Context, arg ListAlbumLoudnessInputsParams) ([]ListAlbumLoudnessInputsRow, error) {
rows, err := q.db.Query(ctx, listAlbumLoudnessInputs, arg.CurrentVersion, arg.AlbumID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListAlbumLoudnessInputsRow
for rows.Next() {
var i ListAlbumLoudnessInputsRow
if err := rows.Scan(
&i.ID,
&i.Settled,
&i.TruePeakDbtp,
&i.BlockHistStart,
&i.BlockHist,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listAlbumsNeedingLoudness = `-- name: ListAlbumsNeedingLoudness :many
WITH present AS (
SELECT t.album_id,
md5(string_agg(
t.id::text || ':' || coalesce(
l.analysis_version::text || '@' || l.analyzed_at::text, '-'),
',' ORDER BY t.id)) AS digest
FROM tracks t
LEFT JOIN track_loudness l
ON l.track_id = t.id AND l.analysis_version >= $3::smallint
WHERE t.missing_since IS NULL
GROUP BY t.album_id
)
SELECT c.album_id, c.digest::text AS digest
FROM present c
LEFT JOIN album_loudness a ON a.album_id = c.album_id
WHERE (a.album_id IS NULL OR a.inputs_digest <> c.digest)
-- Casts: sqlc cannot infer a parameter's type through a CTE alias.
AND c.album_id > $1::uuid
ORDER BY c.album_id
LIMIT $2::integer
`
type ListAlbumsNeedingLoudnessParams struct {
AfterID pgtype.UUID
BatchLimit int32
CurrentVersion int16
}
type ListAlbumsNeedingLoudnessRow struct {
AlbumID pgtype.UUID
Digest string
}
// The album pass's work queue (#4996): albums whose present tracks or their
// measurements have changed since album loudness was last computed, or that
// never had it. The digest is over every present track's id and the
// measurement it holds at the current version ('-' for none), so a track
// joining, leaving or being re-measured changes it. Keyset-paged on album id
// so a pass ends even if storing one album keeps failing.
func (q *Queries) ListAlbumsNeedingLoudness(ctx context.Context, arg ListAlbumsNeedingLoudnessParams) ([]ListAlbumsNeedingLoudnessRow, error) {
rows, err := q.db.Query(ctx, listAlbumsNeedingLoudness, arg.AfterID, arg.BatchLimit, arg.CurrentVersion)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListAlbumsNeedingLoudnessRow
for rows.Next() {
var i ListAlbumsNeedingLoudnessRow
if err := rows.Scan(&i.AlbumID, &i.Digest); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listTracksNeedingLoudness = `-- name: ListTracksNeedingLoudness :many
SELECT t.id, t.file_path, t.duration_ms
FROM tracks t
@@ -159,6 +291,43 @@ func (q *Queries) UpdateLoudnessSettings(ctx context.Context, arg UpdateLoudness
return i, err
}
const upsertAlbumLoudness = `-- name: UpsertAlbumLoudness :exec
INSERT INTO album_loudness (
album_id, integrated_lufs, true_peak_dbtp, tracks_total, tracks_settled, inputs_digest
) VALUES (
$1, $2, $3,
$4, $5, $6
)
ON CONFLICT (album_id) DO UPDATE SET
integrated_lufs = EXCLUDED.integrated_lufs,
true_peak_dbtp = EXCLUDED.true_peak_dbtp,
tracks_total = EXCLUDED.tracks_total,
tracks_settled = EXCLUDED.tracks_settled,
inputs_digest = EXCLUDED.inputs_digest,
computed_at = now()
`
type UpsertAlbumLoudnessParams struct {
AlbumID pgtype.UUID
IntegratedLufs *float32
TruePeakDbtp *float32
TracksTotal int32
TracksSettled int32
InputsDigest string
}
func (q *Queries) UpsertAlbumLoudness(ctx context.Context, arg UpsertAlbumLoudnessParams) error {
_, err := q.db.Exec(ctx, upsertAlbumLoudness,
arg.AlbumID,
arg.IntegratedLufs,
arg.TruePeakDbtp,
arg.TracksTotal,
arg.TracksSettled,
arg.InputsDigest,
)
return err
}
const upsertTrackLoudness = `-- name: UpsertTrackLoudness :exec
INSERT INTO track_loudness (
track_id, integrated_lufs, true_peak_dbtp, loudness_range_lu,
+10
View File
@@ -201,6 +201,16 @@ type Album struct {
CoverArtSourcesVersion int32
}
type AlbumLoudness struct {
AlbumID pgtype.UUID
IntegratedLufs *float32
TruePeakDbtp *float32
TracksTotal int32
TracksSettled int32
InputsDigest string
ComputedAt pgtype.Timestamptz
}
type Artist struct {
ID pgtype.UUID
Name string
@@ -0,0 +1 @@
DROP TABLE IF EXISTS album_loudness;
@@ -0,0 +1,31 @@
-- 0066_album_loudness.up.sql — album loudness for album-mode normalization
-- (Scribe milestone #464, #4996).
--
-- An album's loudness is the gated loudness of every 400 ms block on the album,
-- not an average of its tracks' values: a quiet interlude and a loud single
-- should keep their difference when the album plays in order. It is computed
-- from the per-track block histograms in track_loudness (0065), summed, so no
-- audio is decoded again.
--
-- A derived value, recomputed by the loudness worker whenever its inputs
-- change. inputs_digest is an md5 over the album's present tracks and the
-- measurement each holds; the worker recomputes every album whose stored digest
-- no longer matches. That one comparison covers every way membership changes
-- (a scan moving a track between albums, a duplicate merge, a delete, a file
-- going missing or coming back) without hooking each of them.
CREATE TABLE album_loudness (
album_id uuid PRIMARY KEY REFERENCES albums (id) ON DELETE CASCADE,
-- NULL until every present track has a settled measurement: an album
-- leveled from half its tracks would jump when the rest arrived. Also NULL
-- for an album with no block above the gate. Clients fall back to track
-- gain while it is NULL.
integrated_lufs real,
-- The loudest true peak of any track on the album, so album gain is held
-- to the headroom of its loudest track.
true_peak_dbtp real,
tracks_total integer NOT NULL,
-- Tracks with a settled measurement (measured, silent or unreadable).
tracks_settled integer NOT NULL,
inputs_digest text NOT NULL,
computed_at timestamptz NOT NULL DEFAULT now()
);
+66
View File
@@ -75,3 +75,69 @@ UPDATE loudness_settings
updated_at = now()
WHERE id = true
RETURNING *;
-- name: ListAlbumsNeedingLoudness :many
-- The album pass's work queue (#4996): albums whose present tracks or their
-- measurements have changed since album loudness was last computed, or that
-- never had it. The digest is over every present track's id and the
-- measurement it holds at the current version ('-' for none), so a track
-- joining, leaving or being re-measured changes it. Keyset-paged on album id
-- so a pass ends even if storing one album keeps failing.
WITH present AS (
SELECT t.album_id,
md5(string_agg(
t.id::text || ':' || coalesce(
l.analysis_version::text || '@' || l.analyzed_at::text, '-'),
',' ORDER BY t.id)) AS digest
FROM tracks t
LEFT JOIN track_loudness l
ON l.track_id = t.id AND l.analysis_version >= sqlc.arg(current_version)::smallint
WHERE t.missing_since IS NULL
GROUP BY t.album_id
)
SELECT c.album_id, c.digest::text AS digest
FROM present c
LEFT JOIN album_loudness a ON a.album_id = c.album_id
WHERE (a.album_id IS NULL OR a.inputs_digest <> c.digest)
-- Casts: sqlc cannot infer a parameter's type through a CTE alias.
AND c.album_id > sqlc.arg(after_id)::uuid
ORDER BY c.album_id
LIMIT sqlc.arg(batch_limit)::integer;
-- name: ListAlbumLoudnessInputs :many
-- Every present track on one album with its current measurement, if any.
-- settled is false for a track not yet measured at the current version.
SELECT t.id,
(l.track_id IS NOT NULL)::boolean AS settled,
l.true_peak_dbtp,
l.block_hist_start,
l.block_hist
FROM tracks t
LEFT JOIN track_loudness l
ON l.track_id = t.id AND l.analysis_version >= sqlc.arg(current_version)
WHERE t.album_id = sqlc.arg(album_id)
AND t.missing_since IS NULL;
-- name: UpsertAlbumLoudness :exec
INSERT INTO album_loudness (
album_id, integrated_lufs, true_peak_dbtp, tracks_total, tracks_settled, inputs_digest
) VALUES (
sqlc.arg(album_id), sqlc.narg(integrated_lufs), sqlc.narg(true_peak_dbtp),
sqlc.arg(tracks_total), sqlc.arg(tracks_settled), sqlc.arg(inputs_digest)
)
ON CONFLICT (album_id) DO UPDATE SET
integrated_lufs = EXCLUDED.integrated_lufs,
true_peak_dbtp = EXCLUDED.true_peak_dbtp,
tracks_total = EXCLUDED.tracks_total,
tracks_settled = EXCLUDED.tracks_settled,
inputs_digest = EXCLUDED.inputs_digest,
computed_at = now();
-- name: DeleteOrphanAlbumLoudness :execrows
-- Album rows whose album has no present track left (every file missing). The
-- album row itself survives a missing file, so its loudness would otherwise
-- stay behind describing tracks that are gone; it is recomputed if they return.
DELETE FROM album_loudness a
WHERE NOT EXISTS (
SELECT 1 FROM tracks t WHERE t.album_id = a.album_id AND t.missing_since IS NULL
);
+1
View File
@@ -92,6 +92,7 @@ var dataTables = []string{
"duplicate_sweeps",
"track_fingerprints", // M400
"track_loudness", // M464
"album_loudness", // M464
"tracks",
"albums",
"artists",
+162
View File
@@ -0,0 +1,162 @@
package library
import (
"context"
"fmt"
"github.com/jackc/pgx/v5/pgtype"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
// Album loudness (M464 #4996).
//
// Album-mode normalization plays a whole album at one gain, so the quiet
// interlude stays quieter than the single it sits between. That gain comes
// from the album's loudness: BS.1770's gated loudness over every block on the
// album, which is what summing the tracks' block histograms and gating the sum
// computes. An average of the tracks' values is not the same thing: gating
// over the whole album drops a near-silent hidden track, where averaging
// would let it drag the album quieter.
//
// The values are derived, so they are recomputed rather than maintained: each
// worker tick lists the albums whose inputs digest has moved (see
// ListAlbumsNeedingLoudness) and recomputes those from the stored histograms.
// albumLoudnessBatch is how many albums one query hands the album pass.
// Recomputing an album is a few small reads and arithmetic, no decode.
const albumLoudnessBatch = 200
// albumLoudness is one album's computed values.
type albumLoudness struct {
// integratedLUFS is nil until every track is settled, or when no block on
// the album passed the gate.
integratedLUFS *float32
truePeakDBTP *float32
total, settled int32
}
// computeAlbumLoudness sums the present tracks' histograms and gates the sum.
func computeAlbumLoudness(inputs []dbq.ListAlbumLoudnessInputsRow) albumLoudness {
a := albumLoudness{total: int32(len(inputs))}
hists := make([]blockHistogram, 0, len(inputs))
for _, in := range inputs {
if !in.Settled {
continue
}
a.settled++
if in.TruePeakDbtp != nil && (a.truePeakDBTP == nil || *in.TruePeakDbtp > *a.truePeakDBTP) {
peak := *in.TruePeakDbtp
a.truePeakDBTP = &peak
}
if in.BlockHistStart != nil && len(in.BlockHist) > 0 {
hists = append(hists, blockHistogram{start: *in.BlockHistStart, counts: in.BlockHist})
}
}
// Leveling from part of an album would change its gain as the rest is
// measured, audibly, mid-listen. Wait for all of it.
if a.total == 0 || a.settled < a.total {
return a
}
if lufs, ok := mergeHistograms(hists).gatedLoudness(); ok {
v := float32(lufs)
a.integratedLUFS = &v
}
return a
}
// mergeHistograms sums histograms that may cover different bin ranges into
// one, trimmed to the occupied range. A histogram reaching past the bin range
// (only a corrupt row could) is clipped rather than trusted.
func mergeHistograms(hs []blockHistogram) blockHistogram {
var bins [loudnessHistBins]int32
for _, h := range hs {
for i, c := range h.counts {
if b := int(h.start) + i; b >= 0 && b < loudnessHistBins {
bins[b] += c
}
}
}
return trimBins(&bins)
}
// AlbumLoudnessResult tallies one album pass.
type AlbumLoudnessResult struct {
Recomputed int
Leveled int // stored with an album loudness
Waiting int // stored without one: a track is not yet measured
Failed int
Orphans int64 // rows dropped because the album has no present track
}
// albumPass recomputes every album whose inputs changed, keyset-paged on album
// id so a pass ends even when one album keeps failing to store.
//
// The digest stored is the one the list query computed. If a track changes
// between that query and the read of its inputs, the stored digest is already
// stale and the next pass recomputes the album again, so the values always
// converge on the inputs.
func (w *LoudnessBackfillWorker) albumPass(ctx context.Context) (AlbumLoudnessResult, error) {
q := dbq.New(w.pool)
var res AlbumLoudnessResult
after := pgtype.UUID{Valid: true}
for {
if err := ctx.Err(); err != nil {
return res, err
}
rows, err := q.ListAlbumsNeedingLoudness(ctx, dbq.ListAlbumsNeedingLoudnessParams{
CurrentVersion: loudnessVersion,
AfterID: after,
BatchLimit: w.albumBatch,
})
if err != nil {
return res, fmt.Errorf("list albums needing loudness: %w", err)
}
if len(rows) == 0 {
break
}
for _, row := range rows {
res.Recomputed++
if err := storeAlbumLoudness(ctx, q, row.AlbumID, row.Digest, &res); err != nil {
res.Failed++
w.logger.Warn("album loudness: recompute failed", "album_id", row.AlbumID, "err", err)
}
}
after = rows[len(rows)-1].AlbumID
}
n, err := q.DeleteOrphanAlbumLoudness(ctx)
if err != nil {
return res, fmt.Errorf("drop orphan album loudness: %w", err)
}
res.Orphans = n
return res, nil
}
func storeAlbumLoudness(
ctx context.Context, q *dbq.Queries, albumID pgtype.UUID, digest string, res *AlbumLoudnessResult,
) error {
inputs, err := q.ListAlbumLoudnessInputs(ctx, dbq.ListAlbumLoudnessInputsParams{
CurrentVersion: loudnessVersion,
AlbumID: albumID,
})
if err != nil {
return fmt.Errorf("read inputs: %w", err)
}
a := computeAlbumLoudness(inputs)
if err := q.UpsertAlbumLoudness(ctx, dbq.UpsertAlbumLoudnessParams{
AlbumID: albumID,
IntegratedLufs: a.integratedLUFS,
TruePeakDbtp: a.truePeakDBTP,
TracksTotal: a.total,
TracksSettled: a.settled,
InputsDigest: digest,
}); err != nil {
return fmt.Errorf("store: %w", err)
}
if a.integratedLUFS != nil {
res.Leveled++
} else {
res.Waiting++
}
return nil
}
+245
View File
@@ -0,0 +1,245 @@
package library
import (
"context"
"io"
"log/slog"
"math"
"path/filepath"
"testing"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
// histAt builds a histogram of n blocks all at one loudness.
func histAt(lufs float64, n int32) blockHistogram {
return blockHistogram{
start: int16(math.Round((lufs - loudnessHistFloor) * loudnessHistPerLU)),
counts: []int32{n},
}
}
func TestMergeHistograms_SumsAcrossDifferentRanges(t *testing.T) {
a := blockHistogram{start: 500, counts: []int32{1, 0, 2}} // bins 500..502
b := blockHistogram{start: 501, counts: []int32{4, 0, 0, 5}} // bins 501..504
got := mergeHistograms([]blockHistogram{a, b})
want := blockHistogram{start: 500, counts: []int32{1, 4, 2, 0, 5}}
if got.start != want.start || len(got.counts) != len(want.counts) {
t.Fatalf("merged = %+v, want %+v", got, want)
}
for i := range want.counts {
if got.counts[i] != want.counts[i] {
t.Fatalf("merged = %+v, want %+v", got, want)
}
}
// Bins past the range come only from a corrupt row; they are dropped, not
// allowed to index out of the array.
bad := blockHistogram{start: loudnessHistBins - 1, counts: []int32{1, 9}}
if got := mergeHistograms([]blockHistogram{bad}); len(got.counts) != 1 || got.counts[0] != 1 {
t.Errorf("out-of-range bin not dropped: %+v", got)
}
if !mergeHistograms(nil).empty() {
t.Errorf("merging nothing gave a non-empty histogram")
}
}
func input(settled bool, peak *float32, h blockHistogram) dbq.ListAlbumLoudnessInputsRow {
row := dbq.ListAlbumLoudnessInputsRow{Settled: settled, TruePeakDbtp: peak}
if !h.empty() {
start := h.start
row.BlockHistStart = &start
row.BlockHist = h.counts
}
return row
}
func TestComputeAlbumLoudness(t *testing.T) {
f := func(v float32) *float32 { return &v }
// Album loudness gates over the whole album. A near-silent hidden track is
// dropped by the relative gate, where averaging the tracks' values would
// have let it pull the album 15 LU quieter.
a := computeAlbumLoudness([]dbq.ListAlbumLoudnessInputsRow{
input(true, f(-1.0), histAt(-10, 1800)),
input(true, f(-0.2), histAt(-10, 2400)),
input(true, f(-30), histAt(-40, 600)),
})
if a.integratedLUFS == nil || math.Abs(float64(*a.integratedLUFS)-(-10)) > 0.01 {
t.Errorf("album loudness = %v, want -10 (the hidden track gated out)", a.integratedLUFS)
}
if a.truePeakDBTP == nil || *a.truePeakDBTP != -0.2 {
t.Errorf("album peak = %v, want the loudest track's -0.2", a.truePeakDBTP)
}
if a.total != 3 || a.settled != 3 {
t.Errorf("total/settled = %d/%d, want 3/3", a.total, a.settled)
}
// Energy, not an average of LUFS: two equally long tracks at -8 and -14
// make an album nearer the louder one than the midpoint (-11): -10.037.
a = computeAlbumLoudness([]dbq.ListAlbumLoudnessInputsRow{
input(true, nil, histAt(-8, 1000)),
input(true, nil, histAt(-14, 1000)),
})
if a.integratedLUFS == nil || math.Abs(float64(*a.integratedLUFS)-(-10.037)) > 0.01 {
t.Errorf("album loudness = %v, want -10.037", a.integratedLUFS)
}
// One track not yet measured: no album value until it is.
a = computeAlbumLoudness([]dbq.ListAlbumLoudnessInputsRow{
input(true, f(-1), histAt(-10, 100)),
input(false, nil, blockHistogram{}),
})
if a.integratedLUFS != nil || a.settled != 1 || a.total != 2 {
t.Errorf("partly measured album = %+v, want no loudness, 1 of 2 settled", a)
}
// Settled without blocks (silent, or unreadable): counted as settled and
// leveled from the rest.
a = computeAlbumLoudness([]dbq.ListAlbumLoudnessInputsRow{
input(true, f(-3), histAt(-12, 100)),
input(true, nil, blockHistogram{}),
})
if a.integratedLUFS == nil || math.Abs(float64(*a.integratedLUFS)-(-12)) > 0.01 {
t.Errorf("album with a silent track = %v, want -12 from the other track", a.integratedLUFS)
}
// Every track silent: settled, but nothing to level by.
a = computeAlbumLoudness([]dbq.ListAlbumLoudnessInputsRow{input(true, nil, blockHistogram{})})
if a.integratedLUFS != nil || a.settled != 1 {
t.Errorf("all-silent album = %+v, want no loudness", a)
}
}
// TestAlbumLoudness_Integration pins that the album pass computes from the
// stored histograms, does nothing when nothing changed, and recomputes on
// each kind of membership change.
func TestAlbumLoudness_Integration(t *testing.T) {
pool := newPool(t)
ctx := context.Background()
q := dbq.New(pool)
dir := t.TempDir()
first, album, artist := seedTrack(t, pool, filepath.Join(dir, "a.mp3"))
addTrack := func(name string, albumID dbq.Album) dbq.Track {
t.Helper()
tr, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
Title: name, AlbumID: albumID.ID, ArtistID: artist.ID,
DurationMs: 1000, FilePath: filepath.Join(dir, name+".mp3"), FileSize: 100, FileFormat: "mp3",
})
if err != nil {
t.Fatalf("track %s: %v", name, err)
}
return tr
}
measure := func(tr dbq.Track, lufs float64, peak float32) {
t.Helper()
h := histAt(lufs, 100)
l := float32(lufs)
if err := q.UpsertTrackLoudness(ctx, dbq.UpsertTrackLoudnessParams{
TrackID: tr.ID, IntegratedLufs: &l, TruePeakDbtp: &peak,
BlockHistStart: &h.start, BlockHist: h.counts, AnalysisVersion: loudnessVersion,
}); err != nil {
t.Fatalf("measure %s: %v", tr.Title, err)
}
}
read := func() (lufs, peak *float32, total, settled int32) {
t.Helper()
if err := pool.QueryRow(ctx,
"SELECT integrated_lufs, true_peak_dbtp, tracks_total, tracks_settled FROM album_loudness WHERE album_id = $1",
album.ID).Scan(&lufs, &peak, &total, &settled); err != nil {
t.Fatalf("read album loudness: %v", err)
}
return
}
w := NewLoudnessBackfillWorker(pool, slog.New(slog.NewTextHandler(io.Discard, nil)), nil)
w.albumBatch = 1 // forces the keyset cursor across queries
pass := func() AlbumLoudnessResult {
t.Helper()
res, err := w.albumPass(ctx)
if err != nil {
t.Fatalf("album pass: %v", err)
}
return res
}
second := addTrack("b", album)
measure(first, -10, -1)
// 1. One track unmeasured: the album is stored, waiting, without loudness.
if res := pass(); res.Recomputed != 1 || res.Waiting != 1 {
t.Fatalf("first pass = %+v, want 1 recomputed, waiting", res)
}
if lufs, _, total, settled := read(); lufs != nil || total != 2 || settled != 1 {
t.Fatalf("waiting album = lufs %v, %d/%d settled; want nil, 1/2", lufs, settled, total)
}
// 2. Measuring the second track changes the digest; the album is leveled.
measure(second, -10, -0.5)
if res := pass(); res.Recomputed != 1 || res.Leveled != 1 {
t.Fatalf("second pass = %+v, want 1 leveled", res)
}
lufs, peak, _, _ := read()
if lufs == nil || math.Abs(float64(*lufs)-(-10)) > 0.01 || peak == nil || *peak != -0.5 {
t.Fatalf("leveled album = lufs %v peak %v, want -10 and -0.5", lufs, peak)
}
// 3. Nothing changed: nothing recomputed.
if res := pass(); res.Recomputed != 0 {
t.Fatalf("idle pass = %+v, want nothing recomputed", res)
}
// 4. A track joins (here, retagged onto this album): recomputed.
other, err := q.UpsertAlbum(ctx, dbq.UpsertAlbumParams{Title: "Other", SortTitle: "Other", ArtistID: artist.ID})
if err != nil {
t.Fatalf("other album: %v", err)
}
loud := addTrack("loud", other)
measure(loud, -4, 0.3)
pass()
if _, err := pool.Exec(ctx, "UPDATE tracks SET album_id = $1 WHERE id = $2", album.ID, loud.ID); err != nil {
t.Fatalf("move track: %v", err)
}
if res := pass(); res.Recomputed < 1 {
t.Fatalf("pass after a track joined = %+v, want a recompute", res)
}
// Two tracks at -10 and one at -4, equally long: -7.003 by energy.
if lufs, peak, total, _ := read(); total != 3 || lufs == nil || math.Abs(float64(*lufs)-(-7.003)) > 0.01 ||
peak == nil || *peak != 0.3 {
t.Fatalf("album after a loud track joined = lufs %v peak %v total %d", lufs, peak, total)
}
// 5. A track's file goes missing: recomputed without it.
if _, err := pool.Exec(ctx, "UPDATE tracks SET missing_since = now() WHERE id = $1", loud.ID); err != nil {
t.Fatalf("mark missing: %v", err)
}
pass()
if lufs, peak, total, _ := read(); total != 2 || lufs == nil || math.Abs(float64(*lufs)-(-10)) > 0.01 ||
peak == nil || *peak != -0.5 {
t.Fatalf("album after the loud track went missing = lufs %v peak %v total %d", lufs, peak, total)
}
// 6. A track is deleted (as a duplicate merge does): recomputed.
if _, err := pool.Exec(ctx, "DELETE FROM tracks WHERE id = $1", second.ID); err != nil {
t.Fatalf("delete track: %v", err)
}
pass()
if _, _, total, _ := read(); total != 1 {
t.Fatalf("album after a delete has %d tracks, want 1", total)
}
// 7. Every remaining track missing: the album row is dropped, not left
// describing tracks that are gone.
if _, err := pool.Exec(ctx, "UPDATE tracks SET missing_since = now() WHERE id = $1", first.ID); err != nil {
t.Fatalf("mark missing: %v", err)
}
if res := pass(); res.Orphans != 1 {
t.Fatalf("pass with every track missing = %+v, want 1 orphan dropped", res)
}
var n int
if err := pool.QueryRow(ctx, "SELECT count(*) FROM album_loudness WHERE album_id = $1", album.ID).Scan(&n); err != nil {
t.Fatalf("count: %v", err)
}
if n != 0 {
t.Fatalf("album loudness row survived every track going missing")
}
}
+12 -8
View File
@@ -220,7 +220,6 @@ var (
// histogram, and the closing summary.
type ebur128Parser struct {
bins [loudnessHistBins]int32
blocks int
inSummary bool
summary map[string]string
lastLines []string
@@ -273,7 +272,6 @@ func (p *ebur128Parser) line(raw string) {
}
bin := int(math.Round((v - loudnessHistFloor) * loudnessHistPerLU))
p.bins[min(bin, loudnessHistBins-1)]++
p.blocks++
}
func (p *ebur128Parser) tail() string {
@@ -281,18 +279,24 @@ func (p *ebur128Parser) tail() string {
}
func (p *ebur128Parser) histogram() blockHistogram {
if p.blocks == 0 {
return blockHistogram{}
}
return trimBins(&p.bins)
}
// trimBins turns a full-range bin array into a histogram trimmed to its
// occupied bins; an empty array gives an empty histogram.
func trimBins(bins *[loudnessHistBins]int32) blockHistogram {
first, last := 0, loudnessHistBins-1
for p.bins[first] == 0 {
for first <= last && bins[first] == 0 {
first++
}
for p.bins[last] == 0 {
if first > last {
return blockHistogram{}
}
for bins[last] == 0 {
last--
}
counts := make([]int32, last-first+1)
copy(counts, p.bins[first:last+1])
copy(counts, bins[first:last+1])
return blockHistogram{start: int16(first), counts: counts}
}
+25 -11
View File
@@ -64,11 +64,12 @@ func (r *BackfillLoudnessResult) add(o loudnessOutcome) {
// LoudnessBackfillWorker measures every track that has no current measurement.
type LoudnessBackfillWorker struct {
pool *pgxpool.Pool
logger *slog.Logger
settings *LoudnessSettingsService
tick time.Duration
batch int32
pool *pgxpool.Pool
logger *slog.Logger
settings *LoudnessSettingsService
tick time.Duration
batch int32
albumBatch int32
// analyze is a field so an integration test pins which tracks a pass
// touches, not what ffmpeg prints.
analyze func(ctx context.Context, path string, durationMs int32) loudnessResult
@@ -80,12 +81,13 @@ func NewLoudnessBackfillWorker(
pool *pgxpool.Pool, logger *slog.Logger, settings *LoudnessSettingsService,
) *LoudnessBackfillWorker {
return &LoudnessBackfillWorker{
pool: pool,
logger: logger,
settings: settings,
tick: loudnessBackfillTick,
batch: loudnessBackfillBatch,
analyze: computeLoudness,
pool: pool,
logger: logger,
settings: settings,
tick: loudnessBackfillTick,
batch: loudnessBackfillBatch,
albumBatch: albumLoudnessBatch,
analyze: computeLoudness,
}
}
@@ -121,6 +123,18 @@ func (w *LoudnessBackfillWorker) runOnce(ctx context.Context) {
"processed", res.Processed, "measured", res.Measured, "silent", res.Silent,
"unreadable", res.Unreadable, "inconclusive", res.Inconclusive)
}
// Album loudness follows the tracks (#4996). It runs even with analysis
// switched off: it decodes nothing, and membership still changes as the
// library does.
albums, err := w.albumPass(ctx)
if err != nil && ctx.Err() == nil {
w.logger.Warn("album loudness: pass failed", "err", err, "recomputed", albums.Recomputed)
}
if albums.Recomputed > 0 || albums.Orphans > 0 {
w.logger.Info("album loudness: pass complete",
"recomputed", albums.Recomputed, "leveled", albums.Leveled,
"waiting", albums.Waiting, "failed", albums.Failed, "orphans", albums.Orphans)
}
}
// pass walks every track needing a measurement once, keyset-paged on id. The