fix: a fresh app install syncs the whole library, not the last 30 days of changes (#6778)

The change feed dropped every row older than 30 days, and an app syncing
from cursor 0 builds its library from the feed, so a fresh install, cleared
data or a 410 reset got only what had changed lately.

The compactor now keeps each entity's latest change however old. It drops a
row once a newer one for the same entity exists, and a delete once it is 30
days old, raising a stored mark (library_changes_trimmed) past it; an app
whose cursor is below the mark is sent back to 0, as before. Migration 0101
adds the index the compactor needs, seeds the mark where the old compactor
left off, and gives every artist, album and track whose latest row is not an
upsert a fresh one, so the rows the old compactor dropped come back. The
backup carries the mark with the feed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/db/dbq/library_changes.sql.go

@@ -11,15 +11,38 @@ import (
 	"github.com/jackc/pgx/v5/pgtype"
 )
 
-const deleteOldLibraryChanges = `-- name: DeleteOldLibraryChanges :execrows
-DELETE FROM library_changes WHERE changed_at < $1
+const deleteOldLibraryDeletes = `-- name: DeleteOldLibraryDeletes :one
+WITH gone AS (
+    DELETE FROM library_changes WHERE op = 'delete' AND changed_at < $1 RETURNING id
+), mark AS (
+    UPDATE library_changes_trimmed
+       SET through = GREATEST(through, (SELECT COALESCE(max(id), 0) FROM gone))
+)
+SELECT count(*)::BIGINT FROM gone
 `
 
-// Removes library_changes rows older than the given cutoff. Run from
-// the sync compactor goroutine on a daily tick. Returns the number of
-// rows deleted so the worker can log meaningful progress.
-func (q *Queries) DeleteOldLibraryChanges(ctx context.Context, changedAt pgtype.Timestamptz) (int64, error) {
-	result, err := q.db.Exec(ctx, deleteOldLibraryChanges, changedAt)
+// Removes deletes older than the cutoff and raises the trimmed mark past the
+// highest one, so an app whose cursor is below it starts over. Returns how
+// many went.
+func (q *Queries) DeleteOldLibraryDeletes(ctx context.Context, changedAt pgtype.Timestamptz) (int64, error) {
+	row := q.db.QueryRow(ctx, deleteOldLibraryDeletes, changedAt)
+	var column_1 int64
+	err := row.Scan(&column_1)
+	return column_1, err
+}
+
+const deleteSupersededLibraryChanges = `-- name: DeleteSupersededLibraryChanges :execrows
+DELETE FROM library_changes old
+ WHERE EXISTS (SELECT 1 FROM library_changes newer
+                WHERE newer.entity_type = old.entity_type
+                  AND newer.entity_id = old.entity_id
+                  AND newer.id > old.id)
+`
+
+// Removes every row a newer row for the same entity has superseded. An app
+// reading from any cursor still meets the newer row, so it loses nothing.
+func (q *Queries) DeleteSupersededLibraryChanges(ctx context.Context) (int64, error) {
+	result, err := q.db.Exec(ctx, deleteSupersededLibraryChanges)
 	if err != nil {
 		return 0, err
 	}
@@ -79,6 +102,19 @@ func (q *Queries) GetLibraryChangesSince(ctx context.Context, arg GetLibraryChan
 	return items, nil
 }
 
+const getLibraryChangesTrimmedThrough = `-- name: GetLibraryChangesTrimmedThrough :one
+SELECT through FROM library_changes_trimmed
+`
+
+// The highest id of a delete the compactor has dropped. A cursor below it may
+// have missed that delete.
+func (q *Queries) GetLibraryChangesTrimmedThrough(ctx context.Context) (int64, error) {
+	row := q.db.QueryRow(ctx, getLibraryChangesTrimmedThrough)
+	var through int64
+	err := row.Scan(&through)
+	return through, err
+}
+
 const getMaxLibraryChangeCursor = `-- name: GetMaxLibraryChangeCursor :one
 SELECT COALESCE(MAX(id), 0)::BIGINT FROM library_changes
 `
@@ -90,17 +126,6 @@ func (q *Queries) GetMaxLibraryChangeCursor(ctx context.Context) (int64, error)
 	return column_1, err
 }
 
-const getMinLibraryChangeCursor = `-- name: GetMinLibraryChangeCursor :one
-SELECT COALESCE(MIN(id), 0)::BIGINT FROM library_changes
-`
-
-func (q *Queries) GetMinLibraryChangeCursor(ctx context.Context) (int64, error) {
-	row := q.db.QueryRow(ctx, getMinLibraryChangeCursor)
-	var column_1 int64
-	err := row.Scan(&column_1)
-	return column_1, err
-}
-
 const insertLibraryChange = `-- name: InsertLibraryChange :exec
 INSERT INTO library_changes (entity_type, entity_id, op)
 VALUES ($1, $2, $3)

internal/db/dbq/models.go

@@ -544,6 +544,11 @@ type LibraryChange struct {
 	ChangedAt  pgtype.Timestamptz
 }
 
+type LibraryChangesTrimmed struct {
+	ID      bool
+	Through int64
+}
+
 type LibrarySetting struct {
 	ID        bool
 	ScanPaths []string

internal/db/migrations/0101_library_changes_keep_latest.down.sql

@@ -0,0 +1,2 @@
+DROP TABLE IF EXISTS library_changes_trimmed;
+DROP INDEX IF EXISTS idx_library_changes_entity;

internal/db/migrations/0101_library_changes_keep_latest.up.sql

@@ -0,0 +1,56 @@
+-- 0101_library_changes_keep_latest.up.sql — the change feed keeps every
+-- entity's latest change, so an app syncing from nothing gets the whole
+-- library.
+--
+-- Until now the compactor dropped every row older than 30 days. An app
+-- syncing from cursor 0 (a fresh install, cleared data, or a reset after a
+-- 410) builds its library from the feed, so it got only what had changed in
+-- the last 30 days. From here the compactor drops a row only once a newer row
+-- for the same entity exists, and drops a delete only once it is 30 days old.
+--
+--   idx_library_changes_entity   finds an entity's rows, newest first, for
+--                                the compactor and the fill below.
+--   library_changes_trimmed      one row: the highest id of a delete the
+--                                compactor has dropped. An app whose cursor
+--                                is below it may have missed that delete, so
+--                                it is told to start over (410). It starts
+--                                where the old compactor left off: everything
+--                                below the oldest remaining row is gone.
+--
+-- Then every artist, album and track whose latest row is not an upsert, or
+-- which has no row at all (the old compactor dropped it), gets one, artists
+-- first so an album never arrives before its artist. Likes and playlists are
+-- left as they are: the apps fetch those from their own endpoints.
+CREATE INDEX idx_library_changes_entity ON library_changes (entity_type, entity_id, id DESC);
+
+CREATE TABLE library_changes_trimmed (
+    id      boolean PRIMARY KEY DEFAULT true CONSTRAINT library_changes_trimmed_one_row CHECK (id),
+    through bigint NOT NULL DEFAULT 0
+);
+
+INSERT INTO library_changes_trimmed (through)
+SELECT COALESCE(min(id), 1) - 1 FROM library_changes;
+
+INSERT INTO library_changes (entity_type, entity_id, op)
+SELECT 'artist', a.id::text, 'upsert'
+  FROM artists a
+ WHERE COALESCE((SELECT c.op FROM library_changes c
+                  WHERE c.entity_type = 'artist' AND c.entity_id = a.id::text
+                  ORDER BY c.id DESC LIMIT 1), 'delete') <> 'upsert'
+ ORDER BY a.id;
+
+INSERT INTO library_changes (entity_type, entity_id, op)
+SELECT 'album', al.id::text, 'upsert'
+  FROM albums al
+ WHERE COALESCE((SELECT c.op FROM library_changes c
+                  WHERE c.entity_type = 'album' AND c.entity_id = al.id::text
+                  ORDER BY c.id DESC LIMIT 1), 'delete') <> 'upsert'
+ ORDER BY al.id;
+
+INSERT INTO library_changes (entity_type, entity_id, op)
+SELECT 'track', t.id::text, 'upsert'
+  FROM tracks t
+ WHERE COALESCE((SELECT c.op FROM library_changes c
+                  WHERE c.entity_type = 'track' AND c.entity_id = t.id::text
+                  ORDER BY c.id DESC LIMIT 1), 'delete') <> 'upsert'
+ ORDER BY t.id;

internal/db/queries/library_changes.sql

@@ -28,11 +28,28 @@ SELECT COALESCE(MAX(id), 0)::BIGINT FROM library_changes;
 -- sequence stands at the backup's last change.
 SELECT COALESCE(pg_sequence_last_value(pg_get_serial_sequence('library_changes', 'id')::regclass), 0)::BIGINT;
 
--- name: GetMinLibraryChangeCursor :one
-SELECT COALESCE(MIN(id), 0)::BIGINT FROM library_changes;
+-- name: GetLibraryChangesTrimmedThrough :one
+-- The highest id of a delete the compactor has dropped. A cursor below it may
+-- have missed that delete.
+SELECT through FROM library_changes_trimmed;
 
--- name: DeleteOldLibraryChanges :execrows
--- Removes library_changes rows older than the given cutoff. Run from
--- the sync compactor goroutine on a daily tick. Returns the number of
--- rows deleted so the worker can log meaningful progress.
-DELETE FROM library_changes WHERE changed_at < $1;
+-- name: DeleteSupersededLibraryChanges :execrows
+-- Removes every row a newer row for the same entity has superseded. An app
+-- reading from any cursor still meets the newer row, so it loses nothing.
+DELETE FROM library_changes old
+ WHERE EXISTS (SELECT 1 FROM library_changes newer
+                WHERE newer.entity_type = old.entity_type
+                  AND newer.entity_id = old.entity_id
+                  AND newer.id > old.id);
+
+-- name: DeleteOldLibraryDeletes :one
+-- Removes deletes older than the cutoff and raises the trimmed mark past the
+-- highest one, so an app whose cursor is below it starts over. Returns how
+-- many went.
+WITH gone AS (
+    DELETE FROM library_changes WHERE op = 'delete' AND changed_at < $1 RETURNING id
+), mark AS (
+    UPDATE library_changes_trimmed
+       SET through = GREATEST(through, (SELECT COALESCE(max(id), 0) FROM gone))
+)
+SELECT count(*)::BIGINT FROM gone;