fix: a fresh app install syncs the whole library, not the last 30 days of changes (#6778)

The change feed dropped every row older than 30 days, and an app syncing
from cursor 0 builds its library from the feed, so a fresh install, cleared
data or a 410 reset got only what had changed lately.

The compactor now keeps each entity's latest change however old. It drops a
row once a newer one for the same entity exists, and a delete once it is 30
days old, raising a stored mark (library_changes_trimmed) past it; an app
whose cursor is below the mark is sent back to 0, as before. Migration 0101
adds the index the compactor needs, seeds the mark where the old compactor
left off, and gives every artist, album and track whose latest row is not an
upsert a fresh one, so the rows the old compactor dropped come back. The
backup carries the mark with the feed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/sync/compactor.go

@@ -1,17 +1,22 @@
 // Periodic compactor for the library_changes log. Every mutation in the
-// library writes one row, so the table grows unbounded without a retention
-// policy. This worker trims rows older than CompactorRetention on a daily
-// tick.
+// library writes one row, so the log grows without bound unless trimmed. This
+// worker trims it daily, keeping each entity's latest row:
 //
-// Why 30 days: clients keep a cursor and pull deltas from the last seen id.
-// A client that checks in at least once per retention window never sees its
-// cursor go stale; one offline longer than that gets the /api/library/sync
-// 410 and does a full resync.
+//   - A row a newer row for the same entity has superseded goes at once. An
+//     app reading from any cursor still meets the newer row.
+//   - A delete goes once it is CompactorRetention old, and raises the trimmed
+//     mark (library_changes_trimmed) past its id. An app whose cursor is below
+//     the mark may have missed it, so /api/library/sync answers 410 and the
+//     app syncs again from 0.
+//   - The latest upsert of an entity that still exists stays, however old. An
+//     app syncing from 0 (a fresh install, or after a 410) builds its whole
+//     library from the log, so it must hold every artist, album and track.
 
 package sync
 
 import (
 	"context"
+	"fmt"
 	"log/slog"
 	"time"
 
@@ -21,9 +26,9 @@ import (
 	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
 )
 
-// CompactorRetention is the age past which library_changes rows are
-// dropped. Mutable for tests; production runs should not change it
-// without considering the corresponding client cursor-stale fallback.
+// CompactorRetention is how long a delete stays in the log. An app that
+// checks in at least once in that window never misses one; one away longer
+// is sent back to 0. Mutable for tests.
 var CompactorRetention = 30 * 24 * time.Hour
 
 // CompactorInterval is how often the worker checks. Daily is enough —
@@ -31,8 +36,8 @@ var CompactorRetention = 30 * 24 * time.Hour
 // active library, well below a size where intra-day trimming matters.
 var CompactorInterval = 24 * time.Hour
 
-// Compactor is a background worker that periodically deletes
-// library_changes rows older than CompactorRetention.
+// Compactor is a background worker that trims the library_changes log down
+// to each entity's latest change.
 type Compactor struct {
 	pool     *pgxpool.Pool
 	logger   *slog.Logger
@@ -65,18 +70,28 @@ func (c *Compactor) Run(ctx context.Context) {
 	}
 }
 
-// tickOnce deletes rows older than CompactorRetention and logs the
-// row count. Errors are logged at WARN; we never abort the loop on
-// transient DB failures.
+// tickOnce trims the log once and logs what went. Errors are logged at WARN;
+// the loop never stops on a transient DB failure.
 func (c *Compactor) tickOnce(ctx context.Context) {
-	cutoff := pgtype.Timestamptz{Time: time.Now().Add(-CompactorRetention), Valid: true}
-	rows, err := dbq.New(c.pool).DeleteOldLibraryChanges(ctx, cutoff)
-	if err != nil {
-		c.logger.Warn("library_changes compactor: delete failed", "err", err)
-		return
-	}
-	if rows > 0 {
-		c.logger.Info("library_changes compactor: trimmed rows",
-			"count", rows, "older_than", CompactorRetention)
+	if err := Compact(ctx, c.pool, time.Now().Add(-CompactorRetention), c.logger); err != nil {
+		c.logger.Warn("library_changes compactor: trim failed", "err", err)
 	}
 }
+
+// Compact drops every superseded row, then every delete older than cutoff.
+func Compact(ctx context.Context, db dbq.DBTX, cutoff time.Time, logger *slog.Logger) error {
+	q := dbq.New(db)
+	superseded, err := q.DeleteSupersededLibraryChanges(ctx)
+	if err != nil {
+		return fmt.Errorf("drop superseded changes: %w", err)
+	}
+	deletes, err := q.DeleteOldLibraryDeletes(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true})
+	if err != nil {
+		return fmt.Errorf("drop old deletes: %w", err)
+	}
+	if superseded > 0 || deletes > 0 {
+		logger.Info("library_changes compactor: trimmed rows",
+			"superseded", superseded, "old_deletes", deletes, "deletes_older_than", CompactorRetention)
+	}
+	return nil
+}

internal/sync/compactor_test.go

@@ -0,0 +1,59 @@
+package sync_test
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/stretchr/testify/require"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/sync"
+)
+
+// The compactor keeps each entity's latest change however old, so an app
+// syncing from 0 still gets an artist nobody has touched in months. It drops
+// what a newer row supersedes, and deletes past their retention, raising the
+// trimmed mark past the newest delete it dropped.
+func TestCompact_KeepsEachEntitysLatestChange_Integration(t *testing.T) {
+	pool := testPool(t)
+	ctx := context.Background()
+	old := time.Now().Add(-90 * 24 * time.Hour)
+	recent := time.Now().Add(-time.Hour)
+
+	add := func(entity, op string, at time.Time) int64 {
+		t.Helper()
+		var id int64
+		require.NoError(t, pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op, changed_at)
+			VALUES ('artist', $1, $2, $3) RETURNING id`, entity, op, at).Scan(&id))
+		return id
+	}
+	untouched := add("untouched", "upsert", old)             // kept: latest, however old
+	add("edited", "upsert", old)                             // dropped: superseded
+	edited := add("edited", "upsert", recent)                // kept
+	add("removed-long-ago", "upsert", old)                   // dropped: superseded
+	removedLongAgo := add("removed-long-ago", "delete", old) // dropped: past retention
+	add("removed-lately", "upsert", old)                     // dropped: superseded
+	removedLately := add("removed-lately", "delete", recent) // kept: within retention
+
+	require.NoError(t, sync.Compact(ctx, pool, time.Now().Add(-sync.CompactorRetention),
+		slog.New(slog.NewTextHandler(io.Discard, nil))))
+
+	rows, err := pool.Query(ctx, `SELECT id FROM library_changes ORDER BY id`)
+	require.NoError(t, err)
+	left, err := pgx.CollectRows(rows, pgx.RowTo[int64])
+	require.NoError(t, err)
+	require.Equal(t, []int64{untouched, edited, removedLately}, left)
+
+	var through int64
+	require.NoError(t, pool.QueryRow(ctx, `SELECT through FROM library_changes_trimmed`).Scan(&through))
+	require.Equal(t, removedLongAgo, through, "the trimmed mark stands at the newest dropped delete")
+
+	// A second pass finds nothing more, and leaves the mark where it is.
+	require.NoError(t, sync.Compact(ctx, pool, time.Now().Add(-sync.CompactorRetention),
+		slog.New(slog.NewTextHandler(io.Discard, nil))))
+	require.NoError(t, pool.QueryRow(ctx, `SELECT through FROM library_changes_trimmed`).Scan(&through))
+	require.Equal(t, removedLongAgo, through)
+}