feat(#357): library_changes retention compactor
Closes the last deferred follow-up from #357. The library_changes table is the append-only change log that drives /api/library/sync's delta semantics — every mutation (scanner upsert, like, playlist edit, track delete) writes one row. Without a retention policy the table grows unbounded; the original migration (0025) called out the follow-up explicitly. New goroutine: sync.Compactor runs daily, deletes rows where changed_at < now - 30 days. Logs a row count when non-zero so operators can see compaction activity in the journal. First tick fires on startup so a process that hasn't been compacted in a while catches up immediately. 30-day retention matches the offline-mode spec (docs/superpowers/specs/2026-05-09-flutter-offline-mode-design.md). Clients with a cursor older than that hit the existing 410 fallback path and resync from scratch. Imported as syncpkg in main.go to follow the existing convention (see internal/library/scanner.go). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -25,6 +25,7 @@ import (
|
|||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/server"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/server"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/similarity"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/similarity"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/subsonic"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/subsonic"
|
||||||
|
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -159,6 +160,13 @@ func run() error {
|
|||||||
lidarrReconciler := lidarrrequests.NewReconciler(pool, lidarrCfg, logger.With("component", "lidarr"), bus)
|
lidarrReconciler := lidarrrequests.NewReconciler(pool, lidarrCfg, logger.With("component", "lidarr"), bus)
|
||||||
go lidarrReconciler.Run(ctx)
|
go lidarrReconciler.Run(ctx)
|
||||||
|
|
||||||
|
// library_changes compactor (#357 follow-up). Daily tick; deletes
|
||||||
|
// rows older than the configured retention so the change-log table
|
||||||
|
// doesn't grow unbounded. Clients that drop offline longer than
|
||||||
|
// retention hit the /api/library/sync 410 fallback and resync.
|
||||||
|
libraryChangesCompactor := syncpkg.NewCompactor(pool, logger.With("component", "library_changes_compactor"))
|
||||||
|
go libraryChangesCompactor.Run(ctx)
|
||||||
|
|
||||||
// Per-user system-playlist scheduler (#392 Half B). Fires each
|
// Per-user system-playlist scheduler (#392 Half B). Fires each
|
||||||
// active user's daily build at 03:00 in their stored timezone.
|
// active user's daily build at 03:00 in their stored timezone.
|
||||||
// Replaces the 24h-anchored cron loop (removed in the next commit
|
// Replaces the 24h-anchored cron loop (removed in the next commit
|
||||||
|
|||||||
@@ -7,8 +7,25 @@ package dbq
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const deleteOldLibraryChanges = `-- name: DeleteOldLibraryChanges :execrows
|
||||||
|
DELETE FROM library_changes WHERE changed_at < $1
|
||||||
|
`
|
||||||
|
|
||||||
|
// Removes library_changes rows older than the given cutoff. Run from
|
||||||
|
// the sync compactor goroutine on a daily tick. Returns the number of
|
||||||
|
// rows deleted so the worker can log meaningful progress.
|
||||||
|
func (q *Queries) DeleteOldLibraryChanges(ctx context.Context, changedAt pgtype.Timestamptz) (int64, error) {
|
||||||
|
result, err := q.db.Exec(ctx, deleteOldLibraryChanges, changedAt)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.RowsAffected(), nil
|
||||||
|
}
|
||||||
|
|
||||||
const getLibraryChangesSince = `-- name: GetLibraryChangesSince :many
|
const getLibraryChangesSince = `-- name: GetLibraryChangesSince :many
|
||||||
SELECT id, entity_type, entity_id, op, changed_at
|
SELECT id, entity_type, entity_id, op, changed_at
|
||||||
FROM library_changes
|
FROM library_changes
|
||||||
|
|||||||
@@ -14,3 +14,9 @@ SELECT COALESCE(MAX(id), 0)::BIGINT FROM library_changes;
|
|||||||
|
|
||||||
-- name: GetMinLibraryChangeCursor :one
|
-- name: GetMinLibraryChangeCursor :one
|
||||||
SELECT COALESCE(MIN(id), 0)::BIGINT FROM library_changes;
|
SELECT COALESCE(MIN(id), 0)::BIGINT FROM library_changes;
|
||||||
|
|
||||||
|
-- name: DeleteOldLibraryChanges :execrows
|
||||||
|
-- Removes library_changes rows older than the given cutoff. Run from
|
||||||
|
-- the sync compactor goroutine on a daily tick. Returns the number of
|
||||||
|
-- rows deleted so the worker can log meaningful progress.
|
||||||
|
DELETE FROM library_changes WHERE changed_at < $1;
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
// Periodic compactor for the library_changes log (#357 deferred
|
||||||
|
// follow-up). Every mutation in the library writes one row, so the
|
||||||
|
// table grows unbounded without a retention policy. This worker
|
||||||
|
// trims rows older than CompactorRetention on a daily tick.
|
||||||
|
//
|
||||||
|
// Why daily / 30 days: Flutter clients track a per-app cursor and
|
||||||
|
// pull deltas from the last seen id. As long as a client checks in
|
||||||
|
// at least once per retention window, it never sees the cursor go
|
||||||
|
// stale. Clients that drop offline longer than that hit the /api/
|
||||||
|
// library/sync 410 fallback and do a full resync. 30 days is the
|
||||||
|
// established norm for similar offline-cache delta logs and matches
|
||||||
|
// the spec at docs/superpowers/specs/2026-05-09-flutter-offline-mode-design.md.
|
||||||
|
|
||||||
|
package sync
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CompactorRetention is the age past which library_changes rows are
|
||||||
|
// dropped. Mutable for tests; production runs should not change it
|
||||||
|
// without considering the corresponding client cursor-stale fallback.
|
||||||
|
var CompactorRetention = 30 * 24 * time.Hour
|
||||||
|
|
||||||
|
// CompactorInterval is how often the worker checks. Daily is enough —
|
||||||
|
// the table grows on the order of a few thousand rows/day for an
|
||||||
|
// active library, well below a size where intra-day trimming matters.
|
||||||
|
var CompactorInterval = 24 * time.Hour
|
||||||
|
|
||||||
|
// Compactor is a background worker that periodically deletes
|
||||||
|
// library_changes rows older than CompactorRetention.
|
||||||
|
type Compactor struct {
|
||||||
|
pool *pgxpool.Pool
|
||||||
|
logger *slog.Logger
|
||||||
|
interval time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCompactor builds a compactor with production defaults.
|
||||||
|
func NewCompactor(pool *pgxpool.Pool, logger *slog.Logger) *Compactor {
|
||||||
|
return &Compactor{
|
||||||
|
pool: pool,
|
||||||
|
logger: logger,
|
||||||
|
interval: CompactorInterval,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run blocks until ctx is cancelled, ticking every c.interval.
|
||||||
|
// Fires one tick immediately on startup so a long-running process
|
||||||
|
// that hasn't been compacted in a while catches up.
|
||||||
|
func (c *Compactor) Run(ctx context.Context) {
|
||||||
|
c.tickOnce(ctx)
|
||||||
|
t := time.NewTicker(c.interval)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
c.tickOnce(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tickOnce deletes rows older than CompactorRetention and logs the
|
||||||
|
// row count. Errors are logged at WARN; we never abort the loop on
|
||||||
|
// transient DB failures.
|
||||||
|
func (c *Compactor) tickOnce(ctx context.Context) {
|
||||||
|
cutoff := pgtype.Timestamptz{Time: time.Now().Add(-CompactorRetention), Valid: true}
|
||||||
|
rows, err := dbq.New(c.pool).DeleteOldLibraryChanges(ctx, cutoff)
|
||||||
|
if err != nil {
|
||||||
|
c.logger.Warn("library_changes compactor: delete failed", "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if rows > 0 {
|
||||||
|
c.logger.Info("library_changes compactor: trimmed rows",
|
||||||
|
"count", rows, "older_than", CompactorRetention)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user