-- 0101_library_changes_keep_latest.up.sql — the change feed keeps every -- entity's latest change, so an app syncing from nothing gets the whole -- library. -- -- Until now the compactor dropped every row older than 30 days. An app -- syncing from cursor 0 (a fresh install, cleared data, or a reset after a -- 410) builds its library from the feed, so it got only what had changed in -- the last 30 days. From here the compactor drops a row only once a newer row -- for the same entity exists, and drops a delete only once it is 30 days old. -- -- idx_library_changes_entity finds an entity's rows, newest first, for -- the compactor and the fill below. -- library_changes_trimmed one row: the highest id of a delete the -- compactor has dropped. An app whose cursor -- is below it may have missed that delete, so -- it is told to start over (410). It starts -- where the old compactor left off: everything -- below the oldest remaining row is gone. -- -- Then every artist, album and track whose latest row is not an upsert, or -- which has no row at all (the old compactor dropped it), gets one, artists -- first so an album never arrives before its artist. Likes and playlists are -- left as they are: the apps fetch those from their own endpoints. CREATE INDEX idx_library_changes_entity ON library_changes (entity_type, entity_id, id DESC); CREATE TABLE library_changes_trimmed ( id boolean PRIMARY KEY DEFAULT true CONSTRAINT library_changes_trimmed_one_row CHECK (id), through bigint NOT NULL DEFAULT 0 ); INSERT INTO library_changes_trimmed (through) SELECT COALESCE(min(id), 1) - 1 FROM library_changes; INSERT INTO library_changes (entity_type, entity_id, op) SELECT 'artist', a.id::text, 'upsert' FROM artists a WHERE COALESCE((SELECT c.op FROM library_changes c WHERE c.entity_type = 'artist' AND c.entity_id = a.id::text ORDER BY c.id DESC LIMIT 1), 'delete') <> 'upsert' ORDER BY a.id; INSERT INTO library_changes (entity_type, entity_id, op) SELECT 'album', al.id::text, 'upsert' FROM albums al WHERE COALESCE((SELECT c.op FROM library_changes c WHERE c.entity_type = 'album' AND c.entity_id = al.id::text ORDER BY c.id DESC LIMIT 1), 'delete') <> 'upsert' ORDER BY al.id; INSERT INTO library_changes (entity_type, entity_id, op) SELECT 'track', t.id::text, 'upsert' FROM tracks t WHERE COALESCE((SELECT c.op FROM library_changes c WHERE c.entity_type = 'track' AND c.entity_id = t.id::text ORDER BY c.id DESC LIMIT 1), 'delete') <> 'upsert' ORDER BY t.id;