feat(cache): CacheFiller background metadata sweeper
Periodic worker that walks cached_artists for missing album lists and cached_albums for missing track lists, then fills them via /api/artists/:id + /api/albums/:id. Newly-discovered album covers are pre-warmed into flutter_cache_manager's disk cache too. Solves the "tap an artist → empty album area → pop in" experience: artistAlbumsProvider was drift-first but the cache only got populated when the user navigated TO an artist. SyncController's /api/library/sync delta doesn't carry per-artist album lists (those are query-time derived). Now the filler pre-populates them in the background so the drift hit is real on first tap. Pacing (intentionally conservative): * 10-second initial delay so SyncController has time to land its first sync — the WHERE NOT EXISTS query has nothing to do until cached_artists is populated. * 5-minute interval thereafter. Once steady state is reached the sweep is a cheap drift query + early exit. * 200ms throttle between per-entity REST requests — never competes with active playback. * 200 entities per sweep cap so a fresh install with thousands of artists doesn't tie up the network for one continuous run. Next sweep picks up where this one left off (NOT EXISTS naturally skips already-filled rows). Wall time estimate for a 1000-artist library: ~3-4 minutes spread over multiple sweeps. Single round-trip per artist (new getArtistDetail API method returns artist + albums in one shot). Activated from app.dart's postFrameCallback alongside the existing SyncController / Prefetcher / MetadataPrefetcher / LiveEvents hooks. Disposed via ref.onDispose when the provider scope tears down (effectively process death in practice).
This commit is contained in:
@@ -54,6 +54,24 @@ class LibraryApi {
|
||||
return ArtistRef.fromJson(r.data ?? const {});
|
||||
}
|
||||
|
||||
/// GET /api/artists/{id} — full response with the ArtistRef AND the
|
||||
/// embedded album list, both parsed. Single round-trip variant used
|
||||
/// by CacheFiller and other callers that want to populate both
|
||||
/// cached_artists and cached_albums in one shot. The existing
|
||||
/// getArtist / getArtistAlbums keep working for callers that only
|
||||
/// need one half — they hit the same URL but the per-id Riverpod
|
||||
/// caching layer dedupes.
|
||||
Future<({ArtistRef artist, List<AlbumRef> albums})> getArtistDetail(
|
||||
String id) async {
|
||||
final r = await _dio.get<Map<String, dynamic>>('/api/artists/$id');
|
||||
final body = r.data ?? const <String, dynamic>{};
|
||||
final artist = ArtistRef.fromJson(body);
|
||||
final albums = ((body['albums'] as List?) ?? const [])
|
||||
.map((e) => AlbumRef.fromJson((e as Map).cast<String, dynamic>()))
|
||||
.toList(growable: false);
|
||||
return (artist: artist, albums: albums);
|
||||
}
|
||||
|
||||
/// Pulls the "albums" array out of the same ArtistDetail body. Callers
|
||||
/// that need both the artist and its albums should issue two provider
|
||||
/// reads (artistProvider + artistAlbumsProvider) — both hit the same
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
||||
|
||||
import 'cache/cache_filler.dart';
|
||||
import 'cache/metadata_prefetcher.dart';
|
||||
import 'cache/prefetcher.dart';
|
||||
import 'cache/sync_controller.dart';
|
||||
@@ -39,6 +40,14 @@ class _MinstrelAppState extends ConsumerState<MinstrelApp> {
|
||||
// arrive. Also installs an AppLifecycleState observer for
|
||||
// resume-time defensive invalidation.
|
||||
ref.read(liveEventsDispatcherProvider);
|
||||
// Cache filler: background sweeper that walks cached_artists
|
||||
// / cached_albums for missing relations and fetches the
|
||||
// per-entity detail so tapping an artist surfaces albums
|
||||
// immediately (drift hit, no /api/artists/:id round-trip at
|
||||
// tap time). First sweep 10s after launch; every 5 minutes
|
||||
// thereafter. Throttled 200ms between requests so it never
|
||||
// competes with user activity.
|
||||
ref.read(cacheFillerProvider);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
// Background metadata sweeper that walks the drift cache for missing
|
||||
// relations and fills them via /api/artists/:id + /api/albums/:id.
|
||||
// Cover bytes for newly-filled albums are pre-warmed too so the
|
||||
// per-tile display path doesn't have to wait on network.
|
||||
//
|
||||
// Why this layer on top of SyncController and HydrationQueue:
|
||||
// - SyncController.sync only ingests entities the server emitted
|
||||
// into the library_changes delta for this user. For a fresh
|
||||
// install, that's everything; for an existing user, only recent
|
||||
// changes. The per-artist album list and per-album track list
|
||||
// are NOT in those deltas — they're derived at query time.
|
||||
// - HydrationQueue fills these lazily on tile render. CacheFiller
|
||||
// fills them proactively on a slow schedule so tapping an artist
|
||||
// surfaces albums immediately instead of triggering a fresh
|
||||
// /api/artists/:id at tap time.
|
||||
//
|
||||
// Pacing is conservative — 200ms between requests, max 200 entities
|
||||
// per sweep, 5-minute interval. Designed to never compete with user
|
||||
// activity. Wall time on a 1000-artist library: ~3-4 minutes spread
|
||||
// over multiple sweeps.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:drift/drift.dart' show Variable;
|
||||
import 'package:flutter/foundation.dart' show debugPrint;
|
||||
import 'package:flutter_cache_manager/flutter_cache_manager.dart';
|
||||
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
||||
|
||||
import '../auth/auth_provider.dart'
|
||||
show serverUrlProvider, sessionTokenProvider;
|
||||
import '../cache/adapters.dart';
|
||||
import '../cache/audio_cache_manager.dart' show appDbProvider;
|
||||
import '../cache/connectivity_provider.dart';
|
||||
import '../library/library_providers.dart' show libraryApiProvider;
|
||||
|
||||
class CacheFiller {
|
||||
CacheFiller(this._ref);
|
||||
final Ref _ref;
|
||||
|
||||
Timer? _timer;
|
||||
bool _running = false;
|
||||
bool _disposed = false;
|
||||
|
||||
/// Delay between launch and the first sweep. Long enough for
|
||||
/// SyncController to land its initial /api/library/sync so the
|
||||
/// CacheFiller's "unfilled relations" query has meaningful work.
|
||||
static const _initialDelay = Duration(seconds: 10);
|
||||
|
||||
/// Cadence between sweeps. Once a sweep finds nothing to do (steady
|
||||
/// state) the interval becomes a cheap drift query + early exit.
|
||||
static const _interval = Duration(minutes: 5);
|
||||
|
||||
/// Throttle between per-entity REST requests so the filler doesn't
|
||||
/// saturate the server or compete with user-initiated playback.
|
||||
static const _requestThrottle = Duration(milliseconds: 200);
|
||||
|
||||
/// Per-sweep cap. Without this, a fresh install with thousands of
|
||||
/// artists would tie up the network for many minutes in one go.
|
||||
/// The next sweep continues where this one left off (the WHERE
|
||||
/// NOT EXISTS query naturally skips already-filled rows).
|
||||
static const _maxIdsPerSweep = 200;
|
||||
|
||||
void start() {
|
||||
Timer(_initialDelay, _sweep);
|
||||
_timer = Timer.periodic(_interval, (_) => _sweep());
|
||||
}
|
||||
|
||||
Future<void> _sweep() async {
|
||||
if (_disposed || _running) return;
|
||||
final online = await _ref
|
||||
.read(connectivityProvider.future)
|
||||
.timeout(const Duration(seconds: 3), onTimeout: () => true);
|
||||
if (!online) return;
|
||||
_running = true;
|
||||
try {
|
||||
await _fillArtists();
|
||||
if (_disposed) return;
|
||||
await _fillAlbums();
|
||||
} catch (e, st) {
|
||||
debugPrint('cache_filler: sweep failed: $e\n$st');
|
||||
} finally {
|
||||
_running = false;
|
||||
}
|
||||
}
|
||||
|
||||
/// Find artists with no albums in cache and fetch /api/artists/:id
|
||||
/// for each (single round-trip yields artist + albums via the new
|
||||
/// getArtistDetail API method). Newly-discovered album covers
|
||||
/// land in flutter_cache_manager's disk cache too so the next
|
||||
/// visit to the artist's albums grid paints from disk.
|
||||
Future<void> _fillArtists() async {
|
||||
final db = _ref.read(appDbProvider);
|
||||
final rows = await db.customSelect(
|
||||
'''
|
||||
SELECT a.id FROM cached_artists a
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM cached_albums b WHERE b.artist_id = a.id
|
||||
)
|
||||
LIMIT ?
|
||||
''',
|
||||
variables: [Variable.withInt(_maxIdsPerSweep)],
|
||||
).get();
|
||||
final ids = rows.map((r) => r.read<String>('id')).toList();
|
||||
if (ids.isEmpty) return;
|
||||
|
||||
final api = await _ref.read(libraryApiProvider.future);
|
||||
final newAlbumIds = <String>[];
|
||||
|
||||
for (final id in ids) {
|
||||
if (_disposed) return;
|
||||
try {
|
||||
final detail = await api.getArtistDetail(id);
|
||||
await db.transaction(() async {
|
||||
await db
|
||||
.into(db.cachedArtists)
|
||||
.insertOnConflictUpdate(detail.artist.toDrift());
|
||||
if (detail.albums.isNotEmpty) {
|
||||
await db.batch((b) {
|
||||
b.insertAllOnConflictUpdate(
|
||||
db.cachedAlbums,
|
||||
detail.albums.map((a) => a.toDrift()).toList(),
|
||||
);
|
||||
});
|
||||
newAlbumIds.addAll(detail.albums.map((a) => a.id));
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
debugPrint('cache_filler: fillArtist($id) failed: $e');
|
||||
}
|
||||
await Future.delayed(_requestThrottle);
|
||||
}
|
||||
|
||||
if (newAlbumIds.isNotEmpty) {
|
||||
await _prewarmCovers(newAlbumIds);
|
||||
}
|
||||
}
|
||||
|
||||
/// Find albums with no tracks in cache and fetch /api/albums/:id
|
||||
/// for each. The bulk response carries the album's track list which
|
||||
/// we persist into cached_tracks for instant album detail render
|
||||
/// on the next visit.
|
||||
Future<void> _fillAlbums() async {
|
||||
final db = _ref.read(appDbProvider);
|
||||
final rows = await db.customSelect(
|
||||
'''
|
||||
SELECT a.id FROM cached_albums a
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM cached_tracks t WHERE t.album_id = a.id
|
||||
)
|
||||
LIMIT ?
|
||||
''',
|
||||
variables: [Variable.withInt(_maxIdsPerSweep)],
|
||||
).get();
|
||||
final ids = rows.map((r) => r.read<String>('id')).toList();
|
||||
if (ids.isEmpty) return;
|
||||
|
||||
final api = await _ref.read(libraryApiProvider.future);
|
||||
for (final id in ids) {
|
||||
if (_disposed) return;
|
||||
try {
|
||||
final result = await api.getAlbum(id);
|
||||
if (result.tracks.isNotEmpty) {
|
||||
await db.batch((b) {
|
||||
b.insertAllOnConflictUpdate(
|
||||
db.cachedTracks,
|
||||
result.tracks.map((t) => t.toDrift()).toList(),
|
||||
);
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
debugPrint('cache_filler: fillAlbum($id) failed: $e');
|
||||
}
|
||||
await Future.delayed(_requestThrottle);
|
||||
}
|
||||
}
|
||||
|
||||
/// Pre-warm album cover bytes via flutter_cache_manager so the
|
||||
/// per-tile ServerImage / CachedNetworkImage paints from disk on
|
||||
/// the user's first visit. Mirrors SyncController's prewarm
|
||||
/// helper — duplicated here intentionally to keep this file self-
|
||||
/// contained; a shared helper can come out of #357 follow-up if
|
||||
/// the duplication grows.
|
||||
Future<void> _prewarmCovers(List<String> albumIds) async {
|
||||
final baseUrl = await _ref.read(serverUrlProvider.future);
|
||||
if (baseUrl == null || baseUrl.isEmpty) return;
|
||||
final token = await _ref.read(sessionTokenProvider.future);
|
||||
final headers = (token != null && token.isNotEmpty)
|
||||
? {'Authorization': 'Bearer $token'}
|
||||
: <String, String>{};
|
||||
final base = baseUrl.endsWith('/')
|
||||
? baseUrl.substring(0, baseUrl.length - 1)
|
||||
: baseUrl;
|
||||
final mgr = DefaultCacheManager();
|
||||
|
||||
var idx = 0;
|
||||
Future<void> worker() async {
|
||||
while (idx < albumIds.length) {
|
||||
if (_disposed) return;
|
||||
final i = idx++;
|
||||
try {
|
||||
await mgr.downloadFile(
|
||||
'$base/api/albums/${albumIds[i]}/cover',
|
||||
authHeaders: headers,
|
||||
);
|
||||
} catch (_) {
|
||||
// Best-effort prewarm — 404 (missing collage) and 401
|
||||
// (token refresh races) shouldn't abort the rest.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Concurrency 3 — same as SyncController's prewarm. Covers are
|
||||
// small enough that more parallelism doesn't help much and
|
||||
// burns radio.
|
||||
await Future.wait(List.generate(3, (_) => worker()));
|
||||
}
|
||||
|
||||
void dispose() {
|
||||
_disposed = true;
|
||||
_timer?.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
/// Read once at app start (from app.dart's postFrameCallback) to
|
||||
/// activate the filler. Disposed via ref.onDispose when the provider
|
||||
/// scope tears down; in practice the scope lives for the app's
|
||||
/// lifetime so dispose only fires on uninstall / process death.
|
||||
final cacheFillerProvider = Provider<CacheFiller>((ref) {
|
||||
final filler = CacheFiller(ref);
|
||||
ref.onDispose(filler.dispose);
|
||||
filler.start();
|
||||
return filler;
|
||||
});
|
||||
Reference in New Issue
Block a user