Compare commits
22
Commits
a67897a6cd
...
dev
@@ -24,6 +24,7 @@ import androidx.core.content.ContextCompat
|
||||
import androidx.hilt.navigation.compose.hiltViewModel
|
||||
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
||||
import androidx.lifecycle.lifecycleScope
|
||||
import androidx.navigation.NavHostController
|
||||
import androidx.navigation.compose.rememberNavController
|
||||
import com.fabledsword.minstrel.auth.AuthStore
|
||||
import com.fabledsword.minstrel.auth.ui.AuthGateViewModel
|
||||
@@ -33,6 +34,7 @@ import com.fabledsword.minstrel.connectivity.ServerHealth
|
||||
import com.fabledsword.minstrel.connectivity.NetworkStatusController
|
||||
import com.fabledsword.minstrel.nav.DetailSeedCache
|
||||
import com.fabledsword.minstrel.nav.LocalDetailSeedCache
|
||||
import com.fabledsword.minstrel.nav.Login
|
||||
import com.fabledsword.minstrel.nav.MinstrelNavGraph
|
||||
import com.fabledsword.minstrel.nav.Notifications
|
||||
import com.fabledsword.minstrel.nav.NowPlaying
|
||||
@@ -177,6 +179,7 @@ private fun App(
|
||||
navController.navigate(route) { launchSingleTop = true }
|
||||
onOpenedRoute()
|
||||
}
|
||||
SignInWhenSessionRejected(gate, navController)
|
||||
MinstrelNavGraph(
|
||||
navController = navController,
|
||||
startDestination = resolved,
|
||||
@@ -187,6 +190,24 @@ private fun App(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* When the server ends the session while the app is in use, opens the
|
||||
* sign-in screen rather than leaving the user on screens whose every
|
||||
* request fails.
|
||||
*/
|
||||
@Composable
|
||||
private fun SignInWhenSessionRejected(gate: AuthGateViewModel, navController: NavHostController) {
|
||||
val sessionRejected by gate.sessionRejected.collectAsStateWithLifecycle()
|
||||
LaunchedEffect(sessionRejected, navController) {
|
||||
if (!sessionRejected) return@LaunchedEffect
|
||||
navController.navigate(Login) {
|
||||
popUpTo(0) { inclusive = true }
|
||||
launchSingleTop = true
|
||||
}
|
||||
gate.consumeSessionRejected()
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun BootSplash() {
|
||||
Surface(
|
||||
|
||||
@@ -8,9 +8,9 @@ import javax.inject.Singleton
|
||||
|
||||
/**
|
||||
* Attaches the session cookie from [AuthStore] to Minstrel-server
|
||||
* requests, captures Set-Cookie from successful responses (login
|
||||
* flow), and clears the store on 401 so downstream code can react
|
||||
* to logout.
|
||||
* requests (waiting for the stored one to load at launch), captures
|
||||
* Set-Cookie from successful responses (login flow), and signs out when
|
||||
* the server refuses the cookie a request carried.
|
||||
*
|
||||
* Scoped to the [BaseUrlInterceptor.PLACEHOLDER_HOST] sentinel host
|
||||
* the same way BaseUrlInterceptor is. The shared OkHttpClient also
|
||||
@@ -32,7 +32,7 @@ class AuthCookieInterceptor @Inject constructor(
|
||||
// session. Pass through untouched.
|
||||
return chain.proceed(original)
|
||||
}
|
||||
val cookie = authStore.sessionCookie.value
|
||||
val cookie = authStore.sessionCookieForRequest()
|
||||
val request = original.newBuilder().apply {
|
||||
if (!cookie.isNullOrEmpty()) header("Cookie", cookie)
|
||||
}.build()
|
||||
@@ -40,7 +40,11 @@ class AuthCookieInterceptor @Inject constructor(
|
||||
val response = chain.proceed(request)
|
||||
|
||||
if (response.code == HTTP_UNAUTHORIZED) {
|
||||
authStore.setSessionCookie(null)
|
||||
// Only a refusal of the cookie this request carried ends the
|
||||
// session. A request sent without one says nothing about the
|
||||
// stored login, and a refusal of an older cookie must not
|
||||
// discard one a sign-in has stored since.
|
||||
if (!cookie.isNullOrEmpty()) authStore.dropRejectedSessionCookie(cookie)
|
||||
} else if (response.isSuccessful) {
|
||||
// Capture the first Set-Cookie pair on login; subsequent
|
||||
// responses that include Set-Cookie keep the store fresh.
|
||||
|
||||
@@ -89,8 +89,11 @@ class AuthController @Inject constructor(
|
||||
* + persisted userJson. Carries account-level flags (debug mode)
|
||||
* that aren't in the login response. Best-effort: a network failure
|
||||
* leaves the cached identity untouched. No-op when signed out.
|
||||
* Waits for the stored login to load first: at launch this runs before
|
||||
* it has, and would otherwise read the user as signed out and skip.
|
||||
*/
|
||||
suspend fun refreshProfile() {
|
||||
authStore.awaitSessionHydrated()
|
||||
if (!isSignedIn) return
|
||||
val p = runCatching { meApi.getProfile() }.getOrNull() ?: return
|
||||
val user = UserRef(
|
||||
|
||||
@@ -12,6 +12,7 @@ import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlinx.coroutines.withTimeoutOrNull
|
||||
@@ -148,6 +149,49 @@ class AuthStore @Inject constructor(
|
||||
scope.launch { cookieLock.withLock { storeCookie(value) } }
|
||||
}
|
||||
|
||||
/**
|
||||
* The cookie to attach to an outgoing request. Blocks the calling
|
||||
* (OkHttp) thread until the stored cookie has loaded, up to
|
||||
* [HYDRATION_DEADLINE_MS], so a request made in the first moment after
|
||||
* launch carries the login instead of going out without it. Once
|
||||
* hydration has finished this is a plain read.
|
||||
*/
|
||||
fun sessionCookieForRequest(): String? {
|
||||
if (!cookieHydration.isCompleted) {
|
||||
runBlocking { withTimeoutOrNull(HYDRATION_DEADLINE_MS) { cookieHydration.await() } }
|
||||
}
|
||||
return sessionCookieState.value
|
||||
}
|
||||
|
||||
/**
|
||||
* True once the server has refused the current session (see
|
||||
* [dropRejectedSessionCookie]) and the UI has not yet sent the user to
|
||||
* sign in. Held rather than fired, so a refusal that lands while the app
|
||||
* is in the background is still acted on when it comes back. A sign-out
|
||||
* the user asked for does not set it.
|
||||
*/
|
||||
private val sessionRejectedState = MutableStateFlow(false)
|
||||
val sessionRejected: StateFlow<Boolean> = sessionRejectedState.asStateFlow()
|
||||
|
||||
/** The UI has opened the sign-in screen for a refused session. */
|
||||
fun consumeSessionRejected() {
|
||||
sessionRejectedState.value = false
|
||||
}
|
||||
|
||||
/**
|
||||
* Signs out because the server answered 401 to a request that carried
|
||||
* [rejected]. Does nothing when the store has since moved on to another
|
||||
* cookie (a sign-in landed meanwhile), so a refusal of an old session
|
||||
* never discards a new one. What is persisted is read inside the lock,
|
||||
* so a sign-in racing this call is the value that ends up stored.
|
||||
*/
|
||||
fun dropRejectedSessionCookie(rejected: String) {
|
||||
if (!sessionCookieState.compareAndSet(rejected, null)) return
|
||||
cookieTouched = true
|
||||
scope.launch { cookieLock.withLock { storeCookie(sessionCookieState.value) } }
|
||||
sessionRejectedState.value = true
|
||||
}
|
||||
|
||||
private suspend fun hydrateSessionCookie() = cookieLock.withLock {
|
||||
val legacy = runCatching { dao.get()?.sessionCookie }.getOrNull()
|
||||
if (cookieTouched) return@withLock
|
||||
|
||||
@@ -38,6 +38,15 @@ class AuthGateViewModel @Inject constructor(
|
||||
private val internal = MutableStateFlow<Any?>(null)
|
||||
val startDestination: StateFlow<Any?> = internal.asStateFlow()
|
||||
|
||||
/**
|
||||
* True when the server has ended the session while the app was in use;
|
||||
* the root NavHost answers by opening the sign-in screen, then calls
|
||||
* [consumeSessionRejected].
|
||||
*/
|
||||
val sessionRejected: StateFlow<Boolean> = authStore.sessionRejected
|
||||
|
||||
fun consumeSessionRejected() = authStore.consumeSessionRejected()
|
||||
|
||||
init {
|
||||
viewModelScope.launch {
|
||||
authStore.awaitSessionHydrated()
|
||||
|
||||
@@ -70,6 +70,9 @@ class DiagnosticsUploader @Inject constructor(
|
||||
|
||||
/** Coalesces concurrent triggers via tryLock — a drain in flight wins. */
|
||||
private suspend fun drainSafe() {
|
||||
// A trigger at launch can fire before the stored login has loaded;
|
||||
// wait for it, so a signed-in user is not read as signed out.
|
||||
authStore.awaitSessionHydrated()
|
||||
val clientId = authStore.clientId.value
|
||||
// Signed out / no client id yet → nothing to do. Single guard keeps
|
||||
// the return count within the detekt gate.
|
||||
|
||||
@@ -781,6 +781,13 @@ private fun HomeSuccessContent(
|
||||
onPlayPool = onPlayPool,
|
||||
onPlayPlaylist = onPlayPlaylist,
|
||||
)
|
||||
genreMixSection(
|
||||
playlists = sections.playlists,
|
||||
offline = offline,
|
||||
onPlaylistClick = onPlaylistClick,
|
||||
onPlayPool = onPlayPool,
|
||||
onPlayPlaylist = onPlayPlaylist,
|
||||
)
|
||||
youMightLikeSection(
|
||||
albums = sections.youMightLikeAlbums,
|
||||
artists = sections.youMightLikeArtists,
|
||||
@@ -833,6 +840,31 @@ private fun LazyListScope.songsLikeSection(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The "Your genres" row: one mix per genre the user listens to. Hidden
|
||||
* when there is nothing to show, built or cached.
|
||||
*/
|
||||
private fun LazyListScope.genreMixSection(
|
||||
playlists: List<PlaylistRef>,
|
||||
offline: Boolean,
|
||||
onPlaylistClick: (String) -> Unit,
|
||||
onPlayPool: (OfflinePoolKind) -> Unit,
|
||||
onPlayPlaylist: suspend (PlaylistRef) -> Unit,
|
||||
) {
|
||||
val rowItems = buildGenreMixRow(playlists, offline)
|
||||
if (rowItems.isEmpty()) return
|
||||
item {
|
||||
PlaylistsRow(
|
||||
title = "Your genres",
|
||||
rowItems = rowItems,
|
||||
offline = offline,
|
||||
onPlaylistClick = onPlaylistClick,
|
||||
onPlayPool = onPlayPool,
|
||||
onPlayPlaylist = onPlayPlaylist,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun LazyListScope.recentlyAddedSection(
|
||||
albums: List<HomeTile<AlbumRef>>,
|
||||
onAlbumClick: (String) -> Unit,
|
||||
@@ -1233,6 +1265,25 @@ internal fun buildSongsLikeRow(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the "Your genres" row: every genre mix, in name order so the
|
||||
* shelf reads like a stable channel list. No placeholders — a genre mix
|
||||
* needs listening history or a genre-tagged library, so none built means
|
||||
* nothing to show. Offline, fully-cached mixes come first and the rest are
|
||||
* greyed. Mirrors the web Home's genreMixPlaylists.
|
||||
*/
|
||||
internal fun buildGenreMixRow(
|
||||
owned: List<PlaylistRef>,
|
||||
offline: Boolean,
|
||||
): List<PlaylistRowItem> {
|
||||
val mixes = owned
|
||||
.filter { it.systemVariant == "genre_mix" }
|
||||
.sortedBy { it.name.lowercase() }
|
||||
if (!offline) return mixes.map { PlaylistRowItem.Real(it) }
|
||||
val (available, greyed) = mixes.partition { !it.unavailableOffline }
|
||||
return (available + greyed).map { PlaylistRowItem.Real(it) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Curated real-playlist order (system primaries, then secondary, then user).
|
||||
* Must mirror [buildOnlineRow]'s slot order — the offline row reuses this and
|
||||
|
||||
@@ -26,10 +26,11 @@ data class PlaylistRef(
|
||||
|
||||
/**
|
||||
* Whether the "Regenerate" affordance applies: every system playlist
|
||||
* EXCEPT songs_like_artist (the server's songs-like family exposes no
|
||||
* refresh trigger; rebuilds are scheduler-driven).
|
||||
* EXCEPT the kinds with several playlists per user (songs_like_artist,
|
||||
* genre_mix), which the server can't address by kind; those rebuild on
|
||||
* the daily schedule and play by playlist id.
|
||||
*/
|
||||
val refreshable: Boolean get() = isSystem && systemVariant != "songs_like_artist"
|
||||
val refreshable: Boolean get() = isSystem && systemVariant !in PER_PLAYLIST_SYSTEM_VARIANTS
|
||||
|
||||
/**
|
||||
* Can't be relied on while offline — either it needs the live server to
|
||||
@@ -39,6 +40,9 @@ data class PlaylistRef(
|
||||
val unavailableOffline: Boolean get() = refreshable || !fullyCached
|
||||
}
|
||||
|
||||
/** System kinds with several playlists per user, played by playlist id. */
|
||||
private val PER_PLAYLIST_SYSTEM_VARIANTS = setOf("songs_like_artist", "genre_mix")
|
||||
|
||||
/**
|
||||
* One row of a playlist's tracks.
|
||||
*
|
||||
|
||||
+38
-83
@@ -9,23 +9,17 @@ import android.os.Build
|
||||
import android.provider.Settings
|
||||
import androidx.activity.compose.rememberLauncherForActivityResult
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.foundation.selection.toggleable
|
||||
import androidx.compose.foundation.verticalScroll
|
||||
import androidx.compose.material3.Checkbox
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.Scaffold
|
||||
import androidx.compose.material3.Switch
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
@@ -35,7 +29,6 @@ import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.semantics.Role
|
||||
import androidx.compose.ui.semantics.contentDescription
|
||||
import androidx.compose.ui.semantics.semantics
|
||||
import androidx.compose.ui.text.style.TextAlign
|
||||
@@ -45,8 +38,6 @@ import androidx.hilt.navigation.compose.hiltViewModel
|
||||
import androidx.lifecycle.compose.LifecycleResumeEffect
|
||||
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
||||
import androidx.navigation.NavHostController
|
||||
import com.composables.icons.lucide.ChevronRight
|
||||
import com.composables.icons.lucide.Lucide
|
||||
import com.fabledsword.minstrel.api.endpoints.NotificationKindSettingWire
|
||||
import com.fabledsword.minstrel.api.endpoints.NotificationSettingsWire
|
||||
import com.fabledsword.minstrel.nav.NotificationSettings
|
||||
@@ -54,13 +45,19 @@ import com.fabledsword.minstrel.notifications.data.NotificationChannel
|
||||
import com.fabledsword.minstrel.shared.widgets.LoadingCentered
|
||||
import com.fabledsword.minstrel.shared.widgets.MinstrelTopAppBar
|
||||
import com.fabledsword.minstrel.shared.widgets.ShellContentWindowInsets
|
||||
import com.fabledsword.minstrel.theme.Muted
|
||||
import com.fabledsword.minstrel.theme.RowChevron
|
||||
import com.fabledsword.minstrel.theme.Section
|
||||
import com.fabledsword.minstrel.theme.SectionEdge
|
||||
import com.fabledsword.minstrel.theme.SectionGap
|
||||
import com.fabledsword.minstrel.theme.SettingRow
|
||||
import com.fabledsword.minstrel.theme.SwitchRow
|
||||
|
||||
private val CELL_WIDTH = 56.dp
|
||||
private val CELL_WIDTH = 48.dp
|
||||
|
||||
/**
|
||||
* Per-user notification settings: a row per kind with a box for each
|
||||
* channel. Rows read as a menu; anything that needs explaining
|
||||
* is one short line.
|
||||
* Per-user notification settings: what this device does, then a row per
|
||||
* kind with a box for each channel.
|
||||
*/
|
||||
@Composable
|
||||
fun NotificationSettingsScreen(
|
||||
@@ -109,29 +106,28 @@ private fun SettingsBody(
|
||||
modifier = modifier
|
||||
.fillMaxSize()
|
||||
.verticalScroll(rememberScrollState())
|
||||
.padding(vertical = 8.dp),
|
||||
.padding(horizontal = SectionEdge, vertical = SectionEdge),
|
||||
verticalArrangement = Arrangement.spacedBy(SectionGap),
|
||||
) {
|
||||
deviceRows()
|
||||
HeaderRow()
|
||||
Section("This device") { deviceRows() }
|
||||
Section("Requests", trailing = { ChannelHeaders() }) {
|
||||
settings.kinds.filter { !it.adminOnly }.forEach { KindRow(it, settings.emailAvailable, onSet) }
|
||||
val adminKinds = settings.kinds.filter { it.adminOnly }
|
||||
if (adminKinds.isNotEmpty()) {
|
||||
SectionLabel("Library health")
|
||||
adminKinds.forEach { KindRow(it, settings.emailAvailable, onSet) }
|
||||
}
|
||||
if (!settings.emailAvailable) {
|
||||
Muted(emailUnavailableLine(settings.emailUnavailableReason, isAdmin))
|
||||
}
|
||||
}
|
||||
val adminKinds = settings.kinds.filter { it.adminOnly }
|
||||
if (adminKinds.isNotEmpty()) {
|
||||
Section("Library health", trailing = { ChannelHeaders() }) {
|
||||
adminKinds.forEach { KindRow(it, settings.emailAvailable, onSet) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The channel names over the boxes, in the section's title line. */
|
||||
@Composable
|
||||
private fun HeaderRow() {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth().padding(horizontal = 16.dp, vertical = 4.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
Box(Modifier.weight(1f))
|
||||
private fun ChannelHeaders() {
|
||||
NotificationChannel.entries.forEach { c ->
|
||||
Text(
|
||||
text = channelLabel(c),
|
||||
@@ -141,7 +137,6 @@ private fun HeaderRow() {
|
||||
textAlign = TextAlign.Center,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
@@ -151,11 +146,7 @@ private fun KindRow(
|
||||
onSet: (String, NotificationChannel, Boolean) -> Unit,
|
||||
) {
|
||||
val label = kindLabel(row.kind)
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth().padding(horizontal = 16.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
Text(label, modifier = Modifier.weight(1f), style = MaterialTheme.typography.bodyLarge)
|
||||
SettingRow(title = label) {
|
||||
NotificationChannel.entries.forEach { c ->
|
||||
val enabled = channelEnabled(row, c, emailAvailable)
|
||||
Box(Modifier.width(CELL_WIDTH), contentAlignment = Alignment.Center) {
|
||||
@@ -204,60 +195,24 @@ private fun DeviceRows(backgroundDelivery: Boolean, onBackgroundDelivery: (Boole
|
||||
|
||||
@Composable
|
||||
private fun BackgroundDeliveryRow(checked: Boolean, onChange: (Boolean) -> Unit) {
|
||||
Row(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.toggleable(value = checked, role = Role.Switch, onValueChange = onChange)
|
||||
.padding(horizontal = 16.dp, vertical = 12.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Column(Modifier.weight(1f)) {
|
||||
Text("Notifications when the app is closed", style = MaterialTheme.typography.bodyLarge)
|
||||
// The ongoing notice is the price of not using Google's push;
|
||||
// said plainly rather than left to be discovered.
|
||||
Muted("Keeps a quiet notice in the shade, in place of Google's push", padded = false)
|
||||
}
|
||||
Switch(checked = checked, onCheckedChange = null)
|
||||
}
|
||||
// The ongoing notice is the price of not using Google's push; said
|
||||
// plainly rather than left to be discovered.
|
||||
SwitchRow(
|
||||
title = "Notifications when the app is closed",
|
||||
summary = "Keeps a quiet notice in the shade, in place of Google's push",
|
||||
checked = checked,
|
||||
onCheckedChange = onChange,
|
||||
)
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun PhoneBlockedRow(onClick: () -> Unit) {
|
||||
Row(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.clickable(onClick = onClick)
|
||||
.padding(horizontal = 16.dp, vertical = 12.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Column(Modifier.weight(1f)) {
|
||||
Text("Phone alerts are off", style = MaterialTheme.typography.bodyLarge)
|
||||
Muted("Allow notifications for Minstrel in system settings", padded = false)
|
||||
}
|
||||
Icon(Lucide.ChevronRight, contentDescription = null, tint = MaterialTheme.colorScheme.onSurfaceVariant)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun SectionLabel(text: String) {
|
||||
Text(
|
||||
text = text,
|
||||
modifier = Modifier.padding(start = 16.dp, end = 16.dp, top = 16.dp, bottom = 4.dp),
|
||||
style = MaterialTheme.typography.titleSmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun Muted(text: String, padded: Boolean = true) {
|
||||
Text(
|
||||
text = text,
|
||||
modifier = if (padded) Modifier.padding(horizontal = 16.dp, vertical = 12.dp) else Modifier,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
SettingRow(
|
||||
title = "Phone alerts are off",
|
||||
summary = "Allow notifications for Minstrel in system settings",
|
||||
tone = MaterialTheme.colorScheme.error,
|
||||
onClick = onClick,
|
||||
) { RowChevron() }
|
||||
}
|
||||
|
||||
private fun openAppNotificationSettings(context: Context) {
|
||||
|
||||
@@ -171,6 +171,7 @@ private fun systemLabelFor(variant: String?): String = when (variant) {
|
||||
"new_for_you" -> "New for you"
|
||||
"on_this_day" -> "On this day"
|
||||
"first_listens" -> "First listens"
|
||||
"genre_mix" -> "Genre mix"
|
||||
null -> ""
|
||||
else -> variant.replace('_', ' ').replaceFirstChar { it.uppercase() }
|
||||
}
|
||||
|
||||
+18
-71
@@ -1,20 +1,14 @@
|
||||
package com.fabledsword.minstrel.settings.ui
|
||||
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.ElevatedCard
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.OutlinedTextField
|
||||
import androidx.compose.material3.Switch
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
@@ -25,28 +19,14 @@ import androidx.compose.ui.unit.dp
|
||||
import androidx.hilt.navigation.compose.hiltViewModel
|
||||
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
||||
import com.fabledsword.minstrel.models.ListenBrainzStatus
|
||||
import com.fabledsword.minstrel.theme.Muted
|
||||
import com.fabledsword.minstrel.theme.Section
|
||||
import com.fabledsword.minstrel.theme.SwitchRow
|
||||
|
||||
@Composable
|
||||
fun ListenBrainzCard(viewModel: ListenBrainzViewModel = hiltViewModel()) {
|
||||
val state by viewModel.state.collectAsStateWithLifecycle()
|
||||
ElevatedCard(modifier = Modifier.fillMaxWidth()) {
|
||||
Column(
|
||||
modifier = Modifier.padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Text(
|
||||
text = "ListenBrainz",
|
||||
style = MaterialTheme.typography.titleMedium,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
Text(
|
||||
text = "Send your plays to ListenBrainz so you can keep your " +
|
||||
"scrobble history portable. Get a token at " +
|
||||
"listenbrainz.org/profile — it's stored encrypted on " +
|
||||
"the server and never read back.",
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
Section("ListenBrainz") {
|
||||
if (state.isLoading) {
|
||||
Box(
|
||||
modifier = Modifier.fillMaxWidth().padding(16.dp),
|
||||
@@ -64,7 +44,6 @@ fun ListenBrainzCard(viewModel: ListenBrainzViewModel = hiltViewModel()) {
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
@@ -77,25 +56,25 @@ private fun ListenBrainzForm(
|
||||
onSaveToken: () -> Unit,
|
||||
onSetEnabled: (Boolean) -> Unit,
|
||||
) {
|
||||
TokenField(
|
||||
tokenInput = tokenInput,
|
||||
tokenSet = status?.tokenSet == true,
|
||||
onChange = onTokenChange,
|
||||
val tokenSet = status?.tokenSet == true
|
||||
SwitchRow(
|
||||
title = "Send my plays to ListenBrainz",
|
||||
summary = when {
|
||||
!tokenSet -> "Save a token first"
|
||||
status?.lastScrobbledAt != null -> "Last scrobble: ${status.lastScrobbledAt}"
|
||||
else -> null
|
||||
},
|
||||
checked = status?.enabled == true,
|
||||
onCheckedChange = onSetEnabled,
|
||||
enabled = !isSaving && tokenSet,
|
||||
)
|
||||
Spacer(Modifier.height(4.dp))
|
||||
TokenField(tokenInput = tokenInput, tokenSet = tokenSet, onChange = onTokenChange)
|
||||
SaveTokenButton(
|
||||
isSaving = isSaving,
|
||||
enabled = !isSaving && tokenInput.isNotBlank(),
|
||||
onClick = onSaveToken,
|
||||
)
|
||||
EnabledRow(status = status, isSaving = isSaving, onSetEnabled = onSetEnabled)
|
||||
if (message != null) {
|
||||
Text(
|
||||
text = message,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
if (message != null) Muted(message)
|
||||
}
|
||||
|
||||
@Composable
|
||||
@@ -105,6 +84,7 @@ private fun TokenField(tokenInput: String, tokenSet: Boolean, onChange: (String)
|
||||
onValueChange = onChange,
|
||||
label = { Text(if (tokenSet) "Replace token" else "Token") },
|
||||
placeholder = { Text(if (tokenSet) "Token saved" else "") },
|
||||
supportingText = { Text("From listenbrainz.org/profile; never shown again") },
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
singleLine = true,
|
||||
visualTransformation = PasswordVisualTransformation(),
|
||||
@@ -129,36 +109,3 @@ private fun SaveTokenButton(isSaving: Boolean, enabled: Boolean, onClick: () ->
|
||||
Text(if (isSaving) "Saving…" else "Save token")
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun EnabledRow(
|
||||
status: ListenBrainzStatus?,
|
||||
isSaving: Boolean,
|
||||
onSetEnabled: (Boolean) -> Unit,
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.SpaceBetween,
|
||||
) {
|
||||
Column(modifier = Modifier.padding(end = 12.dp)) {
|
||||
Text(
|
||||
text = "Send my plays to ListenBrainz",
|
||||
style = MaterialTheme.typography.bodyMedium,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
if (status?.lastScrobbledAt != null) {
|
||||
Text(
|
||||
text = "Last scrobble: ${status.lastScrobbledAt}",
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
}
|
||||
Switch(
|
||||
checked = status?.enabled == true,
|
||||
onCheckedChange = onSetEnabled,
|
||||
enabled = !isSaving && status?.tokenSet == true,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
+19
-93
@@ -1,30 +1,15 @@
|
||||
package com.fabledsword.minstrel.settings.ui
|
||||
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.material3.ElevatedCard
|
||||
import androidx.compose.material3.ExperimentalMaterial3Api
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.SegmentedButton
|
||||
import androidx.compose.material3.SegmentedButtonDefaults
|
||||
import androidx.compose.material3.SingleChoiceSegmentedButtonRow
|
||||
import androidx.compose.material3.Switch
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.hilt.navigation.compose.hiltViewModel
|
||||
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
||||
import com.fabledsword.minstrel.settings.data.NormalizationBoost
|
||||
import com.fabledsword.minstrel.settings.data.NormalizationMode
|
||||
import com.fabledsword.minstrel.settings.data.NormalizationPrefs
|
||||
import com.fabledsword.minstrel.theme.ChoiceRow
|
||||
import com.fabledsword.minstrel.theme.Section
|
||||
import com.fabledsword.minstrel.theme.SwitchRow
|
||||
|
||||
/**
|
||||
* Volume leveling. The choice is stored on the server, so the
|
||||
@@ -36,80 +21,29 @@ fun NormalizationCard(viewModel: NormalizationViewModel = hiltViewModel()) {
|
||||
NormalizationCardContent(prefs = prefs, onChange = viewModel::update)
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalMaterial3Api::class)
|
||||
@Composable
|
||||
internal fun NormalizationCardContent(
|
||||
prefs: NormalizationPrefs,
|
||||
onChange: (NormalizationPrefs) -> Unit,
|
||||
) {
|
||||
ElevatedCard(modifier = Modifier.fillMaxWidth()) {
|
||||
Column(
|
||||
modifier = Modifier.padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Text(
|
||||
text = "Volume leveling",
|
||||
style = MaterialTheme.typography.titleMedium,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
Section("Volume leveling") {
|
||||
ChoiceRow(
|
||||
title = "Leveling",
|
||||
summary = modeHint(prefs.mode),
|
||||
options = NormalizationMode.entries.map { it to modeLabel(it) },
|
||||
selected = prefs.mode,
|
||||
onSelect = { onChange(prefs.copy(mode = it)) },
|
||||
)
|
||||
Hint(modeHint(prefs.mode))
|
||||
val modes = NormalizationMode.entries
|
||||
SingleChoiceSegmentedButtonRow(modifier = Modifier.fillMaxWidth()) {
|
||||
modes.forEachIndexed { index, mode ->
|
||||
SegmentedButton(
|
||||
selected = mode == prefs.mode,
|
||||
onClick = { onChange(prefs.copy(mode = mode)) },
|
||||
shape = SegmentedButtonDefaults.itemShape(
|
||||
index = index,
|
||||
count = modes.size,
|
||||
),
|
||||
) { Text(modeLabel(mode)) }
|
||||
}
|
||||
}
|
||||
if (prefs.mode != NormalizationMode.OFF) {
|
||||
TargetRow(prefs = prefs, onChange = onChange)
|
||||
BoostRow(prefs = prefs, onChange = onChange)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalMaterial3Api::class)
|
||||
@Composable
|
||||
private fun TargetRow(prefs: NormalizationPrefs, onChange: (NormalizationPrefs) -> Unit) {
|
||||
Text(
|
||||
text = "Target loudness",
|
||||
style = MaterialTheme.typography.bodyLarge,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
ChoiceRow(
|
||||
title = "Target loudness",
|
||||
options = NormalizationPrefs.TARGETS.map { it to "$it LUFS" },
|
||||
selected = prefs.targetLufs,
|
||||
onSelect = { onChange(prefs.copy(targetLufs = it)) },
|
||||
)
|
||||
val targets = NormalizationPrefs.TARGETS
|
||||
SingleChoiceSegmentedButtonRow(modifier = Modifier.fillMaxWidth()) {
|
||||
targets.forEachIndexed { index, lufs ->
|
||||
SegmentedButton(
|
||||
selected = lufs == prefs.targetLufs,
|
||||
onClick = { onChange(prefs.copy(targetLufs = lufs)) },
|
||||
shape = SegmentedButtonDefaults.itemShape(
|
||||
index = index,
|
||||
count = targets.size,
|
||||
),
|
||||
) { Text("$lufs LUFS") }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun BoostRow(prefs: NormalizationPrefs, onChange: (NormalizationPrefs) -> Unit) {
|
||||
Row(verticalAlignment = Alignment.CenterVertically) {
|
||||
Column(modifier = Modifier.weight(1f)) {
|
||||
Text(
|
||||
text = "Boost quiet tracks fully",
|
||||
style = MaterialTheme.typography.bodyLarge,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
Hint("A limiter catches the peaks.")
|
||||
}
|
||||
Spacer(Modifier.size(12.dp))
|
||||
Switch(
|
||||
SwitchRow(
|
||||
title = "Boost quiet tracks fully",
|
||||
summary = "A limiter catches the peaks",
|
||||
checked = prefs.boost == NormalizationBoost.LIMITER,
|
||||
onCheckedChange = { on ->
|
||||
val boost = if (on) NormalizationBoost.LIMITER else NormalizationBoost.HEADROOM
|
||||
@@ -117,15 +51,7 @@ private fun BoostRow(prefs: NormalizationPrefs, onChange: (NormalizationPrefs) -
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun Hint(text: String) {
|
||||
Text(
|
||||
text = text,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun modeLabel(mode: NormalizationMode): String = when (mode) {
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
package com.fabledsword.minstrel.settings.ui
|
||||
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
@@ -10,7 +8,6 @@ import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.text.KeyboardOptions
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.ElevatedCard
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.OutlinedTextField
|
||||
import androidx.compose.material3.Text
|
||||
@@ -22,20 +19,13 @@ import androidx.compose.ui.text.input.PasswordVisualTransformation
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.hilt.navigation.compose.hiltViewModel
|
||||
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
||||
import com.fabledsword.minstrel.theme.Muted
|
||||
import com.fabledsword.minstrel.theme.Section
|
||||
|
||||
@Composable
|
||||
fun PasswordCard(viewModel: PasswordViewModel = hiltViewModel()) {
|
||||
val state by viewModel.state.collectAsStateWithLifecycle()
|
||||
ElevatedCard(modifier = Modifier.fillMaxWidth()) {
|
||||
Column(
|
||||
modifier = Modifier.padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Text(
|
||||
text = "Password",
|
||||
style = MaterialTheme.typography.titleMedium,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
Section("Password") {
|
||||
PasswordField(
|
||||
value = state.current,
|
||||
label = "Current password",
|
||||
@@ -67,14 +57,7 @@ fun PasswordCard(viewModel: PasswordViewModel = hiltViewModel()) {
|
||||
}
|
||||
Text(if (state.isChanging) "Changing…" else "Change password")
|
||||
}
|
||||
if (state.message != null) {
|
||||
Text(
|
||||
text = state.message!!,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
}
|
||||
state.message?.let { Muted(it) }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package com.fabledsword.minstrel.settings.ui
|
||||
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
@@ -10,7 +8,6 @@ import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.ElevatedCard
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.OutlinedTextField
|
||||
import androidx.compose.material3.Text
|
||||
@@ -21,20 +18,13 @@ import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.hilt.navigation.compose.hiltViewModel
|
||||
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
||||
import com.fabledsword.minstrel.theme.Muted
|
||||
import com.fabledsword.minstrel.theme.Section
|
||||
|
||||
@Composable
|
||||
fun ProfileCard(viewModel: ProfileViewModel = hiltViewModel()) {
|
||||
val state by viewModel.state.collectAsStateWithLifecycle()
|
||||
ElevatedCard(modifier = Modifier.fillMaxWidth()) {
|
||||
Column(
|
||||
modifier = Modifier.padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Text(
|
||||
text = "Profile",
|
||||
style = MaterialTheme.typography.titleMedium,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
Section("Profile") {
|
||||
if (state.isLoading) {
|
||||
Box(
|
||||
modifier = Modifier.fillMaxWidth().padding(16.dp),
|
||||
@@ -52,7 +42,6 @@ fun ProfileCard(viewModel: ProfileViewModel = hiltViewModel()) {
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
@@ -95,11 +84,5 @@ private fun ProfileForm(
|
||||
}
|
||||
Text(if (isSaving) "Saving…" else "Save profile")
|
||||
}
|
||||
if (message != null) {
|
||||
Text(
|
||||
text = message,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
if (message != null) Muted(message)
|
||||
}
|
||||
|
||||
@@ -1,34 +1,19 @@
|
||||
@file:Suppress("TooManyFunctions") // Settings screen + per-card helper composables
|
||||
@file:Suppress("TooManyFunctions") // Settings screen + per-section composables
|
||||
|
||||
package com.fabledsword.minstrel.settings.ui
|
||||
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.foundation.verticalScroll
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.PaddingValues
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.foundation.verticalScroll
|
||||
import androidx.compose.material3.AlertDialog
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.ButtonDefaults
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.ElevatedCard
|
||||
import androidx.compose.material3.ExperimentalMaterial3Api
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.Scaffold
|
||||
import androidx.compose.material3.SegmentedButton
|
||||
import androidx.compose.material3.SegmentedButtonDefaults
|
||||
import androidx.compose.material3.SingleChoiceSegmentedButtonRow
|
||||
import androidx.compose.material3.Switch
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.material3.TextButton
|
||||
import androidx.compose.runtime.Composable
|
||||
@@ -37,16 +22,11 @@ import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.vector.ImageVector
|
||||
import androidx.compose.ui.text.style.TextOverflow
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.hilt.navigation.compose.hiltViewModel
|
||||
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
||||
import androidx.navigation.NavHostController
|
||||
import com.composables.icons.lucide.Bell
|
||||
import com.composables.icons.lucide.ChevronRight
|
||||
import com.composables.icons.lucide.ListMusic
|
||||
import com.composables.icons.lucide.LogOut
|
||||
import com.composables.icons.lucide.Lucide
|
||||
@@ -54,10 +34,18 @@ import com.composables.icons.lucide.Shield
|
||||
import com.fabledsword.minstrel.nav.Admin
|
||||
import com.fabledsword.minstrel.nav.NotificationSettings
|
||||
import com.fabledsword.minstrel.nav.Requests
|
||||
import com.fabledsword.minstrel.nav.Settings as SettingsRoute
|
||||
import com.fabledsword.minstrel.nav.ServerUrl
|
||||
import com.fabledsword.minstrel.shared.widgets.ShellContentWindowInsets
|
||||
import com.fabledsword.minstrel.nav.Settings as SettingsRoute
|
||||
import com.fabledsword.minstrel.shared.widgets.MinstrelTopAppBar
|
||||
import com.fabledsword.minstrel.shared.widgets.ShellContentWindowInsets
|
||||
import com.fabledsword.minstrel.theme.ChoiceRow
|
||||
import com.fabledsword.minstrel.theme.RowChevron
|
||||
import com.fabledsword.minstrel.theme.RowSpinner
|
||||
import com.fabledsword.minstrel.theme.Section
|
||||
import com.fabledsword.minstrel.theme.SectionEdge
|
||||
import com.fabledsword.minstrel.theme.SectionGap
|
||||
import com.fabledsword.minstrel.theme.SettingRow
|
||||
import com.fabledsword.minstrel.theme.SwitchRow
|
||||
import com.fabledsword.minstrel.theme.ThemeMode
|
||||
import com.fabledsword.minstrel.theme.ThemePreferenceViewModel
|
||||
import com.fabledsword.minstrel.update.ui.AppVersionCard
|
||||
@@ -119,7 +107,7 @@ fun SettingsScreen(
|
||||
|
||||
@Composable
|
||||
private fun SettingsList(
|
||||
inner: androidx.compose.foundation.layout.PaddingValues,
|
||||
inner: PaddingValues,
|
||||
state: SettingsState,
|
||||
themeMode: ThemeMode,
|
||||
onPickTheme: (ThemeMode) -> Unit,
|
||||
@@ -134,48 +122,26 @@ private fun SettingsList(
|
||||
.fillMaxSize()
|
||||
.padding(inner)
|
||||
.verticalScroll(rememberScrollState())
|
||||
.padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(16.dp),
|
||||
.padding(horizontal = SectionEdge, vertical = SectionEdge),
|
||||
verticalArrangement = Arrangement.spacedBy(SectionGap),
|
||||
) {
|
||||
AccountCard(
|
||||
username = state.username,
|
||||
isAdmin = state.isAdmin,
|
||||
serverUrl = state.serverUrl,
|
||||
AccountSection(
|
||||
state = state,
|
||||
onNavToRequests = onNavToRequests,
|
||||
onNavToNotifications = onNavToNotifications,
|
||||
onNavToAdmin = onNavToAdmin,
|
||||
onSignOutClick = onSignOutClick,
|
||||
)
|
||||
NavTile(
|
||||
icon = Lucide.ListMusic,
|
||||
title = "My requests",
|
||||
subtitle = "Track what you've asked Minstrel to add",
|
||||
onClick = onNavToRequests,
|
||||
)
|
||||
NavTile(
|
||||
icon = Lucide.Bell,
|
||||
title = "Notifications",
|
||||
subtitle = "What reaches you, and where",
|
||||
onClick = onNavToNotifications,
|
||||
)
|
||||
if (state.isAdmin) {
|
||||
NavTile(
|
||||
icon = Lucide.Shield,
|
||||
title = "Admin",
|
||||
subtitle = "Manage requests, quarantine, and users",
|
||||
onClick = onNavToAdmin,
|
||||
)
|
||||
}
|
||||
ProfileCard()
|
||||
PasswordCard()
|
||||
ListenBrainzCard()
|
||||
if (state.diagnosticsEnabledByAdmin) {
|
||||
DiagnosticsCard(
|
||||
optOut = state.diagnosticsOptOut,
|
||||
onToggle = onToggleDiagnostics,
|
||||
)
|
||||
DiagnosticsSection(optOut = state.diagnosticsOptOut, onToggle = onToggleDiagnostics)
|
||||
}
|
||||
NormalizationCard()
|
||||
AppearanceCard(themeMode = themeMode, onPick = onPickTheme)
|
||||
AppearanceSection(themeMode = themeMode, onPick = onPickTheme)
|
||||
StorageCard()
|
||||
AboutCard()
|
||||
SignOutButton(isSigningOut = state.isSigningOut, onSignOut = onSignOutClick)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -205,84 +171,48 @@ private fun SignOutConfirmDialog(onConfirm: () -> Unit, onDismiss: () -> Unit) {
|
||||
)
|
||||
}
|
||||
|
||||
/** Who is signed in, where, the screens that belong to the account, and signing out. */
|
||||
@Composable
|
||||
private fun NavTile(
|
||||
icon: ImageVector,
|
||||
title: String,
|
||||
subtitle: String,
|
||||
onClick: () -> Unit,
|
||||
private fun AccountSection(
|
||||
state: SettingsState,
|
||||
onNavToRequests: () -> Unit,
|
||||
onNavToNotifications: () -> Unit,
|
||||
onNavToAdmin: () -> Unit,
|
||||
onSignOutClick: () -> Unit,
|
||||
) {
|
||||
ElevatedCard(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.clickable(onClick = onClick),
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
Icon(
|
||||
imageVector = icon,
|
||||
contentDescription = null,
|
||||
tint = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
Spacer(Modifier.size(16.dp))
|
||||
Column(modifier = Modifier.weight(1f)) {
|
||||
Text(
|
||||
text = title,
|
||||
style = MaterialTheme.typography.titleMedium,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
Text(
|
||||
text = subtitle,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
Icon(
|
||||
imageVector = Lucide.ChevronRight,
|
||||
contentDescription = null,
|
||||
tint = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun AccountCard(username: String, isAdmin: Boolean, serverUrl: String) {
|
||||
ElevatedCard(modifier = Modifier.fillMaxWidth()) {
|
||||
Column(
|
||||
modifier = Modifier.padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(4.dp),
|
||||
) {
|
||||
Text(
|
||||
text = "Account",
|
||||
style = MaterialTheme.typography.titleMedium,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
val adminTag = if (isAdmin) " · admin" else ""
|
||||
Text(
|
||||
text = when {
|
||||
username.isEmpty() -> "Signed in"
|
||||
else -> "Signed in as $username$adminTag"
|
||||
},
|
||||
style = MaterialTheme.typography.bodyMedium,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
Spacer(Modifier.height(8.dp))
|
||||
Text(
|
||||
text = "Server",
|
||||
style = MaterialTheme.typography.titleSmall,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
Text(
|
||||
text = serverUrl,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis,
|
||||
Section("Account") {
|
||||
SettingRow(
|
||||
title = state.username.ifEmpty { "Signed in" },
|
||||
summary = if (state.isAdmin) "Signed in · admin" else "Signed in",
|
||||
)
|
||||
SettingRow(title = "Server", summary = state.serverUrl)
|
||||
SettingRow(
|
||||
title = "My requests",
|
||||
summary = "What you've asked Minstrel to add",
|
||||
icon = Lucide.ListMusic,
|
||||
onClick = onNavToRequests,
|
||||
) { RowChevron() }
|
||||
SettingRow(
|
||||
title = "Notifications",
|
||||
summary = "What reaches you, and where",
|
||||
icon = Lucide.Bell,
|
||||
onClick = onNavToNotifications,
|
||||
) { RowChevron() }
|
||||
if (state.isAdmin) {
|
||||
SettingRow(
|
||||
title = "Admin",
|
||||
summary = "Requests, quarantine and users",
|
||||
icon = Lucide.Shield,
|
||||
onClick = onNavToAdmin,
|
||||
) { RowChevron() }
|
||||
}
|
||||
SettingRow(
|
||||
title = if (state.isSigningOut) "Signing out…" else "Sign out",
|
||||
icon = Lucide.LogOut,
|
||||
tone = MaterialTheme.colorScheme.error,
|
||||
enabled = !state.isSigningOut,
|
||||
onClick = onSignOutClick,
|
||||
) { if (state.isSigningOut) RowSpinner() }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -290,74 +220,27 @@ private fun AccountCard(username: String, isAdmin: Boolean, serverUrl: String) {
|
||||
// the user turn reporting OFF on this device (battery/privacy) without
|
||||
// touching the account flag. Checked = reporting on = NOT opted out.
|
||||
@Composable
|
||||
private fun DiagnosticsCard(optOut: Boolean, onToggle: (Boolean) -> Unit) {
|
||||
ElevatedCard(modifier = Modifier.fillMaxWidth()) {
|
||||
Column(
|
||||
modifier = Modifier.padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Text(
|
||||
text = "Diagnostics",
|
||||
style = MaterialTheme.typography.titleMedium,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
Row(verticalAlignment = Alignment.CenterVertically) {
|
||||
Column(modifier = Modifier.weight(1f)) {
|
||||
Text(
|
||||
text = "Send diagnostic reports",
|
||||
style = MaterialTheme.typography.bodyLarge,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
Text(
|
||||
text = "Your admin enabled debug mode to investigate an " +
|
||||
"issue. This device streams connectivity and playback " +
|
||||
"diagnostics. Turn off any time.",
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
Spacer(Modifier.size(12.dp))
|
||||
Switch(
|
||||
private fun DiagnosticsSection(optOut: Boolean, onToggle: (Boolean) -> Unit) {
|
||||
Section("Diagnostics") {
|
||||
SwitchRow(
|
||||
title = "Send diagnostic reports",
|
||||
summary = "Your admin turned on debug mode",
|
||||
checked = !optOut,
|
||||
onCheckedChange = { checked -> onToggle(!checked) },
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalMaterial3Api::class)
|
||||
@Composable
|
||||
private fun AppearanceCard(themeMode: ThemeMode, onPick: (ThemeMode) -> Unit) {
|
||||
val options = ThemeMode.entries
|
||||
ElevatedCard(modifier = Modifier.fillMaxWidth()) {
|
||||
Column(
|
||||
modifier = Modifier.padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Text(
|
||||
text = "Appearance",
|
||||
style = MaterialTheme.typography.titleMedium,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
private fun AppearanceSection(themeMode: ThemeMode, onPick: (ThemeMode) -> Unit) {
|
||||
Section("Appearance") {
|
||||
ChoiceRow(
|
||||
title = "Theme",
|
||||
summary = if (themeMode == ThemeMode.SYSTEM) "Follows your device" else null,
|
||||
options = ThemeMode.entries.map { it to displayName(it) },
|
||||
selected = themeMode,
|
||||
onSelect = onPick,
|
||||
)
|
||||
Text(
|
||||
text = "System follows your device setting.",
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
SingleChoiceSegmentedButtonRow(modifier = Modifier.fillMaxWidth()) {
|
||||
options.forEachIndexed { index, mode ->
|
||||
SegmentedButton(
|
||||
selected = mode == themeMode,
|
||||
onClick = { onPick(mode) },
|
||||
shape = SegmentedButtonDefaults.itemShape(
|
||||
index = index,
|
||||
count = options.size,
|
||||
),
|
||||
) { Text(displayName(mode)) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -376,29 +259,3 @@ private fun AboutCard(model: UpdateViewModel = hiltViewModel()) {
|
||||
CheckWhileShown(model)
|
||||
AppVersionCard(model)
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun SignOutButton(isSigningOut: Boolean, onSignOut: () -> Unit) {
|
||||
Button(
|
||||
onClick = onSignOut,
|
||||
enabled = !isSigningOut,
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
colors = ButtonDefaults.buttonColors(
|
||||
containerColor = MaterialTheme.colorScheme.errorContainer,
|
||||
contentColor = MaterialTheme.colorScheme.onErrorContainer,
|
||||
),
|
||||
) {
|
||||
if (isSigningOut) {
|
||||
CircularProgressIndicator(
|
||||
modifier = Modifier.size(20.dp),
|
||||
color = MaterialTheme.colorScheme.onErrorContainer,
|
||||
strokeWidth = 2.dp,
|
||||
)
|
||||
} else {
|
||||
Box(modifier = Modifier.padding(end = 4.dp)) {
|
||||
Icon(Lucide.LogOut, contentDescription = null)
|
||||
}
|
||||
Text("Sign out")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,23 +1,9 @@
|
||||
package com.fabledsword.minstrel.settings.ui
|
||||
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.selection.toggleable
|
||||
import androidx.compose.material3.AlertDialog
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.ButtonDefaults
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.DropdownMenu
|
||||
import androidx.compose.material3.DropdownMenuItem
|
||||
import androidx.compose.material3.ElevatedCard
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.OutlinedButton
|
||||
import androidx.compose.material3.Switch
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.material3.TextButton
|
||||
import androidx.compose.runtime.Composable
|
||||
@@ -25,12 +11,13 @@ import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.semantics.Role
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.hilt.navigation.compose.hiltViewModel
|
||||
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
||||
import com.fabledsword.minstrel.theme.ChoiceRow
|
||||
import com.fabledsword.minstrel.theme.RowSpinner
|
||||
import com.fabledsword.minstrel.theme.Section
|
||||
import com.fabledsword.minstrel.theme.SettingRow
|
||||
import com.fabledsword.minstrel.theme.SwitchRow
|
||||
|
||||
private const val GIB: Long = 1024L * 1024 * 1024
|
||||
private const val MIB: Long = 1024L * 1024
|
||||
@@ -48,44 +35,39 @@ private val CAP_OPTIONS: List<Pair<Long, String>> = listOf(
|
||||
fun StorageCard(viewModel: StorageViewModel = hiltViewModel()) {
|
||||
val state by viewModel.state.collectAsStateWithLifecycle()
|
||||
var showClearConfirm by remember { mutableStateOf(false) }
|
||||
ElevatedCard(modifier = Modifier.fillMaxWidth()) {
|
||||
Column(
|
||||
modifier = Modifier.padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Text(
|
||||
text = "Storage",
|
||||
style = MaterialTheme.typography.titleMedium,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
Section("Storage") {
|
||||
// Two budgets: a liked track that is also recently played counts
|
||||
// against Liked songs only. The prefetch window has no control;
|
||||
// AudioPrefetcher uses the stored value (default 5).
|
||||
CapDropdown(
|
||||
label = "Liked songs",
|
||||
hint = "${formatBytes(state.likedBytes)} used",
|
||||
CapRow(
|
||||
title = "Liked songs",
|
||||
used = state.likedBytes,
|
||||
value = state.settings.likedCapBytes,
|
||||
onSet = viewModel::setLikedCapBytes,
|
||||
)
|
||||
CapDropdown(
|
||||
label = "Recently played",
|
||||
hint = "${formatBytes(state.recentBytes)} used",
|
||||
CapRow(
|
||||
title = "Recently played",
|
||||
used = state.recentBytes,
|
||||
value = state.settings.rollingCapBytes,
|
||||
onSet = viewModel::setRollingCapBytes,
|
||||
)
|
||||
SwitchRow(
|
||||
label = "Keep liked songs offline",
|
||||
hint = "Downloads on Wi-Fi",
|
||||
title = "Keep liked songs offline",
|
||||
summary = "Downloads on Wi-Fi",
|
||||
checked = state.settings.cacheLikedTracks,
|
||||
onCheckedChange = viewModel::setCacheLikedTracks,
|
||||
)
|
||||
StorageActions(
|
||||
isSyncing = state.isSyncing,
|
||||
isClearing = state.isClearing,
|
||||
onSync = viewModel::syncNow,
|
||||
onClear = { showClearConfirm = true },
|
||||
)
|
||||
}
|
||||
SettingRow(
|
||||
title = if (state.isSyncing) "Syncing…" else "Sync library now",
|
||||
enabled = !state.isSyncing,
|
||||
onClick = viewModel::syncNow,
|
||||
) { if (state.isSyncing) RowSpinner() }
|
||||
SettingRow(
|
||||
title = if (state.isClearing) "Clearing…" else "Clear cached audio",
|
||||
tone = MaterialTheme.colorScheme.error,
|
||||
enabled = !state.isClearing,
|
||||
onClick = { showClearConfirm = true },
|
||||
) { if (state.isClearing) RowSpinner() }
|
||||
}
|
||||
if (showClearConfirm) {
|
||||
ClearCacheConfirmDialog(
|
||||
@@ -98,128 +80,19 @@ fun StorageCard(viewModel: StorageViewModel = hiltViewModel()) {
|
||||
}
|
||||
}
|
||||
|
||||
/** A storage budget: how much it holds now, and the cap it may grow to. */
|
||||
@Composable
|
||||
private fun CapDropdown(label: String, hint: String, value: Long, onSet: (Long) -> Unit) {
|
||||
val selectedLabel = CAP_OPTIONS.firstOrNull { it.first == value }?.second
|
||||
?: formatBytes(value)
|
||||
LabeledDropdown(
|
||||
label = label,
|
||||
hint = hint,
|
||||
selectedLabel = selectedLabel,
|
||||
private fun CapRow(title: String, used: Long, value: Long, onSet: (Long) -> Unit) {
|
||||
ChoiceRow(
|
||||
title = title,
|
||||
summary = "${formatBytes(used)} used",
|
||||
options = CAP_OPTIONS,
|
||||
selected = value,
|
||||
onSelect = onSet,
|
||||
selectedLabel = CAP_OPTIONS.firstOrNull { it.first == value }?.second ?: formatBytes(value),
|
||||
)
|
||||
}
|
||||
|
||||
/** A setting's name, with one short line under it. */
|
||||
@Composable
|
||||
private fun RowLabel(label: String, hint: String, modifier: Modifier = Modifier) {
|
||||
Column(modifier = modifier) {
|
||||
Text(
|
||||
text = label,
|
||||
style = MaterialTheme.typography.bodyMedium,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
Text(
|
||||
text = hint,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun SwitchRow(label: String, hint: String, checked: Boolean, onCheckedChange: (Boolean) -> Unit) {
|
||||
Row(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.toggleable(value = checked, role = Role.Switch, onValueChange = onCheckedChange),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
RowLabel(label = label, hint = hint, modifier = Modifier.weight(1f))
|
||||
Spacer(Modifier.size(12.dp))
|
||||
Switch(checked = checked, onCheckedChange = null)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Label and hint on the left, button-with-current-value on the right that
|
||||
* opens a DropdownMenu of [options]. Used by the cap selectors.
|
||||
*/
|
||||
@Composable
|
||||
private fun <T> LabeledDropdown(
|
||||
label: String,
|
||||
hint: String,
|
||||
selectedLabel: String,
|
||||
options: List<Pair<T, String>>,
|
||||
onSelect: (T) -> Unit,
|
||||
) {
|
||||
var expanded by remember { mutableStateOf(false) }
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
RowLabel(label = label, hint = hint, modifier = Modifier.weight(1f))
|
||||
OutlinedButton(onClick = { expanded = true }) {
|
||||
Text(selectedLabel)
|
||||
DropdownMenu(
|
||||
expanded = expanded,
|
||||
onDismissRequest = { expanded = false },
|
||||
) {
|
||||
options.forEach { (value, name) ->
|
||||
DropdownMenuItem(
|
||||
text = { Text(name) },
|
||||
onClick = {
|
||||
expanded = false
|
||||
onSelect(value)
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun StorageActions(
|
||||
isSyncing: Boolean,
|
||||
isClearing: Boolean,
|
||||
onSync: () -> Unit,
|
||||
onClear: () -> Unit,
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
OutlinedButton(
|
||||
onClick = onClear,
|
||||
enabled = !isClearing,
|
||||
) {
|
||||
if (isClearing) {
|
||||
CircularProgressIndicator(
|
||||
modifier = Modifier.size(16.dp),
|
||||
strokeWidth = 2.dp,
|
||||
)
|
||||
Spacer(Modifier.size(8.dp))
|
||||
}
|
||||
Text(if (isClearing) "Clearing…" else "Clear cache")
|
||||
}
|
||||
OutlinedButton(
|
||||
onClick = onSync,
|
||||
enabled = !isSyncing,
|
||||
) {
|
||||
if (isSyncing) {
|
||||
CircularProgressIndicator(
|
||||
modifier = Modifier.size(16.dp),
|
||||
strokeWidth = 2.dp,
|
||||
)
|
||||
Spacer(Modifier.size(8.dp))
|
||||
}
|
||||
Text(if (isSyncing) "Syncing…" else "Sync now")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ClearCacheConfirmDialog(onConfirm: () -> Unit, onDismiss: () -> Unit) {
|
||||
AlertDialog(
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
@file:Suppress("TooManyFunctions") // the settings vocabulary: a plate and its row kinds
|
||||
|
||||
package com.fabledsword.minstrel.theme
|
||||
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.ColumnScope
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.RowScope
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.heightIn
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.selection.toggleable
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.DropdownMenu
|
||||
import androidx.compose.material3.DropdownMenuItem
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.material3.Switch
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.draw.alpha
|
||||
import androidx.compose.ui.draw.clip
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.vector.ImageVector
|
||||
import androidx.compose.ui.semantics.Role
|
||||
import androidx.compose.ui.unit.dp
|
||||
import com.composables.icons.lucide.ChevronRight
|
||||
import com.composables.icons.lucide.ChevronDown
|
||||
import com.composables.icons.lucide.Lucide
|
||||
|
||||
/**
|
||||
* The plate every group of settings sits on. Every section gets the same
|
||||
* plate, so the screen reads as one thing; a section that matters earns its
|
||||
* weight with what is inside it, never with a louder box.
|
||||
*
|
||||
* The rhythm between lines lives here, not at the call sites.
|
||||
*/
|
||||
@Composable
|
||||
fun Section(
|
||||
title: String,
|
||||
modifier: Modifier = Modifier,
|
||||
trailing: @Composable (RowScope.() -> Unit)? = null,
|
||||
content: @Composable ColumnScope.() -> Unit,
|
||||
) {
|
||||
Surface(
|
||||
modifier = modifier.fillMaxWidth(),
|
||||
shape = RoundedCornerShape(FabledSwordFlatTokens.radiusLg),
|
||||
// `iron`, the raised surface, on the `obsidian` ground.
|
||||
color = MaterialTheme.colorScheme.surface,
|
||||
contentColor = MaterialTheme.colorScheme.onSurface,
|
||||
) {
|
||||
Column(
|
||||
modifier = Modifier.padding(SectionInset),
|
||||
verticalArrangement = Arrangement.spacedBy(SectionRow),
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.SpaceBetween,
|
||||
) {
|
||||
Text(title, style = MaterialTheme.typography.titleMedium)
|
||||
if (trailing != null) {
|
||||
Row(verticalAlignment = Alignment.CenterVertically, content = trailing)
|
||||
}
|
||||
}
|
||||
Spacer(Modifier.height(TITLE_GAP - SectionRow))
|
||||
content()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The quiet line: a caption, a count, a consequence. */
|
||||
@Composable
|
||||
fun Muted(said: String, modifier: Modifier = Modifier, tone: Color = Color.Unspecified) {
|
||||
Text(
|
||||
said,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = if (tone == Color.Unspecified) MaterialTheme.colorScheme.onSurfaceVariant else tone,
|
||||
modifier = modifier,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* One line of a settings section: what it is, one short line about it, and
|
||||
* its control at the end. A settings screen is a menu, so the summary is one
|
||||
* short line, never an explanation.
|
||||
*
|
||||
* With [onClick] the whole row is the tap target. [tone] colours the title of
|
||||
* a row reporting something wrong, so the control beside it still reads as a
|
||||
* control.
|
||||
*/
|
||||
@Composable
|
||||
fun SettingRow(
|
||||
title: String,
|
||||
modifier: Modifier = Modifier,
|
||||
summary: String? = null,
|
||||
icon: ImageVector? = null,
|
||||
tone: Color = Color.Unspecified,
|
||||
enabled: Boolean = true,
|
||||
onClick: (() -> Unit)? = null,
|
||||
trailing: @Composable (RowScope.() -> Unit)? = null,
|
||||
) {
|
||||
val tappable = if (onClick != null) {
|
||||
Modifier
|
||||
.clip(RoundedCornerShape(FabledSwordFlatTokens.radiusMd))
|
||||
.clickable(enabled = enabled, role = Role.Button, onClick = onClick)
|
||||
} else {
|
||||
Modifier
|
||||
}
|
||||
Row(
|
||||
modifier = modifier
|
||||
.fillMaxWidth()
|
||||
.heightIn(min = ROW_HEIGHT)
|
||||
.then(tappable)
|
||||
.alpha(if (enabled) 1f else DISABLED_ALPHA),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
if (icon != null) {
|
||||
Icon(
|
||||
icon,
|
||||
contentDescription = null,
|
||||
tint = if (tone == Color.Unspecified) MaterialTheme.colorScheme.onSurfaceVariant else tone,
|
||||
modifier = Modifier.padding(end = ICON_GAP).size(ICON),
|
||||
)
|
||||
}
|
||||
Column(Modifier.weight(1f).padding(vertical = ROW_PAD)) {
|
||||
Text(
|
||||
title,
|
||||
style = MaterialTheme.typography.bodyLarge,
|
||||
color = if (tone == Color.Unspecified) MaterialTheme.colorScheme.onSurface else tone,
|
||||
)
|
||||
if (summary != null) Muted(summary)
|
||||
}
|
||||
if (trailing != null) {
|
||||
Row(
|
||||
modifier = Modifier.padding(start = ICON_GAP),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
content = trailing,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A row whose control is a switch; the whole row toggles it. */
|
||||
@Composable
|
||||
fun SwitchRow(
|
||||
title: String,
|
||||
checked: Boolean,
|
||||
onCheckedChange: (Boolean) -> Unit,
|
||||
modifier: Modifier = Modifier,
|
||||
summary: String? = null,
|
||||
enabled: Boolean = true,
|
||||
) {
|
||||
SettingRow(
|
||||
title = title,
|
||||
summary = summary,
|
||||
enabled = enabled,
|
||||
modifier = modifier
|
||||
.clip(RoundedCornerShape(FabledSwordFlatTokens.radiusMd))
|
||||
.toggleable(value = checked, enabled = enabled, role = Role.Switch, onValueChange = onCheckedChange),
|
||||
) {
|
||||
Switch(checked = checked, onCheckedChange = null, enabled = enabled)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A one-of-N setting: the row shows the current choice, and a tap opens the
|
||||
* choices under it.
|
||||
*/
|
||||
@Composable
|
||||
fun <T> ChoiceRow(
|
||||
title: String,
|
||||
options: List<Pair<T, String>>,
|
||||
selected: T,
|
||||
onSelect: (T) -> Unit,
|
||||
modifier: Modifier = Modifier,
|
||||
summary: String? = null,
|
||||
selectedLabel: String = options.firstOrNull { it.first == selected }?.second.orEmpty(),
|
||||
) {
|
||||
var open by remember { mutableStateOf(false) }
|
||||
Box(modifier) {
|
||||
SettingRow(title = title, summary = summary, onClick = { open = true }) {
|
||||
RowValue(selectedLabel)
|
||||
Icon(
|
||||
Lucide.ChevronDown,
|
||||
contentDescription = null,
|
||||
tint = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
modifier = Modifier.padding(start = 4.dp).size(SMALL_ICON),
|
||||
)
|
||||
}
|
||||
DropdownMenu(expanded = open, onDismissRequest = { open = false }) {
|
||||
options.forEach { (value, name) ->
|
||||
DropdownMenuItem(
|
||||
text = { Text(name) },
|
||||
onClick = {
|
||||
open = false
|
||||
onSelect(value)
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A row's value, set to the right: a count, a version, a size. */
|
||||
@Composable
|
||||
fun RowValue(said: String) {
|
||||
Text(
|
||||
said,
|
||||
style = MaterialTheme.typography.bodyMedium,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
|
||||
/** Work in progress, at the size a row's trailing slot expects. */
|
||||
@Composable
|
||||
fun RowSpinner() {
|
||||
CircularProgressIndicator(modifier = Modifier.size(SPINNER), strokeWidth = 2.dp)
|
||||
}
|
||||
|
||||
/** "This opens something", for a row whose tap leads somewhere. */
|
||||
@Composable
|
||||
fun RowChevron() {
|
||||
Icon(Lucide.ChevronRight, contentDescription = null, tint = MaterialTheme.colorScheme.onSurfaceVariant)
|
||||
}
|
||||
|
||||
/** The screen margin either side of a plate. */
|
||||
val SectionEdge = 16.dp
|
||||
|
||||
/** Between one plate and the next. */
|
||||
val SectionGap = 14.dp
|
||||
|
||||
/** Inside a plate. */
|
||||
val SectionInset = 18.dp
|
||||
|
||||
/** Between one line of a plate and the next. */
|
||||
val SectionRow = 6.dp
|
||||
|
||||
private val TITLE_GAP = 12.dp
|
||||
|
||||
/** Material's one-line list item height: the smallest a thumb reliably hits. */
|
||||
private val ROW_HEIGHT = 48.dp
|
||||
private val ROW_PAD = 6.dp
|
||||
private val ICON = 22.dp
|
||||
private val SMALL_ICON = 16.dp
|
||||
private val ICON_GAP = 14.dp
|
||||
private const val DISABLED_ALPHA = 0.5f
|
||||
private val SPINNER = 16.dp
|
||||
@@ -1,24 +1,23 @@
|
||||
package com.fabledsword.minstrel.update.ui
|
||||
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.ElevatedCard
|
||||
import androidx.compose.material3.LinearProgressIndicator
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.material3.TextButton
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.lifecycle.compose.LifecycleResumeEffect
|
||||
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
||||
import com.fabledsword.minstrel.BuildConfig
|
||||
import com.fabledsword.minstrel.models.UpdateInfo
|
||||
import com.fabledsword.minstrel.settings.ui.formatBytes
|
||||
import com.fabledsword.minstrel.theme.Muted
|
||||
import com.fabledsword.minstrel.theme.RowSpinner
|
||||
import com.fabledsword.minstrel.theme.RowValue
|
||||
import com.fabledsword.minstrel.theme.Section
|
||||
import com.fabledsword.minstrel.theme.SettingRow
|
||||
import com.fabledsword.minstrel.update.data.InstallStage
|
||||
import com.fabledsword.minstrel.update.data.UpdateUiState
|
||||
|
||||
@@ -36,53 +35,41 @@ fun AppVersionCard(model: UpdateViewModel, modifier: Modifier = Modifier) {
|
||||
onPauseOrDispose {}
|
||||
}
|
||||
val state by model.state.collectAsStateWithLifecycle()
|
||||
ElevatedCard(modifier = modifier.fillMaxWidth()) {
|
||||
Column(
|
||||
modifier = Modifier.padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Text(
|
||||
text = "About",
|
||||
style = MaterialTheme.typography.titleMedium,
|
||||
color = MaterialTheme.colorScheme.onSurface,
|
||||
)
|
||||
Text(
|
||||
text = "Installed: Minstrel ${BuildConfig.VERSION_NAME} (${BuildConfig.VERSION_CODE})",
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
Section("About", modifier) {
|
||||
SettingRow(title = "Version") {
|
||||
RowValue("${BuildConfig.VERSION_NAME} (${BuildConfig.VERSION_CODE})")
|
||||
}
|
||||
val offer = state.offer
|
||||
if (offer != null) {
|
||||
Offered(offer, state, model::update)
|
||||
} else {
|
||||
Text(
|
||||
text = versionWords(state, selfUpdates = !BuildConfig.DEBUG),
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
state.problem?.let {
|
||||
Text(it, style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.error)
|
||||
}
|
||||
TextButton(
|
||||
onClick = model::check,
|
||||
enabled = !BuildConfig.DEBUG && !state.checking && state.stage == InstallStage.IDLE,
|
||||
) { Text(if (state.checking) "Checking…" else "Check for updates") }
|
||||
val canCheck = !BuildConfig.DEBUG && !state.checking && state.stage == InstallStage.IDLE
|
||||
SettingRow(
|
||||
title = "Check for updates",
|
||||
summary = versionWords(state, selfUpdates = !BuildConfig.DEBUG),
|
||||
enabled = canCheck || state.checking,
|
||||
onClick = if (canCheck) model::check else null,
|
||||
) { if (state.checking) RowSpinner() }
|
||||
}
|
||||
state.problem?.let { Muted(it, tone = MaterialTheme.colorScheme.error) }
|
||||
}
|
||||
}
|
||||
|
||||
/** The newer build, and the one button that fetches and installs it. */
|
||||
@Composable
|
||||
private fun Offered(offer: UpdateInfo, state: UpdateUiState, update: () -> Unit) {
|
||||
Text(offeredWords(offer, state.ready), style = MaterialTheme.typography.bodyMedium)
|
||||
SettingRow(
|
||||
title = if (state.ready) "Ready to install" else "New version on your server",
|
||||
summary = describe(offer),
|
||||
) {
|
||||
when (state.stage) {
|
||||
InstallStage.IDLE -> Button(onClick = update, modifier = Modifier.fillMaxWidth()) {
|
||||
Text(if (state.ready) "Install" else "Update")
|
||||
InstallStage.IDLE -> Button(onClick = update) { Text(if (state.ready) "Install" else "Update") }
|
||||
InstallStage.DOWNLOADING -> RowSpinner()
|
||||
InstallStage.INSTALLING -> RowValue("Installing…")
|
||||
}
|
||||
InstallStage.DOWNLOADING ->
|
||||
}
|
||||
if (state.stage == InstallStage.DOWNLOADING) {
|
||||
LinearProgressIndicator(progress = { state.progress }, modifier = Modifier.fillMaxWidth())
|
||||
InstallStage.INSTALLING -> Text("Installing…", style = MaterialTheme.typography.bodyMedium)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,14 +82,6 @@ internal fun versionWords(state: UpdateUiState, selfUpdates: Boolean = true): St
|
||||
else -> "This is the newest version."
|
||||
}
|
||||
|
||||
/** The newer build, and whether it is already downloaded. */
|
||||
internal fun offeredWords(offer: UpdateInfo, ready: Boolean): String =
|
||||
if (ready) {
|
||||
"A new version of the app is downloaded and ready: ${describe(offer)}"
|
||||
} else {
|
||||
"A new version of the app is on your server: ${describe(offer)}"
|
||||
}
|
||||
|
||||
/** The name, the channel when it is the test line, and the size. */
|
||||
internal fun describe(offer: UpdateInfo): String {
|
||||
val channel = if (offer.channel == "dev") " (test build)" else ""
|
||||
|
||||
@@ -2,23 +2,32 @@ package com.fabledsword.minstrel.api
|
||||
|
||||
import com.fabledsword.minstrel.auth.AuthStore
|
||||
import com.fabledsword.minstrel.auth.FakeSessionVault
|
||||
import com.fabledsword.minstrel.auth.SessionVault
|
||||
import com.fabledsword.minstrel.cache.db.dao.AuthSessionDao
|
||||
import com.fabledsword.minstrel.cache.db.entities.AuthSessionEntity
|
||||
import io.mockk.coEvery
|
||||
import io.mockk.every
|
||||
import io.mockk.mockk
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.flow.flowOf
|
||||
import kotlinx.coroutines.test.TestScope
|
||||
import kotlinx.coroutines.test.UnconfinedTestDispatcher
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.mockwebserver.Dispatcher
|
||||
import okhttp3.mockwebserver.MockResponse
|
||||
import okhttp3.mockwebserver.MockWebServer
|
||||
import okhttp3.mockwebserver.RecordedRequest
|
||||
import org.junit.jupiter.api.AfterEach
|
||||
import org.junit.jupiter.api.BeforeEach
|
||||
import org.junit.jupiter.api.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertNull
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
class AuthCookieInterceptorTest {
|
||||
@@ -92,6 +101,87 @@ class AuthCookieInterceptorTest {
|
||||
assertNull(authStore.sessionCookie.value)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a refused session asks the UI to open sign-in`() {
|
||||
authStore.setSessionCookie("session=expired")
|
||||
server.enqueue(MockResponse().setResponseCode(401))
|
||||
|
||||
client.newCall(Request.Builder().url(placeholderUrl("/api/test")).build()).execute()
|
||||
|
||||
assertTrue(authStore.sessionRejected.value)
|
||||
}
|
||||
|
||||
// A refusal of a request that carried no cookie says nothing about the
|
||||
// stored login, and must not raise the sign-in screen either.
|
||||
@Test
|
||||
fun `a 401 to a request sent without a cookie signs nobody out`() {
|
||||
server.enqueue(MockResponse().setResponseCode(401))
|
||||
|
||||
client.newCall(Request.Builder().url(placeholderUrl("/api/test")).build()).execute()
|
||||
|
||||
assertFalse(authStore.sessionRejected.value)
|
||||
}
|
||||
|
||||
// The interceptor reads the cookie when the request starts. A sign-in
|
||||
// that lands while an old request is in flight must survive that old
|
||||
// request's 401.
|
||||
@Test
|
||||
fun `a 401 for an older cookie keeps the one stored since`() {
|
||||
authStore.setSessionCookie("session=old")
|
||||
server.dispatcher =
|
||||
object : Dispatcher() {
|
||||
override fun dispatch(request: RecordedRequest): MockResponse {
|
||||
authStore.setSessionCookie("session=new")
|
||||
return MockResponse().setResponseCode(401)
|
||||
}
|
||||
}
|
||||
|
||||
client.newCall(Request.Builder().url(placeholderUrl("/api/test")).build()).execute()
|
||||
|
||||
assertEquals("session=new", authStore.sessionCookie.value)
|
||||
assertFalse(authStore.sessionRejected.value)
|
||||
}
|
||||
|
||||
// At launch the stored cookie loads from the Keystore off the main
|
||||
// thread. A request made before that finishes waits for it rather than
|
||||
// going out without the login.
|
||||
@Test
|
||||
fun `a request made while the stored cookie is still loading carries it`() {
|
||||
val slowVault =
|
||||
object : SessionVault {
|
||||
override fun read(): String? {
|
||||
Thread.sleep(SLOW_VAULT_MS)
|
||||
return "session=stored"
|
||||
}
|
||||
|
||||
override fun write(value: String?): Boolean = true
|
||||
}
|
||||
// The store runs on a real background dispatcher here, so its row
|
||||
// collector lands after construction; the row itself carries the
|
||||
// server URL so that collector cannot reset it to the default.
|
||||
val row = AuthSessionEntity(baseUrl = server.url("/").toString())
|
||||
val dao =
|
||||
mockk<AuthSessionDao> {
|
||||
every { observe() } returns flowOf(row)
|
||||
coEvery { get() } returns row
|
||||
coEvery { upsert(any()) } returns Unit
|
||||
coEvery { setSessionCookie(any()) } returns Unit
|
||||
coEvery { setBaseUrl(any()) } returns Unit
|
||||
}
|
||||
val loading = AuthStore(dao, slowVault, CoroutineScope(SupervisorJob() + Dispatchers.Default))
|
||||
val launchClient =
|
||||
OkHttpClient.Builder()
|
||||
.addInterceptor(AuthCookieInterceptor(loading))
|
||||
.addInterceptor(BaseUrlInterceptor(loading))
|
||||
.build()
|
||||
server.enqueue(MockResponse().setResponseCode(200))
|
||||
|
||||
launchClient.newCall(Request.Builder().url(placeholderUrl("/api/library/sync")).build()).execute()
|
||||
|
||||
assertEquals("session=stored", server.takeRequest().getHeader("Cookie"))
|
||||
assertEquals("session=stored", loading.sessionCookie.value)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `captures Set-Cookie on successful response`() {
|
||||
server.enqueue(
|
||||
@@ -145,4 +235,8 @@ class AuthCookieInterceptorTest {
|
||||
|
||||
assertEquals("session=keep", authStore.sessionCookie.value)
|
||||
}
|
||||
|
||||
private companion object {
|
||||
const val SLOW_VAULT_MS = 300L
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package com.fabledsword.minstrel.auth
|
||||
|
||||
import com.fabledsword.minstrel.cache.db.dao.AuthSessionDao
|
||||
import com.fabledsword.minstrel.cache.db.entities.AuthSessionEntity
|
||||
import com.jakewharton.retrofit2.converter.kotlinx.serialization.asConverterFactory
|
||||
import io.mockk.coEvery
|
||||
import io.mockk.every
|
||||
import io.mockk.mockk
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.flow.flowOf
|
||||
import kotlinx.serialization.json.Json
|
||||
import okhttp3.MediaType.Companion.toMediaType
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.mockwebserver.MockResponse
|
||||
import okhttp3.mockwebserver.MockWebServer
|
||||
import org.junit.jupiter.api.AfterEach
|
||||
import org.junit.jupiter.api.BeforeEach
|
||||
import org.junit.jupiter.api.Test
|
||||
import retrofit2.Retrofit
|
||||
import java.util.concurrent.TimeUnit
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertNotNull
|
||||
|
||||
class AuthControllerTest {
|
||||
private lateinit var server: MockWebServer
|
||||
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
|
||||
@BeforeEach
|
||||
fun setup() {
|
||||
server = MockWebServer().apply { start() }
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
fun teardown() {
|
||||
scope.cancel()
|
||||
server.shutdown()
|
||||
}
|
||||
|
||||
// At launch the controller refreshes the profile (it carries account
|
||||
// flags such as debug mode). The stored login loads from the Keystore
|
||||
// off the main thread, so the refresh has to wait for it rather than
|
||||
// read the user as signed out and skip.
|
||||
@Test
|
||||
fun `the launch profile refresh waits for the stored login`() {
|
||||
val slowVault =
|
||||
object : SessionVault {
|
||||
override fun read(): String? {
|
||||
Thread.sleep(SLOW_VAULT_MS)
|
||||
return "session=stored"
|
||||
}
|
||||
|
||||
override fun write(value: String?): Boolean = true
|
||||
}
|
||||
val row = AuthSessionEntity(baseUrl = server.url("/").toString())
|
||||
val dao =
|
||||
mockk<AuthSessionDao> {
|
||||
every { observe() } returns flowOf(row)
|
||||
coEvery { get() } returns row
|
||||
coEvery { upsert(any()) } returns Unit
|
||||
coEvery { setSessionCookie(any()) } returns Unit
|
||||
coEvery { setBaseUrl(any()) } returns Unit
|
||||
}
|
||||
val store = AuthStore(dao, slowVault, scope)
|
||||
val retrofit =
|
||||
Retrofit.Builder()
|
||||
.baseUrl(server.url("/"))
|
||||
.client(OkHttpClient.Builder().build())
|
||||
.addConverterFactory(json.asConverterFactory("application/json".toMediaType()))
|
||||
.build()
|
||||
server.enqueue(MockResponse().setResponseCode(200).setBody("""{"id":"u1","username":"listener"}"""))
|
||||
|
||||
AuthController(store, json, scope, retrofit)
|
||||
|
||||
val request = server.takeRequest(REQUEST_WAIT_SECONDS, TimeUnit.SECONDS)
|
||||
assertNotNull(request, "the profile refresh never reached the server")
|
||||
assertEquals("/api/me", request.path)
|
||||
}
|
||||
|
||||
private companion object {
|
||||
const val SLOW_VAULT_MS = 300L
|
||||
const val REQUEST_WAIT_SECONDS = 5L
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import com.fabledsword.minstrel.models.PlaylistRef
|
||||
import com.fabledsword.minstrel.models.SystemPlaylistsStatus
|
||||
import org.junit.jupiter.api.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class BuildPlaylistsRowTest {
|
||||
@@ -103,4 +104,40 @@ class BuildPlaylistsRowTest {
|
||||
|
||||
assertTrue(buildSongsLikeRow(emptyList(), SystemPlaylistsStatus(), offline = true).isEmpty())
|
||||
}
|
||||
|
||||
private fun genreMix(name: String, cached: Boolean) = PlaylistRef(
|
||||
id = name,
|
||||
userId = "u",
|
||||
name = name,
|
||||
systemVariant = "genre_mix",
|
||||
trackCount = 50,
|
||||
fullyCached = cached,
|
||||
)
|
||||
|
||||
@Test
|
||||
fun `genre mixes get their own row in name order and stay out of the carousel`() {
|
||||
val owned = listOf(
|
||||
genreMix("Shoegaze mix", cached = true),
|
||||
genreMix("dub mix", cached = true),
|
||||
genreMix("Jazz mix", cached = true),
|
||||
user("u1", cached = true),
|
||||
)
|
||||
val row = buildGenreMixRow(owned, offline = false)
|
||||
val names = row.filterIsInstance<PlaylistRowItem.Real>().map { it.playlist.name }
|
||||
assertEquals(listOf("dub mix", "Jazz mix", "Shoegaze mix"), names)
|
||||
|
||||
val carousel = buildPlaylistsRow(owned, SystemPlaylistsStatus(), offline = false)
|
||||
.filterIsInstance<PlaylistRowItem.Real>().map { it.playlist.id }
|
||||
assertEquals(listOf("u1"), carousel)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `genre mixes are not refreshable, so a cached one plays offline`() {
|
||||
assertFalse(genreMix("Jazz mix", cached = true).refreshable)
|
||||
val owned = listOf(genreMix("A mix", cached = false), genreMix("B mix", cached = true))
|
||||
val names = buildGenreMixRow(owned, offline = true)
|
||||
.filterIsInstance<PlaylistRowItem.Real>().map { it.playlist.name }
|
||||
assertEquals(listOf("B mix", "A mix"), names)
|
||||
assertTrue(buildGenreMixRow(emptyList(), offline = false).isEmpty())
|
||||
}
|
||||
}
|
||||
|
||||
+54
-1
@@ -16,6 +16,8 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/acoustid"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/buildinfo"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/catalogue"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/config"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/coplay"
|
||||
@@ -59,14 +61,40 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
case "restore":
|
||||
if err := runRestore(os.Args[2:]); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "minstrel restore: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := run(); err != nil {
|
||||
err := run()
|
||||
if errors.Is(err, errRestart) {
|
||||
err = restartInPlace()
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "minstrel: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// errRestart is run's answer when the server stopped in order to start again:
|
||||
// after a web restore, so the restored settings are read at boot.
|
||||
var errRestart = errors.New("restart requested")
|
||||
|
||||
// restartInPlace replaces this process with a fresh start of the same binary,
|
||||
// arguments and environment. The process id stays the same, so a container
|
||||
// keeps running without needing a restart policy.
|
||||
func restartInPlace() error {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return fmt.Errorf("restart: %w", err)
|
||||
}
|
||||
// Exec returns only when it failed.
|
||||
return fmt.Errorf("restart: %w", syscall.Exec(exe, os.Args, os.Environ()))
|
||||
}
|
||||
|
||||
// safetyNetScanInterval backstops the fsnotify watcher with a low-frequency
|
||||
// full delta walk, catching anything the watcher missed (inotify watch-limit
|
||||
// exhaustion, dropped events). Fixed — there is no operator configuration.
|
||||
@@ -470,7 +498,20 @@ func run() error {
|
||||
// The sweeper above holds this same instance, so a save from the admin
|
||||
// card changes what it does on its next tick.
|
||||
srv.ReacqSettings = reacqSettings
|
||||
// The server's own backup: the loop runs the nightly schedule, and the
|
||||
// admin page starts, tests and configures the same instance.
|
||||
backups := backup.NewService(pool, logger.With("component", "backup"), buildinfo.Version)
|
||||
backups.Roots = scanner.Roots
|
||||
go backups.Loop(ctx)
|
||||
srv.Backups = backups
|
||||
srv.StreamSecret = cfg.StreamSecret
|
||||
restart := make(chan struct{}, 1)
|
||||
srv.Restart = func() {
|
||||
select {
|
||||
case restart <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
httpServer := &http.Server{
|
||||
Addr: cfg.Server.Address,
|
||||
Handler: srv.Router(),
|
||||
@@ -496,20 +537,32 @@ func run() error {
|
||||
close(errCh)
|
||||
}()
|
||||
|
||||
restarting := false
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.Info("shutdown signal received")
|
||||
case <-restart:
|
||||
logger.Info("restarting to load the restored backup")
|
||||
restarting = true
|
||||
case err := <-errCh:
|
||||
if err != nil {
|
||||
return fmt.Errorf("server: %w", err)
|
||||
}
|
||||
}
|
||||
// Ends the background work (scans, workers, schedules) as a signal would.
|
||||
stop()
|
||||
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := httpServer.Shutdown(shutdownCtx); err != nil {
|
||||
return fmt.Errorf("shutdown: %w", err)
|
||||
}
|
||||
// A backup still running is stopped and records that it was cut off.
|
||||
backups.Close()
|
||||
if restarting {
|
||||
logger.Info("minstrel stopped; starting again")
|
||||
return errRestart
|
||||
}
|
||||
logger.Info("minstrel stopped")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/config"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/logging"
|
||||
)
|
||||
|
||||
const restoreUsage = "usage: minstrel restore [-config PATH] FILE"
|
||||
|
||||
// runRestore loads a backup file into a new Minstrel's database: one with no
|
||||
// accounts and no library yet. It brings the schema up to date first, so a
|
||||
// backup from an older Minstrel loads into this one. The server isn't running
|
||||
// meanwhile, so the library scan waits for it to start.
|
||||
func runRestore(args []string) error {
|
||||
fs := flag.NewFlagSet("restore", flag.ContinueOnError)
|
||||
configPath := fs.String("config", os.Getenv("MINSTREL_CONFIG"), "path to YAML config file")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if fs.NArg() != 1 {
|
||||
return errors.New(restoreUsage)
|
||||
}
|
||||
cfg, err := config.Load(*configPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
if cfg.Database.URL == "" {
|
||||
return errors.New("no database URL configured (set it in the config file or MINSTREL_DATABASE_URL)")
|
||||
}
|
||||
logger, err := logging.New(os.Stderr, cfg.Log.Level, cfg.Log.Format)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init logger: %w", err)
|
||||
}
|
||||
|
||||
f, err := os.Open(fs.Arg(0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
manifest, contents, err := backup.Read(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.Migrate(cfg.Database.URL, logger); err != nil {
|
||||
return fmt.Errorf("migrate: %w", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := db.Open(ctx, cfg.Database.URL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open db: %w", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := backup.Restore(ctx, pool, contents); err != nil {
|
||||
return err
|
||||
}
|
||||
printRestored(os.Stdout, manifest)
|
||||
return nil
|
||||
}
|
||||
|
||||
// printRestored says what came back and what happens next.
|
||||
func printRestored(w io.Writer, m backup.Manifest) {
|
||||
_, _ = fmt.Fprintf(w, "minstrel: restored the backup made %s by Minstrel %s (%d accounts, %d tracks, %d playlists).\n",
|
||||
m.CreatedAt.Local().Format("2 Jan 2006 15:04 MST"), m.Build,
|
||||
m.Rows["users"], m.Rows["tracks"], m.Rows["playlists"])
|
||||
_, _ = fmt.Fprintln(w, "Start the server. The library scan (at startup, or Scan now under Admin › Library) finds each music file at its path, and the artwork, tags and analysis the backup leaves out are fetched and worked out again.")
|
||||
_, _ = fmt.Fprintln(w, "Everyone signs in with the password they had. Keys for other services, and the backup's own destination, are entered again in Admin.")
|
||||
}
|
||||
@@ -83,6 +83,71 @@ Creating the first account (which becomes the admin) requires that token, so
|
||||
whoever reaches a newly exposed server first can't claim it. The token
|
||||
changes on every restart until an account exists.
|
||||
|
||||
## Backups
|
||||
|
||||
Minstrel can back itself up once a night: accounts, likes, playlists,
|
||||
listening history, requests and settings, in one file. It is off until an
|
||||
admin switches it on under **Admin › Backups**. Anything Minstrel can fetch
|
||||
or work out again (artwork, fingerprints, MusicBrainz data) is left out, and
|
||||
so are the keys and passwords Minstrel keeps for other services, which are
|
||||
entered again on a restored server. Sign-in passwords are kept.
|
||||
|
||||
A backup goes to one of:
|
||||
- **an S3 bucket**, path-style (MinIO, Garage, or any S3-compatible store).
|
||||
The key only needs to add and list objects. Files land under `database/`.
|
||||
- **a folder** on the server, under `database/` inside it. Mount a volume
|
||||
there; Minstrel refuses to write into a folder that isn't there.
|
||||
|
||||
Minstrel only adds files, named by date, and never deletes one. Keep as many
|
||||
as you like with the destination's own retention. **Test destination** lists
|
||||
what is there without writing anything.
|
||||
|
||||
**Include the music** also sends the library's files to `music/` in the same
|
||||
destination, laid out as they are under the library folders. After each
|
||||
night's database file has landed, Minstrel sends the files it hasn't sent
|
||||
yet, one at a time, for up to six hours; a large library takes several
|
||||
nights the first time, and the panel shows how much is in. After that, a
|
||||
night sends only new files and files that changed. A changed file goes to
|
||||
its old name, and Minstrel still never deletes anything there, so a file
|
||||
taken out of the library stays in the backup. On a bucket that keeps
|
||||
versions or locks objects, set its retention to keep what you want. A file
|
||||
the bucket refuses (an overwrite a locked object won't take) is listed on
|
||||
the panel and goes again once it changes or the settings are saved; three
|
||||
refusals in a row stop that night's music, since that is usually a key that
|
||||
may not write under `music/`. A bucket's upload limit applies
|
||||
to the music as well.
|
||||
|
||||
### Restoring
|
||||
|
||||
A backup restores into a new Minstrel: one nobody has signed up to yet. If
|
||||
it has already scanned the music, that's fine; the backup's library takes
|
||||
the place of what the scan found.
|
||||
|
||||
1. **Put the music back first**, at the path the library had. The restored
|
||||
settings name the old library folders, and each track is found again by
|
||||
its path. If the backup includes the music, copy what is inside `music/`
|
||||
into the library folder (with several library folders, into the folder
|
||||
that holds them all).
|
||||
2. **Load the backup**, either way:
|
||||
- In the web app, choose **Create account**, then **Restore a backup**.
|
||||
Pick a `minstrel-….tar` file from the backup's `database/` folder and
|
||||
enter the setup token from the log (see *First account*). Minstrel
|
||||
restarts by itself once the backup is in, so it runs on the restored
|
||||
settings.
|
||||
- Or, with the server stopped, run
|
||||
`docker compose run --rm minstrel restore /backups/database/minstrel-20261102T030000Z.tar`,
|
||||
with the file on a volume the container can read. It brings the database
|
||||
schema up to date first, so a backup from an older Minstrel loads into a
|
||||
newer one; a backup from a newer Minstrel is refused.
|
||||
3. **Sign in** with the password you had. Everyone else's accounts, likes,
|
||||
playlists and history are back too.
|
||||
|
||||
The library scan then finds each file, and the artwork, tags, fingerprints
|
||||
and loudness are fetched or worked out again over the following hours. Keys
|
||||
for other services (ListenBrainz, AcoustID, artwork and tag providers,
|
||||
email, download clients) and the backup's own destination are entered again
|
||||
under **Admin**. An app that synced with the old server starts its sync over.
|
||||
|
||||
## Android app
|
||||
|
||||
The app connects over whatever URL you give it, HTTP or HTTPS. Once the
|
||||
|
||||
@@ -56,6 +56,7 @@ value itself:
|
||||
- the API keys and passwords for indexers, Prowlarr and download clients;
|
||||
- the AcoustID key, and the cover-art and tag provider keys;
|
||||
- the SMTP password;
|
||||
- the secret key of the S3 bucket backups go to;
|
||||
- each user's ListenBrainz token and Subsonic password.
|
||||
|
||||
These are encrypted in the database (AES-256-GCM). Each column has its own key,
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||
)
|
||||
|
||||
// GET /api/admin/backups — the backup's verdict, settings and recent runs.
|
||||
func (h *handlers) handleGetBackups(w http.ResponseWriter, r *http.Request) {
|
||||
st, err := h.backups.Status(r.Context())
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "admin backups: status", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, st)
|
||||
}
|
||||
|
||||
func decodeBackupUpdate(w http.ResponseWriter, r *http.Request) (backup.Update, bool) {
|
||||
var u backup.Update
|
||||
if err := json.NewDecoder(r.Body).Decode(&u); err != nil {
|
||||
writeErr(w, apierror.BadRequest("invalid_body", ""))
|
||||
return u, false
|
||||
}
|
||||
return u, true
|
||||
}
|
||||
|
||||
// fieldErr writes a setting that can't be kept as a 400 whose message says
|
||||
// which setting and why, and reports whether err was one.
|
||||
func fieldErr(w http.ResponseWriter, err error) bool {
|
||||
var fe *backup.FieldError
|
||||
if !errors.As(err, &fe) {
|
||||
return false
|
||||
}
|
||||
writeErr(w, apierror.BadRequest("invalid_backup_setting", fe.Message))
|
||||
return true
|
||||
}
|
||||
|
||||
// PUT /api/admin/backups — save the settings. An empty secret_key keeps the
|
||||
// stored one. Returns the new status.
|
||||
func (h *handlers) handleUpdateBackups(w http.ResponseWriter, r *http.Request) {
|
||||
u, ok := decodeBackupUpdate(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := h.backups.Save(r.Context(), u); err != nil {
|
||||
if !fieldErr(w, err) {
|
||||
writeErrWithLog(w, h.logger, "admin backups: save", apierror.Internal(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
h.handleGetBackups(w, r)
|
||||
}
|
||||
|
||||
// POST /api/admin/backups/test — check the destination in the body without
|
||||
// saving it, by listing the backups already there. Nothing is written.
|
||||
func (h *handlers) handleTestBackups(w http.ResponseWriter, r *http.Request) {
|
||||
u, ok := decodeBackupUpdate(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
res, err := h.backups.Test(r.Context(), u)
|
||||
if err != nil {
|
||||
if fieldErr(w, err) {
|
||||
return
|
||||
}
|
||||
msg := strings.TrimPrefix(err.Error(), "backup: ")
|
||||
writeErr(w, &apierror.Error{Status: http.StatusBadGateway, Code: "destination_failed", Message: msg})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, res)
|
||||
}
|
||||
|
||||
// POST /api/admin/backups/run — start a backup now. 202 once it has started;
|
||||
// 409 while one runs or before the destination is set up.
|
||||
func (h *handlers) handleRunBackup(w http.ResponseWriter, r *http.Request) {
|
||||
err := h.backups.Start(r.Context(), backup.TriggerManual)
|
||||
switch {
|
||||
case err == nil:
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
case errors.Is(err, backup.ErrRunning):
|
||||
writeErr(w, apierror.Conflict("backup_running", "A backup is already running."))
|
||||
case errors.Is(err, backup.ErrNotReady):
|
||||
writeErr(w, apierror.Conflict("backup_not_ready", "Set up where backups go first."))
|
||||
case errors.Is(err, backup.ErrClosed):
|
||||
writeErr(w, apierror.Conflict("shutting_down", "The server is shutting down."))
|
||||
default:
|
||||
writeErrWithLog(w, h.logger, "admin backups: start", apierror.Internal(err))
|
||||
}
|
||||
}
|
||||
+15
-1
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/catalogue"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/config"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
|
||||
@@ -40,7 +41,7 @@ import (
|
||||
// Mount attaches /api/* handlers to r. Public endpoints (login) are outside
|
||||
// RequireUser; everything else is gated by the middleware. The events writer
|
||||
// is shared with the Subsonic mount so /rest/scrobble feeds the same store.
|
||||
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, requestSvc *requests.Service, quarantineSvc *quarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService, fpSettings *library.FingerprintSettingsService, loudSettings *library.LoudnessSettingsService, acoustIDLookup *library.AcoustIDLookupWorker) {
|
||||
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, requestSvc *requests.Service, quarantineSvc *quarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService, fpSettings *library.FingerprintSettingsService, loudSettings *library.LoudnessSettingsService, acoustIDLookup *library.AcoustIDLookupWorker, backups *backup.Service, restart func()) {
|
||||
rng := rand.New(rand.NewSource(rand.Int63()))
|
||||
setupToken, err := auth.NewSetupToken()
|
||||
if err != nil {
|
||||
@@ -74,6 +75,8 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
|
||||
fingerprintSettings: fpSettings,
|
||||
loudnessSettings: loudSettings,
|
||||
acoustIDLookup: acoustIDLookup,
|
||||
backups: backups,
|
||||
restart: restart,
|
||||
acquisition: acquire.NewStore(pool),
|
||||
catalogue: cat,
|
||||
wanted: wanted.New(pool, cat, logger),
|
||||
@@ -97,6 +100,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
|
||||
api.Post("/auth/login", h.handleLogin)
|
||||
api.Post("/auth/register", h.handleRegister)
|
||||
api.Get("/auth/setup-status", h.handleSetupStatus)
|
||||
api.Post("/auth/restore", h.handleRestore)
|
||||
api.Post("/auth/forgot-password", h.handleForgotPassword)
|
||||
api.Post("/auth/reset-password", h.handleResetPassword)
|
||||
|
||||
@@ -358,6 +362,11 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
|
||||
admin.Post("/tag-sources/{provider_id}/test", h.handleTestTagSource)
|
||||
admin.Post("/tag-sources/research", h.handleResearchTags)
|
||||
|
||||
admin.Get("/backups", h.handleGetBackups)
|
||||
admin.Put("/backups", h.handleUpdateBackups)
|
||||
admin.Post("/backups/test", h.handleTestBackups)
|
||||
admin.Post("/backups/run", h.handleRunBackup)
|
||||
|
||||
admin.Get("/smtp-config", h.handleGetSMTPConfig)
|
||||
admin.Put("/smtp-config", h.handleUpdateSMTPConfig)
|
||||
admin.Post("/smtp-config/test", h.handleTestSMTPConfig)
|
||||
@@ -429,6 +438,11 @@ type handlers struct {
|
||||
// acoustIDLookup is the AcoustID lookup worker: its settings, its
|
||||
// status and "look up now". Nil serves the defaults (off).
|
||||
acoustIDLookup *library.AcoustIDLookupWorker
|
||||
// backups is the server's own backup: settings, status, test and run.
|
||||
backups *backup.Service
|
||||
// restart stops the server and starts it again in place, so a restored
|
||||
// backup's settings are read at boot. Nil in tests.
|
||||
restart func()
|
||||
// acquisition is the configured indexers and download clients.
|
||||
acquisition *acquire.Store
|
||||
// catalogue is the release catalogue read from MusicBrainz, and
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
)
|
||||
|
||||
// SetupTokenHeader carries the setup token on a restore upload, whose body is
|
||||
// the backup file itself.
|
||||
const SetupTokenHeader = "X-Setup-Token"
|
||||
|
||||
type restoreResp struct {
|
||||
// Accounts is how many accounts came back; everyone signs in with the
|
||||
// password they had.
|
||||
Accounts int64 `json:"accounts"`
|
||||
// Restarting says the server restarts now to load the restored settings.
|
||||
// The scan at its start finds each music file and fills in what the
|
||||
// backup leaves out.
|
||||
Restarting bool `json:"restarting"`
|
||||
}
|
||||
|
||||
// handleRestore implements POST /api/auth/restore: a new server, with no
|
||||
// accounts yet, loads a backup file sent as the request body. It asks for the
|
||||
// setup token from the server log, as creating the first account does, since
|
||||
// the backup's admins become this server's admins. A library the server's
|
||||
// first scan found is replaced by the backup's.
|
||||
//
|
||||
// Then the server restarts. Settings are read once, at boot, so the restored
|
||||
// ones (the library folders among them) take effect the same way they do
|
||||
// after `minstrel restore`, and the startup scan finds each file at its path.
|
||||
//
|
||||
// 200 restored · 400 not_a_backup · 403 setup_token_invalid ·
|
||||
// 409 not_empty (accounts already) · 409 newer_backup
|
||||
func (h *handlers) handleRestore(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
q := dbq.New(h.pool)
|
||||
n, err := q.CountUsers(ctx)
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "restore: count users", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
writeErr(w, apierror.Conflict("not_empty", "this server already has accounts; a backup restores into a new one"))
|
||||
return
|
||||
}
|
||||
if h.requireSetupToken && !h.setupToken.Matches(r.Header.Get(SetupTokenHeader)) {
|
||||
h.logger.Warn("restore: needs the setup token", "setup_token", h.setupToken.Value())
|
||||
writeErr(w, apierror.Forbidden("setup_token_invalid", "the setup token from the server log is required to restore a backup"))
|
||||
return
|
||||
}
|
||||
|
||||
manifest, contents, err := backup.Read(r.Body)
|
||||
if err != nil {
|
||||
var tooBig *http.MaxBytesError
|
||||
if errors.As(err, &tooBig) {
|
||||
writeErr(w, &apierror.Error{Status: http.StatusRequestEntityTooLarge, Code: "too_large", Message: "the file is larger than a backup can be"})
|
||||
return
|
||||
}
|
||||
writeErr(w, apierror.BadRequest("not_a_backup", "that file isn't a Minstrel backup"))
|
||||
return
|
||||
}
|
||||
switch err := backup.Restore(ctx, h.pool, contents); {
|
||||
case errors.Is(err, backup.ErrNotEmpty):
|
||||
writeErr(w, apierror.Conflict("not_empty", "this server already has accounts; a backup restores into a new one"))
|
||||
return
|
||||
case errors.Is(err, backup.ErrNewerSchema):
|
||||
writeErr(w, apierror.Conflict("newer_backup", "the backup was made by a newer Minstrel; update this server first"))
|
||||
return
|
||||
case errors.Is(err, backup.ErrNotABackup):
|
||||
writeErr(w, apierror.BadRequest("not_a_backup", "that file isn't a Minstrel backup"))
|
||||
return
|
||||
case err != nil:
|
||||
writeErrWithLog(w, h.logger, "restore: load the backup", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
|
||||
out := restoreResp{}
|
||||
if out.Accounts, err = q.CountUsers(ctx); err != nil {
|
||||
h.logger.Warn("restore: count restored accounts", "err", err)
|
||||
}
|
||||
h.logger.Info("restored a backup", "made_at", manifest.CreatedAt, "build", manifest.Build,
|
||||
"schema", manifest.Schema, "accounts", out.Accounts)
|
||||
out.Restarting = h.restart != nil
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
if h.restart != nil {
|
||||
// The server drains this request before it stops, so the answer
|
||||
// reaches the browser first.
|
||||
h.restart()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||
)
|
||||
|
||||
// A new server takes a backup only with the setup token from its log, only
|
||||
// while it has no accounts, and only a file that is a backup; then the
|
||||
// backup's accounts sign in again.
|
||||
func TestRestore_ANewServerTakesABackupWithTheSetupToken(t *testing.T) {
|
||||
h, pool := testHandlers(t)
|
||||
ctx := context.Background()
|
||||
truncateLibrary(t, pool)
|
||||
resetUsers(t, h)
|
||||
seedUser(t, pool, "restored", "abcd1234", true)
|
||||
c, err := backup.Snapshot(ctx, pool)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var file bytes.Buffer
|
||||
if _, err := backup.Write(&file, c, time.Now(), "test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resetUsers(t, h)
|
||||
|
||||
token, err := auth.NewSetupToken()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.setupToken = token
|
||||
h.requireSetupToken = true
|
||||
restarts := 0
|
||||
h.restart = func() { restarts++ }
|
||||
restore := func(setupToken string, body []byte) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/auth/restore", bytes.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/x-tar")
|
||||
if setupToken != "" {
|
||||
req.Header.Set(SetupTokenHeader, setupToken)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
h.handleRestore(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
if w := restore("", file.Bytes()); w.Code != http.StatusForbidden {
|
||||
t.Errorf("no setup token: status = %d, want 403", w.Code)
|
||||
}
|
||||
if w := restore("wrong", file.Bytes()); w.Code != http.StatusForbidden {
|
||||
t.Errorf("a wrong setup token: status = %d, want 403", w.Code)
|
||||
}
|
||||
if w := restore(token.Value(), []byte("not a tar file")); w.Code != http.StatusBadRequest ||
|
||||
!strings.Contains(w.Body.String(), "not_a_backup") {
|
||||
t.Errorf("not a backup: status = %d body = %s, want 400 not_a_backup", w.Code, w.Body.String())
|
||||
}
|
||||
var n int
|
||||
if err := pool.QueryRow(ctx, "SELECT count(*) FROM users").Scan(&n); err != nil || n != 0 {
|
||||
t.Fatalf("a refused restore left %d accounts (%v)", n, err)
|
||||
}
|
||||
if restarts != 0 {
|
||||
t.Errorf("a refused restore restarted the server %d times", restarts)
|
||||
}
|
||||
|
||||
w := restore(token.Value(), file.Bytes())
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("restore: status = %d body = %s", w.Code, w.Body.String())
|
||||
}
|
||||
var out restoreResp
|
||||
if err := json.NewDecoder(w.Body).Decode(&out); err != nil || out.Accounts != 1 || !out.Restarting {
|
||||
t.Errorf("restore answered %+v (%v), want one account and a restart", out, err)
|
||||
}
|
||||
if restarts != 1 {
|
||||
t.Errorf("the restore restarted the server %d times, want once so the restored settings load", restarts)
|
||||
}
|
||||
var admin bool
|
||||
if err := pool.QueryRow(ctx, "SELECT is_admin FROM users WHERE username = $1",
|
||||
dbtest.TestUserPrefix+"restored").Scan(&admin); err != nil || !admin {
|
||||
t.Errorf("the restored admin: is_admin = %v, %v", admin, err)
|
||||
}
|
||||
|
||||
if w := restore(token.Value(), file.Bytes()); w.Code != http.StatusConflict ||
|
||||
!strings.Contains(w.Body.String(), "not_empty") {
|
||||
t.Errorf("a second restore: status = %d body = %s, want 409 not_empty", w.Code, w.Body.String())
|
||||
}
|
||||
if restarts != 1 {
|
||||
t.Errorf("a refused second restore restarted the server (%d restarts)", restarts)
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,9 @@ type syncResponse struct {
|
||||
// Responses:
|
||||
// - 204 No Content — no changes since cursor
|
||||
// - 200 OK — JSON syncResponse with upserts, deletes, new cursor
|
||||
// - 410 Gone — cursor older than oldest log row; client must reset
|
||||
// - 410 Gone — the client must reset its cursor to 0: the cursor is
|
||||
// older than a delete the log has dropped, or past anything this server
|
||||
// has handed out
|
||||
// - 401 / 500 — standard envelope errors
|
||||
func (h *handlers) handleLibrarySync(w http.ResponseWriter, r *http.Request) {
|
||||
user, ok := requireUser(w, r)
|
||||
@@ -52,16 +54,17 @@ func (h *handlers) handleLibrarySync(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
q := dbq.New(h.pool)
|
||||
|
||||
// 410 path: if the requested cursor is older than the oldest row,
|
||||
// tell the client to reset. (No compaction job in this slice; the
|
||||
// path exists for future compaction support.)
|
||||
// The log keeps every entity's latest change, so a client syncing from
|
||||
// 0 gets the whole library. It drops deletes older than its retention,
|
||||
// and a cursor below the newest dropped one may have missed it: the
|
||||
// client resets and syncs again from 0.
|
||||
if since > 0 {
|
||||
minCursor, err := q.GetMinLibraryChangeCursor(ctx)
|
||||
trimmed, err := q.GetLibraryChangesTrimmedThrough(ctx)
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "library_sync: GetMinLibraryChangeCursor", err)
|
||||
writeErrWithLog(w, h.logger, "library_sync: GetLibraryChangesTrimmedThrough", err)
|
||||
return
|
||||
}
|
||||
if minCursor > 0 && since+1 < minCursor {
|
||||
if since < trimmed {
|
||||
writeErr(w, &apierror.Error{
|
||||
Status: http.StatusGone,
|
||||
Code: "cursor_too_old",
|
||||
@@ -69,6 +72,23 @@ func (h *handlers) handleLibrarySync(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
return
|
||||
}
|
||||
// A cursor past anything this server has handed out comes from a
|
||||
// server restored from an older backup: the changes between the
|
||||
// backup and the cursor are gone, and the ones made from here on would
|
||||
// reuse those ids. The app resets the same way.
|
||||
lastCursor, err := q.GetLastIssuedLibraryChangeCursor(ctx)
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "library_sync: GetLastIssuedLibraryChangeCursor", err)
|
||||
return
|
||||
}
|
||||
if since > lastCursor {
|
||||
writeErr(w, &apierror.Error{
|
||||
Status: http.StatusGone,
|
||||
Code: "cursor_ahead",
|
||||
Message: "this server holds no changes up to that cursor; reset client cursor to 0 and resync",
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
changes, err := q.GetLibraryChangesSince(ctx, dbq.GetLibraryChangesSinceParams{
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
|
||||
@@ -204,3 +205,83 @@ func TestSplitOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A server restored from an older backup has handed out fewer cursors than an
|
||||
// app remembers; the app is told to reset rather than wait for ids it would
|
||||
// then skip. A compacted log is not that: the cursor the server handed out
|
||||
// still counts.
|
||||
func TestLibrarySync_ACursorPastTheServersLastChangeIsReset(t *testing.T) {
|
||||
h, pool := testHandlers(t)
|
||||
truncateLibrary(t, pool)
|
||||
u := seedUser(t, pool, "alice", "x", false)
|
||||
|
||||
ctx := context.Background()
|
||||
var artistID string
|
||||
if err := pool.QueryRow(ctx,
|
||||
`INSERT INTO artists (name, sort_name) VALUES ('A', 'A') RETURNING id::text`).Scan(&artistID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := sync.LogChange(ctx, tx, sync.EntityArtist, artistID, sync.OpUpsert); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var last int64
|
||||
if err := pool.QueryRow(ctx, `SELECT max(id) FROM library_changes`).Scan(&last); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
w := callLibrarySync(h, u, strconv.FormatInt(last+50, 10))
|
||||
if w.Code != http.StatusGone || !strings.Contains(w.Body.String(), "cursor_ahead") {
|
||||
t.Errorf("a cursor past the last change: status = %d body = %s, want 410 cursor_ahead", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
// The compactor trims every row; the app's cursor is still one this
|
||||
// server handed out.
|
||||
if _, err := pool.Exec(ctx, `DELETE FROM library_changes`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w := callLibrarySync(h, u, strconv.FormatInt(last, 10)); w.Code != http.StatusNoContent {
|
||||
t.Errorf("the last cursor after compaction: status = %d body = %s, want 204", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// An app is sent back to 0 only when its cursor is below a delete the log has
|
||||
// dropped, which it may have missed. A cursor at that delete has seen it, and
|
||||
// one between dropped and kept rows replays what is still there.
|
||||
func TestLibrarySync_ACursorBelowADroppedDeleteIsReset(t *testing.T) {
|
||||
h, pool := testHandlers(t)
|
||||
truncateLibrary(t, pool)
|
||||
u := seedUser(t, pool, "alice", "x", false)
|
||||
ctx := context.Background()
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('artist', $1, 'upsert')`,
|
||||
strconv.Itoa(i)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var dropped int64
|
||||
if err := pool.QueryRow(ctx, `SELECT min(id) + 2 FROM library_changes`).Scan(&dropped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE library_changes_trimmed SET through = $1`, dropped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if w := callLibrarySync(h, u, strconv.FormatInt(dropped-1, 10)); w.Code != http.StatusGone ||
|
||||
!strings.Contains(w.Body.String(), "cursor_too_old") {
|
||||
t.Errorf("a cursor below the dropped delete: status = %d body = %s, want 410 cursor_too_old", w.Code, w.Body.String())
|
||||
}
|
||||
if w := callLibrarySync(h, u, strconv.FormatInt(dropped, 10)); w.Code != http.StatusOK {
|
||||
t.Errorf("a cursor at the dropped delete: status = %d body = %s, want 200", w.Code, w.Body.String())
|
||||
}
|
||||
if w := callLibrarySync(h, u, "0"); w.Code != http.StatusOK {
|
||||
t.Errorf("a sync from 0: status = %d body = %s, want 200", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/config"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/eventbus"
|
||||
@@ -467,7 +468,7 @@ func TestRoutesRegisteredInMount(t *testing.T) {
|
||||
r := chi.NewRouter()
|
||||
w := playevents.NewWriter(h.pool, slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
30*time.Minute, 0.5, 30000)
|
||||
Mount(r, h.pool, h.logger, w, config.RecommendationConfig{RadioSize: 50, RadioSizeMax: 200, RecentlyPlayedHours: 1}, h.recSettings, h.requests, h.quarantine, h.tracks, h.playlists, h.coverart, h.coverSettings, h.tagSettings, h.scanner, h.scanCfg, h.dataDir, nil, eventbus.New(), nil, nil, h.netSettings, nil, nil, nil, nil)
|
||||
Mount(r, h.pool, h.logger, w, config.RecommendationConfig{RadioSize: 50, RadioSizeMax: 200, RecentlyPlayedHours: 1}, h.recSettings, h.requests, h.quarantine, h.tracks, h.playlists, h.coverart, h.coverSettings, h.tagSettings, h.scanner, h.scanCfg, h.dataDir, nil, eventbus.New(), nil, nil, h.netSettings, nil, nil, nil, nil, backup.NewService(h.pool, h.logger, "test"), nil)
|
||||
|
||||
paths := []string{
|
||||
"/api/artists",
|
||||
|
||||
@@ -88,6 +88,8 @@ func bucketRecSource(src string) recFamily {
|
||||
return recFamily{"for_you", "For You", intentGoTo}
|
||||
case src == "songs_like_artist":
|
||||
return recFamily{"songs_like_artist", "Songs like…", intentGoTo}
|
||||
case src == "genre_mix":
|
||||
return recFamily{"genre_mix", "Genre mixes", intentGoTo}
|
||||
case src == "radio" || strings.HasPrefix(src, "radio:"):
|
||||
return recFamily{"radio", "Radio", intentGoTo}
|
||||
case src == "discover":
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The backup file is laid out so that what didn't change between two days is
|
||||
// the same bytes both times, which suits a deduplicating off-site copy such as
|
||||
// Fabled Archives' restic:
|
||||
//
|
||||
// - Never compressed. Compression rewrites every byte after the first
|
||||
// change, so no chunk would match again.
|
||||
// - One member per table, tables/<table>.jsonl: one row per line, in
|
||||
// primary-key order, keys sorted, no spacing.
|
||||
// - What changes every run lives only in manifest.json, the first member.
|
||||
// Member headers are fixed (mtime 0, owner 0, mode 0644).
|
||||
// - A new dated name each run, so the target keeps a history and a sender
|
||||
// only ever adds.
|
||||
const (
|
||||
FormatName = "minstrel-backup-tar"
|
||||
FormatVersion = 1
|
||||
|
||||
manifestName = "manifest.json"
|
||||
tablesDir = "tables/"
|
||||
tableSuffix = ".jsonl"
|
||||
namePrefix = "minstrel-"
|
||||
nameSuffix = ".tar"
|
||||
|
||||
// maxMember bounds any one member a read takes in: far past the largest
|
||||
// library's tracks table, small enough that a hostile file can't fill
|
||||
// memory.
|
||||
maxMember = 1 << 30
|
||||
)
|
||||
|
||||
// Manifest is the file's first member.
|
||||
type Manifest struct {
|
||||
Format string `json:"format"`
|
||||
FormatVersion int `json:"format_version"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
// Build is the Minstrel version that wrote the file.
|
||||
Build string `json:"build"`
|
||||
// Schema is the database's migration version when the file was written.
|
||||
// A restore refuses a file from a newer schema than its own.
|
||||
Schema int64 `json:"schema"`
|
||||
// Rows is each table's row count, so a read can tell a whole member from
|
||||
// a cut-off one.
|
||||
Rows map[string]int `json:"rows"`
|
||||
}
|
||||
|
||||
// FileName is a backup's file name: minstrel-20261102T030000Z.tar, in UTC, so
|
||||
// names sort by time.
|
||||
func FileName(at time.Time) string {
|
||||
return namePrefix + at.UTC().Format("20060102T150405Z") + nameSuffix
|
||||
}
|
||||
|
||||
// IsFileName reports whether a name is one FileName gives.
|
||||
func IsFileName(name string) bool {
|
||||
stamp, ok := strings.CutPrefix(name, namePrefix)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
stamp, ok = strings.CutSuffix(stamp, nameSuffix)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, err := time.Parse("20060102T150405Z", stamp)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// Write writes c as a backup file to w and returns its manifest.
|
||||
func Write(w io.Writer, c Contents, at time.Time, build string) (Manifest, error) {
|
||||
names := make([]string, 0, len(c.Tables))
|
||||
m := Manifest{
|
||||
Format: FormatName,
|
||||
FormatVersion: FormatVersion,
|
||||
CreatedAt: at.UTC(),
|
||||
Build: build,
|
||||
Schema: c.Schema,
|
||||
Rows: make(map[string]int, len(c.Tables)),
|
||||
}
|
||||
for name, lines := range c.Tables {
|
||||
names = append(names, name)
|
||||
m.Rows[name] = len(lines)
|
||||
}
|
||||
slices.Sort(names)
|
||||
|
||||
manifest, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
tw := tar.NewWriter(w)
|
||||
if err := member(tw, manifestName, append(manifest, '\n')); err != nil {
|
||||
return m, err
|
||||
}
|
||||
for _, name := range names {
|
||||
var body bytes.Buffer
|
||||
for _, line := range c.Tables[name] {
|
||||
body.Write(line)
|
||||
body.WriteByte('\n')
|
||||
}
|
||||
if err := member(tw, tablesDir+name+tableSuffix, body.Bytes()); err != nil {
|
||||
return m, err
|
||||
}
|
||||
}
|
||||
return m, tw.Close()
|
||||
}
|
||||
|
||||
// WriteFile writes c into dir under its dated name. The bytes go to a hidden
|
||||
// .part file first, are synced, and only then take the backup's name, so
|
||||
// anything watching the folder never sees half a backup under a backup's
|
||||
// name, and a run that fails leaves nothing behind.
|
||||
func WriteFile(dir string, c Contents, at time.Time, build string) (string, Manifest, error) {
|
||||
final := filepath.Join(dir, FileName(at))
|
||||
part := filepath.Join(dir, "."+FileName(at)+".part")
|
||||
f, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
|
||||
if err != nil {
|
||||
return "", Manifest{}, fmt.Errorf("backup: create %s: %w", part, err)
|
||||
}
|
||||
m, err := Write(f, c, at, build)
|
||||
if err == nil {
|
||||
err = f.Sync()
|
||||
}
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err == nil {
|
||||
err = os.Rename(part, final)
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(part)
|
||||
return "", m, fmt.Errorf("backup: write %s: %w", final, err)
|
||||
}
|
||||
if d, err := os.Open(dir); err == nil {
|
||||
_ = d.Sync()
|
||||
_ = d.Close()
|
||||
}
|
||||
return final, m, nil
|
||||
}
|
||||
|
||||
func member(tw *tar.Writer, name string, data []byte) error {
|
||||
hdr := &tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(data)),
|
||||
ModTime: time.Unix(0, 0),
|
||||
Typeflag: tar.TypeReg,
|
||||
Format: tar.FormatUSTAR,
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tw.Write(data)
|
||||
return err
|
||||
}
|
||||
|
||||
// Read reads a backup file, refusing anything that isn't a whole one.
|
||||
func Read(r io.Reader) (Manifest, Contents, error) {
|
||||
var m Manifest
|
||||
c := Contents{Tables: map[string][][]byte{}}
|
||||
tr := tar.NewReader(r)
|
||||
first := true
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return m, c, fmt.Errorf("%w: %w", ErrNotABackup, err)
|
||||
}
|
||||
if hdr.Size > maxMember {
|
||||
return m, c, fmt.Errorf("%w: %s is too large", ErrNotABackup, hdr.Name)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(tr, maxMember))
|
||||
if err != nil {
|
||||
return m, c, fmt.Errorf("%w: %w", ErrNotABackup, err)
|
||||
}
|
||||
if first {
|
||||
first = false
|
||||
if hdr.Name != manifestName || json.Unmarshal(data, &m) != nil || m.Format != FormatName {
|
||||
return m, c, ErrNotABackup
|
||||
}
|
||||
if m.FormatVersion > FormatVersion {
|
||||
return m, c, fmt.Errorf("%w: it was written by a newer Minstrel (format %d)", ErrNotABackup, m.FormatVersion)
|
||||
}
|
||||
c.Schema = m.Schema
|
||||
continue
|
||||
}
|
||||
table, ok := strings.CutPrefix(hdr.Name, tablesDir)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if table, ok = strings.CutSuffix(table, tableSuffix); !ok || table == "" {
|
||||
continue
|
||||
}
|
||||
var lines [][]byte
|
||||
for line := range bytes.SplitSeq(data, []byte("\n")) {
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
if !json.Valid(line) {
|
||||
return m, c, fmt.Errorf("%w: a row in %s is not JSON", ErrNotABackup, table)
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
c.Tables[table] = lines
|
||||
}
|
||||
if first {
|
||||
return m, c, ErrNotABackup
|
||||
}
|
||||
for table, want := range m.Rows {
|
||||
if got := len(c.Tables[table]); got != want {
|
||||
return m, c, fmt.Errorf("%w: %s holds %d rows, the manifest says %d", ErrNotABackup, table, got, want)
|
||||
}
|
||||
}
|
||||
return m, c, nil
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func sample() Contents {
|
||||
return Contents{
|
||||
Schema: 98,
|
||||
Tables: map[string][][]byte{
|
||||
"users": {[]byte(`{"id":"u1","username":"ada"}`), []byte(`{"id":"u2","username":"bo"}`)},
|
||||
"tracks": {[]byte(`{"file_path":"/music/a.flac","id":"t1"}`)},
|
||||
"wants": {},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func members(t *testing.T, file []byte) map[string][]byte {
|
||||
t.Helper()
|
||||
out := map[string][]byte{}
|
||||
tr := tar.NewReader(bytes.NewReader(file))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return out
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, _ := io.ReadAll(tr)
|
||||
out[hdr.Name] = data
|
||||
}
|
||||
}
|
||||
|
||||
func TestABackupFileReadsBackAsWritten(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC)
|
||||
if _, err := Write(&buf, sample(), at, "2026.11.02.0300"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, c, err := Read(&buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Schema != 98 || c.Schema != 98 || m.Build != "2026.11.02.0300" || !m.CreatedAt.Equal(at) {
|
||||
t.Errorf("manifest = %+v", m)
|
||||
}
|
||||
want := sample()
|
||||
for name, lines := range want.Tables {
|
||||
got := c.Tables[name]
|
||||
if len(got) != len(lines) {
|
||||
t.Errorf("%s: %d rows, want %d", name, len(got), len(lines))
|
||||
continue
|
||||
}
|
||||
for i := range lines {
|
||||
if !bytes.Equal(got[i], lines[i]) {
|
||||
t.Errorf("%s row %d = %s, want %s", name, i, got[i], lines[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Two days over unchanged data differ only in the manifest, so a
|
||||
// deduplicating copy stores the tables once.
|
||||
func TestOnlyTheManifestChangesBetweenDays(t *testing.T) {
|
||||
var day1, day2 bytes.Buffer
|
||||
if _, err := Write(&day1, sample(), time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC), "a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Write(&day2, sample(), time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC), "b"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, b := members(t, day1.Bytes()), members(t, day2.Bytes())
|
||||
if len(a) != 4 {
|
||||
t.Fatalf("members = %d, want the manifest and three tables", len(a))
|
||||
}
|
||||
for name, data := range a {
|
||||
same := bytes.Equal(data, b[name])
|
||||
if name == manifestName && same {
|
||||
t.Error("the manifest is identical across days; it should carry the run's time")
|
||||
}
|
||||
if name != manifestName && !same {
|
||||
t.Errorf("%s differs between days over the same rows", name)
|
||||
}
|
||||
}
|
||||
// The table members' bytes, headers included, sit unchanged in both files.
|
||||
tracks := a[tablesDir+"tracks"+tableSuffix]
|
||||
if !bytes.Contains(day1.Bytes(), tracks) || !bytes.Contains(day2.Bytes(), tracks) {
|
||||
t.Error("the tracks member is not stored verbatim")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadRefusesWhatIsNotAWholeBackup(t *testing.T) {
|
||||
var good bytes.Buffer
|
||||
if _, err := Write(&good, sample(), time.Now(), "x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tarOf := func(name string, data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = member(tw, name, data)
|
||||
_ = tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
short := func() []byte {
|
||||
// The manifest promises two users; the member holds one.
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = member(tw, manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":1,"rows":{"users":2}}`))
|
||||
_ = member(tw, tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`+"\n"))
|
||||
_ = tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
cases := map[string][]byte{
|
||||
"empty": nil,
|
||||
"not a tar": []byte("hello"),
|
||||
"cut off": good.Bytes()[:good.Len()/2],
|
||||
"no manifest": tarOf(tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`)),
|
||||
"another format": tarOf(manifestName, []byte(`{"format":"fablednotary-backup-tar","format_version":1}`)),
|
||||
"a newer format": tarOf(manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":99}`)),
|
||||
"a member short": short(),
|
||||
}
|
||||
for name, file := range cases {
|
||||
if _, _, err := Read(bytes.NewReader(file)); !errors.Is(err, ErrNotABackup) {
|
||||
t.Errorf("%s: err = %v, want ErrNotABackup", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteFileLandsUnderTheDatedNameOnly(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC)
|
||||
path, _, err := WriteFile(dir, sample(), at, "x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if filepath.Base(path) != "minstrel-20261102T030000Z.tar" {
|
||||
t.Errorf("path = %s", path)
|
||||
}
|
||||
entries, _ := os.ReadDir(dir)
|
||||
if len(entries) != 1 || entries[0].Name() != filepath.Base(path) {
|
||||
t.Errorf("the folder holds %v, want only the backup", entries)
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
if _, _, err := Read(f); err != nil {
|
||||
t.Errorf("the written file doesn't read: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedWriteFileLeavesNothingBehind(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "missing")
|
||||
if _, _, err := WriteFile(dir, sample(), time.Now(), "x"); err == nil {
|
||||
t.Fatal("writing into a missing folder succeeded")
|
||||
}
|
||||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||
t.Errorf("the failed write created %s", dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileNamesSortByTimeAndAreRecognised(t *testing.T) {
|
||||
earlier := FileName(time.Date(2026, 1, 9, 23, 0, 0, 0, time.UTC))
|
||||
// 01:00 at UTC-5 is 06:00 UTC: the name is in UTC, not the clock's zone.
|
||||
later := FileName(time.Date(2026, 1, 10, 1, 0, 0, 0, time.FixedZone("x", -5*3600)))
|
||||
if later != "minstrel-20260110T060000Z.tar" {
|
||||
t.Errorf("name = %s, want it in UTC", later)
|
||||
}
|
||||
if earlier >= later {
|
||||
t.Errorf("%s does not sort before %s", earlier, later)
|
||||
}
|
||||
for name, want := range map[string]bool{
|
||||
earlier: true,
|
||||
"minstrel-20261102T030000Z.tar": true,
|
||||
".minstrel-20261102T030000Z.tar.part": false,
|
||||
"minstrel-yesterday.tar": false,
|
||||
"fablednotary-20261102T030000Z.tar": false,
|
||||
} {
|
||||
if IsFileName(name) != want {
|
||||
t.Errorf("IsFileName(%q) = %v, want %v", name, !want, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -189,7 +189,16 @@ func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) {
|
||||
t.Fatalf("restore from a newer schema: err = %v, want ErrNewerSchema", err)
|
||||
}
|
||||
|
||||
mustExec(t, pool, "TRUNCATE users, artists, library_changes CASCADE")
|
||||
// A new server scans its music before anyone signs up, so it already has
|
||||
// a library, here with a track the backup doesn't know. It still takes
|
||||
// the backup: the backup's library replaces what the scan found, which the
|
||||
// next scan finds again under the backup's ids.
|
||||
mustExec(t, pool, "TRUNCATE users CASCADE")
|
||||
scannedArtist := mustID(t, pool, `INSERT INTO artists (name, sort_name) VALUES ('Scanned', 'scanned') RETURNING id::text`)
|
||||
scannedAlbum := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id) VALUES ('Scanned', 'scanned', $1) RETURNING id::text`, scannedArtist)
|
||||
scanned := mustID(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format)
|
||||
VALUES ('Scanned', $1, $2, 1000, '/music/b.flac', 10, 'flac') RETURNING id::text`, scannedAlbum, scannedArtist)
|
||||
mustExec(t, pool, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert')`, scanned)
|
||||
if err := backup.Restore(ctx, pool, first); err != nil {
|
||||
t.Fatalf("restore: %v", err)
|
||||
}
|
||||
@@ -224,10 +233,19 @@ func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) {
|
||||
}
|
||||
|
||||
// The change feed carries on after the restored ids.
|
||||
var restoredMax, next int64
|
||||
var restoredMax, issued, next int64
|
||||
if err := pool.QueryRow(ctx, "SELECT max(id) FROM library_changes").Scan(&restoredMax); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The sequence says the restored history ends there, which is what the
|
||||
// library sync compares an app's cursor with.
|
||||
if err := pool.QueryRow(ctx, `SELECT COALESCE(pg_sequence_last_value(
|
||||
pg_get_serial_sequence('library_changes', 'id')::regclass), 0)`).Scan(&issued); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if issued != restoredMax {
|
||||
t.Errorf("the change sequence stands at %d, want the restored %d", issued, restoredMax)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op)
|
||||
VALUES ('track', $1, 'upsert') RETURNING id`, track).Scan(&next); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
)
|
||||
|
||||
const (
|
||||
// The music's folder inside the destination, beside database/.
|
||||
musicDir = "music"
|
||||
// MusicBudget is how long one run may spend sending music. A library
|
||||
// bigger than a night's worth carries on from where it stopped on the
|
||||
// next run, so the first copy of a large library spans several nights.
|
||||
MusicBudget = 6 * time.Hour
|
||||
musicBatch = 200
|
||||
// refusedInARow stops a pass: one refusal is that file, several in a row
|
||||
// are the key or the bucket refusing the music as a whole.
|
||||
refusedInARow = 3
|
||||
)
|
||||
|
||||
// MusicProgress is how much of the library the backup holds as it is now.
|
||||
type MusicProgress struct {
|
||||
TotalFiles int64 `json:"total_files"`
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
SentFiles int64 `json:"sent_files"`
|
||||
SentBytes int64 `json:"sent_bytes"`
|
||||
RefusedFiles int64 `json:"refused_files"`
|
||||
Refused []RefusedFile `json:"refused"`
|
||||
}
|
||||
|
||||
// RefusedFile is a library file the destination would not take, with its
|
||||
// answer. It is tried again once the file changes.
|
||||
type RefusedFile struct {
|
||||
Path string `json:"path"`
|
||||
Error string `json:"error"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// musicSent is what one run's music pass did.
|
||||
type musicSent struct {
|
||||
files int
|
||||
bytes int64
|
||||
note string
|
||||
}
|
||||
|
||||
// commonRoot is the deepest folder holding every library root, so the music's
|
||||
// paths in the backup are the same whichever root a file is under.
|
||||
func commonRoot(roots []string) string {
|
||||
if len(roots) == 0 {
|
||||
return ""
|
||||
}
|
||||
common := strings.Split(filepath.Clean(roots[0]), string(filepath.Separator))
|
||||
for _, r := range roots[1:] {
|
||||
parts := strings.Split(filepath.Clean(r), string(filepath.Separator))
|
||||
n := 0
|
||||
for n < len(common) && n < len(parts) && common[n] == parts[n] {
|
||||
n++
|
||||
}
|
||||
common = common[:n]
|
||||
}
|
||||
root := strings.Join(common, string(filepath.Separator))
|
||||
if root == "" {
|
||||
return string(filepath.Separator)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
// musicPath is file's path inside music/, relative to root, with forward
|
||||
// slashes. A file outside root keeps its whole path.
|
||||
func musicPath(root, file string) string {
|
||||
rel, err := filepath.Rel(root, file)
|
||||
if root == "" || err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
rel = strings.TrimPrefix(filepath.Clean(file), string(filepath.Separator))
|
||||
}
|
||||
return musicDir + "/" + filepath.ToSlash(rel)
|
||||
}
|
||||
|
||||
func (s *Service) musicProgress(ctx context.Context) (*MusicProgress, error) {
|
||||
q := dbq.New(s.pool)
|
||||
p, err := q.BackupMusicProgress(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read music progress: %w", err)
|
||||
}
|
||||
refused, err := q.RefusedBackupMusic(ctx, recentRuns)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: list refused music: %w", err)
|
||||
}
|
||||
out := &MusicProgress{
|
||||
TotalFiles: p.TotalFiles, TotalBytes: p.TotalBytes,
|
||||
SentFiles: p.SentFiles, SentBytes: p.SentBytes,
|
||||
RefusedFiles: p.RefusedFiles, Refused: make([]RefusedFile, 0, len(refused)),
|
||||
}
|
||||
for _, r := range refused {
|
||||
out.Refused = append(out.Refused, RefusedFile{Path: r.FilePath, Error: r.Error, At: r.CheckedAt.Time})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sendMusic sends library files the destination doesn't hold as they are now,
|
||||
// one at a time in path order, until all are sent or ctx ends. Every file is
|
||||
// recorded as it lands, so a run cut short loses nothing. It never deletes
|
||||
// anything at the destination.
|
||||
func (s *Service) sendMusic(ctx context.Context, st dbq.BackupSetting) musicSent {
|
||||
var out musicSent
|
||||
q := dbq.New(s.pool)
|
||||
var roots []string
|
||||
if s.Roots != nil {
|
||||
roots = s.Roots()
|
||||
}
|
||||
root := commonRoot(roots)
|
||||
var bucket Bucket
|
||||
if st.Destination == DestinationS3 {
|
||||
b, err := bucketOf(st)
|
||||
if err != nil {
|
||||
out.note = describe(err)
|
||||
return out
|
||||
}
|
||||
bucket = b
|
||||
}
|
||||
|
||||
after := ""
|
||||
refused := 0
|
||||
for {
|
||||
rows, err := q.PendingBackupMusic(ctx, dbq.PendingBackupMusicParams{AfterPath: after, MaxRows: musicBatch})
|
||||
if err != nil {
|
||||
out.note = musicStopped(ctx, err)
|
||||
return out
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return out
|
||||
}
|
||||
for _, r := range rows {
|
||||
after = r.FilePath
|
||||
if err := ctx.Err(); err != nil {
|
||||
out.note = musicStopped(ctx, err)
|
||||
return out
|
||||
}
|
||||
info, err := os.Stat(r.FilePath)
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
// Gone or unreadable: the library scan settles that, and the
|
||||
// file is looked at again on the next run.
|
||||
continue
|
||||
}
|
||||
mtime := info.ModTime().Truncate(time.Microsecond)
|
||||
key := musicPath(root, r.FilePath)
|
||||
location := key
|
||||
if st.Destination == DestinationFolder {
|
||||
location = filepath.Join(st.Folder, filepath.FromSlash(key))
|
||||
}
|
||||
rec := dbq.RecordBackupMusicParams{
|
||||
TrackID: r.ID, Size: info.Size(),
|
||||
Mtime: pgtype.Timestamptz{Time: mtime, Valid: true},
|
||||
Location: location,
|
||||
}
|
||||
if r.SentSize != nil && *r.SentSize == info.Size() && r.SentMtime.Valid && r.SentMtime.Time.Equal(mtime) &&
|
||||
(r.SentError == nil || *r.SentError == "") {
|
||||
// Touched by a scan but the same file: nothing to send.
|
||||
if err := q.RecordBackupMusic(ctx, rec); err != nil {
|
||||
out.note = musicStopped(ctx, err)
|
||||
return out
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if st.Destination == DestinationFolder {
|
||||
err = copyFile(r.FilePath, location, mtime)
|
||||
} else {
|
||||
err = putFile(ctx, bucket, key, r.FilePath, int64(st.UploadLimitKib)*1024)
|
||||
}
|
||||
switch {
|
||||
case err == nil:
|
||||
out.files++
|
||||
out.bytes += info.Size()
|
||||
refused = 0
|
||||
case ObjectRefused(err):
|
||||
rec.Error = strings.TrimPrefix(err.Error(), "backup: ")
|
||||
if refused++; refused >= refusedInARow {
|
||||
out.note = fmt.Sprintf("Music stopped: the destination refused %d files in a row (%s). "+
|
||||
"Check the key may write under %s/; refused files go again once you save the settings.",
|
||||
refused, rec.Error, musicDir)
|
||||
}
|
||||
case ctx.Err() == nil && errors.Is(err, errSourceRead):
|
||||
// The library file couldn't be read through: looked at again next run.
|
||||
continue
|
||||
default:
|
||||
out.note = musicStopped(ctx, err)
|
||||
return out
|
||||
}
|
||||
if err := q.RecordBackupMusic(ctx, rec); err != nil {
|
||||
out.note = musicStopped(ctx, err)
|
||||
return out
|
||||
}
|
||||
if out.note != "" {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// musicStopped is why a music pass ended early, as the panel shows it.
|
||||
func musicStopped(ctx context.Context, err error) string {
|
||||
switch {
|
||||
case errors.Is(ctx.Err(), context.DeadlineExceeded):
|
||||
return fmt.Sprintf("Music stopped for tonight after %d hours; it carries on with the next backup.", int(MusicBudget/time.Hour))
|
||||
case ctx.Err() != nil:
|
||||
return "Music stopped because the server stopped; it carries on with the next backup."
|
||||
}
|
||||
return "Music stopped: " + describe(err) + " It carries on with the next backup."
|
||||
}
|
||||
|
||||
// errSourceRead marks a library file that could not be read.
|
||||
var errSourceRead = errors.New("backup: the library file could not be read")
|
||||
|
||||
// putFile uploads file to key. It reads the file twice, once to hash it
|
||||
// (S3 signs over the body's hash) and once to send it through the limit.
|
||||
func putFile(ctx context.Context, b Bucket, key, file string, perSecond int64) error {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", errSourceRead, err)
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
h := sha256.New()
|
||||
size, err := io.Copy(h, f)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", errSourceRead, err)
|
||||
}
|
||||
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("%w: %v", errSourceRead, err)
|
||||
}
|
||||
return b.Put(ctx, key, Throttle(ctx, f, perSecond), size, hex.EncodeToString(h.Sum(nil)))
|
||||
}
|
||||
|
||||
// copyFile copies file to dst under a hidden .part name first, so the folder
|
||||
// never holds half a file under the file's own name. The copy keeps the
|
||||
// file's modification time. The upload limit is the bucket's: a folder copy
|
||||
// runs at the disk's pace.
|
||||
func copyFile(file, dst string, mtime time.Time) error {
|
||||
in, err := os.Open(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", errSourceRead, err)
|
||||
}
|
||||
defer func() { _ = in.Close() }()
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil {
|
||||
return fmt.Errorf("backup: create %s: %w", filepath.Dir(dst), err)
|
||||
}
|
||||
part := filepath.Join(filepath.Dir(dst), "."+filepath.Base(dst)+".part")
|
||||
out, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: create %s: %w", part, err)
|
||||
}
|
||||
_, err = io.Copy(out, in)
|
||||
if err == nil {
|
||||
err = out.Sync()
|
||||
}
|
||||
if cerr := out.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err == nil {
|
||||
err = os.Chtimes(part, mtime, mtime)
|
||||
}
|
||||
if err == nil {
|
||||
err = os.Rename(part, dst)
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(part)
|
||||
return fmt.Errorf("backup: write %s: %w", dst, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package backup_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||
)
|
||||
|
||||
// library writes music files under a new library folder and adds a track for
|
||||
// each. It returns the folder.
|
||||
func library(t *testing.T, pool *pgxpool.Pool, files map[string]string) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
artist := mustID(t, pool, `INSERT INTO artists (name, sort_name) VALUES ('Artist', 'artist') RETURNING id::text`)
|
||||
album := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id) VALUES ('Album', 'album', $1) RETURNING id::text`, artist)
|
||||
for rel, body := range files {
|
||||
path := filepath.Join(root, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustExec(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format)
|
||||
VALUES ($1, $2, $3, 1000, $4, $5, 'flac')`, rel, album, artist, path, len(body))
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func runOnce(t *testing.T, svc *backup.Service) backup.Run {
|
||||
t.Helper()
|
||||
if err := svc.Start(context.Background(), backup.TriggerManual); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitIdle(t, svc)
|
||||
st := status(t, svc)
|
||||
if len(st.Runs) == 0 || st.Runs[0].Status != "succeeded" {
|
||||
t.Fatalf("the run didn't succeed: %+v", st.Runs)
|
||||
}
|
||||
return st.Runs[0]
|
||||
}
|
||||
|
||||
func TestTheMusicGoesToTheFolderOnceAndAgainOnlyWhenItChanges_Integration(t *testing.T) {
|
||||
svc, pool := newService(t)
|
||||
ctx := context.Background()
|
||||
lib := library(t, pool, map[string]string{"A/one.flac": "one", "B/two.flac": "second"})
|
||||
svc.Roots = func() []string { return []string{lib} }
|
||||
dest := t.TempDir()
|
||||
if err := svc.Save(ctx, backup.Update{Enabled: true, Destination: backup.DestinationFolder, Folder: dest, RunHour: 3, IncludeMusic: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
first := runOnce(t, svc)
|
||||
if first.MusicFiles != 2 || first.MusicBytes != int64(len("one")+len("second")) || first.MusicNote != "" {
|
||||
t.Errorf("the first run sent %d files, %d bytes, note %q", first.MusicFiles, first.MusicBytes, first.MusicNote)
|
||||
}
|
||||
got, err := os.ReadFile(filepath.Join(dest, "music", "B", "two.flac"))
|
||||
if err != nil || string(got) != "second" {
|
||||
t.Errorf("music/B/two.flac = %q, %v", got, err)
|
||||
}
|
||||
src, _ := os.Stat(filepath.Join(lib, "A", "one.flac"))
|
||||
dst, err := os.Stat(filepath.Join(dest, "music", "A", "one.flac"))
|
||||
// The database keeps microseconds, so the copy's time is the file's to the µs.
|
||||
if err != nil || !dst.ModTime().Equal(src.ModTime().Truncate(time.Microsecond)) {
|
||||
t.Errorf("the copy's time = %v, want the file's %v (%v)", dst, src.ModTime(), err)
|
||||
}
|
||||
if p := status(t, svc).Music; p == nil || p.TotalFiles != 2 || p.SentFiles != 2 || p.RefusedFiles != 0 {
|
||||
t.Errorf("progress = %+v, want 2 of 2", p)
|
||||
}
|
||||
|
||||
if again := runOnce(t, svc); again.MusicFiles != 0 {
|
||||
t.Errorf("a second run sent %d files, want none", again.MusicFiles)
|
||||
}
|
||||
|
||||
// A scan that touched the track but found the same file sends nothing.
|
||||
mustExec(t, pool, `UPDATE tracks SET updated_at = now()`)
|
||||
if touched := runOnce(t, svc); touched.MusicFiles != 0 {
|
||||
t.Errorf("after a scan touched the tracks the run sent %d files, want none", touched.MusicFiles)
|
||||
}
|
||||
if p := status(t, svc).Music; p == nil || p.SentFiles != 2 {
|
||||
t.Errorf("progress after the scan = %+v, want 2 sent", p)
|
||||
}
|
||||
|
||||
// A changed file goes again, to the same place.
|
||||
path := filepath.Join(lib, "A", "one.flac")
|
||||
if err := os.WriteFile(path, []byte("one, retagged"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
later := time.Now().Add(time.Minute)
|
||||
if err := os.Chtimes(path, later, later); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustExec(t, pool, `UPDATE tracks SET file_size = $1 WHERE file_path = $2`, len("one, retagged"), path)
|
||||
if changed := runOnce(t, svc); changed.MusicFiles != 1 {
|
||||
t.Errorf("after one file changed the run sent %d files, want 1", changed.MusicFiles)
|
||||
}
|
||||
if got, _ := os.ReadFile(filepath.Join(dest, "music", "A", "one.flac")); string(got) != "one, retagged" {
|
||||
t.Errorf("music/A/one.flac = %q, want the changed file", got)
|
||||
}
|
||||
if entries, _ := os.ReadDir(filepath.Join(dest, "music", "A")); len(entries) != 1 {
|
||||
t.Errorf("music/A holds %d entries, want just the file", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
// lockedBucket is a fakeBucket that refuses to overwrite an object, like a
|
||||
// bucket with object lock. With all set it refuses every music object, like
|
||||
// a key that may write only under database/.
|
||||
type lockedBucket struct {
|
||||
*fakeBucket
|
||||
all *atomic.Bool
|
||||
}
|
||||
|
||||
func (b lockedBucket) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/minstrel/music/") {
|
||||
b.mu.Lock()
|
||||
_, held := b.puts[r.URL.Path]
|
||||
b.mu.Unlock()
|
||||
if held || (b.all != nil && b.all.Load()) {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte(`<Error><Code>AccessDenied</Code><Message>object is locked</Message></Error>`))
|
||||
return
|
||||
}
|
||||
}
|
||||
b.fakeBucket.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func TestTheMusicGoesToTheBucketAndARefusedOverwriteIsKept_Integration(t *testing.T) {
|
||||
svc, pool := newService(t)
|
||||
ctx := context.Background()
|
||||
fake := &fakeBucket{puts: map[string][]byte{}}
|
||||
srv := httptest.NewServer(lockedBucket{fakeBucket: fake})
|
||||
defer srv.Close()
|
||||
lib := library(t, pool, map[string]string{"A/one.flac": "one", "B/two.flac": "second"})
|
||||
svc.Roots = func() []string { return []string{lib} }
|
||||
u := bucketUpdate(srv.URL)
|
||||
u.IncludeMusic = true
|
||||
if err := svc.Save(ctx, u); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if first := runOnce(t, svc); first.MusicFiles != 2 {
|
||||
t.Fatalf("the first run sent %d files, want 2", first.MusicFiles)
|
||||
}
|
||||
fake.mu.Lock()
|
||||
body := string(fake.puts["/minstrel/music/A/one.flac"])
|
||||
fake.mu.Unlock()
|
||||
if body != "one" {
|
||||
t.Errorf("music/A/one.flac in the bucket = %q", body)
|
||||
}
|
||||
|
||||
path := filepath.Join(lib, "A", "one.flac")
|
||||
if err := os.WriteFile(path, []byte("one, retagged"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustExec(t, pool, `UPDATE tracks SET file_size = $1, updated_at = now() WHERE file_path = $2`, len("one, retagged"), path)
|
||||
run := runOnce(t, svc)
|
||||
if run.MusicFiles != 0 || run.MusicNote != "" {
|
||||
t.Errorf("a refused overwrite: sent %d, note %q; want nothing sent and no note", run.MusicFiles, run.MusicNote)
|
||||
}
|
||||
p := status(t, svc).Music
|
||||
if p == nil || p.SentFiles != 1 || p.RefusedFiles != 1 || len(p.Refused) != 1 ||
|
||||
p.Refused[0].Path != path || !strings.Contains(p.Refused[0].Error, "AccessDenied") {
|
||||
t.Errorf("progress = %+v, want one sent and the overwrite refused", p)
|
||||
}
|
||||
|
||||
// Refused is settled until the file changes again.
|
||||
if again := runOnce(t, svc); again.MusicFiles != 0 {
|
||||
t.Errorf("the run after a refusal sent %d files, want none", again.MusicFiles)
|
||||
}
|
||||
if p := status(t, svc).Music; p == nil || p.RefusedFiles != 1 {
|
||||
t.Errorf("progress after the next run = %+v, want the refusal kept", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeyThatMayNotWriteTheMusicStopsThePassAndASaveTriesAgain_Integration(t *testing.T) {
|
||||
svc, pool := newService(t)
|
||||
ctx := context.Background()
|
||||
fake := &fakeBucket{puts: map[string][]byte{}}
|
||||
all := &atomic.Bool{}
|
||||
all.Store(true)
|
||||
srv := httptest.NewServer(lockedBucket{fakeBucket: fake, all: all})
|
||||
defer srv.Close()
|
||||
lib := library(t, pool, map[string]string{"a.flac": "a", "b.flac": "b", "c.flac": "c", "d.flac": "d"})
|
||||
svc.Roots = func() []string { return []string{lib} }
|
||||
u := bucketUpdate(srv.URL)
|
||||
u.IncludeMusic = true
|
||||
if err := svc.Save(ctx, u); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
run := runOnce(t, svc)
|
||||
if run.MusicFiles != 0 || !strings.Contains(run.MusicNote, "refused 3 files in a row") {
|
||||
t.Errorf("sent %d, note %q; want the pass stopped after three refusals", run.MusicFiles, run.MusicNote)
|
||||
}
|
||||
if p := status(t, svc).Music; p == nil || p.RefusedFiles != 3 {
|
||||
t.Errorf("progress = %+v, want three refused and the fourth not tried", p)
|
||||
}
|
||||
|
||||
// The key is fixed and the settings saved: the refused files go again.
|
||||
all.Store(false)
|
||||
if err := svc.Save(ctx, u); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again := runOnce(t, svc); again.MusicFiles != 4 || again.MusicNote != "" {
|
||||
t.Errorf("after the save: sent %d, note %q; want all four", again.MusicFiles, again.MusicNote)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCommonRootIsTheFolderHoldingEveryLibraryRoot(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
roots []string
|
||||
want string
|
||||
}{
|
||||
{nil, ""},
|
||||
{[]string{"/music"}, "/music"},
|
||||
{[]string{"/music/"}, "/music"},
|
||||
{[]string{"/data/music/a", "/data/music/b"}, "/data/music"},
|
||||
{[]string{"/data/music", "/data/musicals"}, "/data"},
|
||||
{[]string{"/music", "/audiobooks"}, "/"},
|
||||
} {
|
||||
if got := commonRoot(c.roots); got != c.want {
|
||||
t.Errorf("commonRoot(%q) = %q, want %q", c.roots, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMusicPathIsTheFileUnderTheCommonRoot(t *testing.T) {
|
||||
for _, c := range []struct{ root, file, want string }{
|
||||
{"/data/music", "/data/music/a/Artist/01 Song.flac", "music/a/Artist/01 Song.flac"},
|
||||
{"/", "/music/x.mp3", "music/music/x.mp3"},
|
||||
{"/data/music", "/elsewhere/x.mp3", "music/elsewhere/x.mp3"},
|
||||
{"", "/music/x.mp3", "music/music/x.mp3"},
|
||||
} {
|
||||
if got := musicPath(c.root, c.file); got != c.want {
|
||||
t.Errorf("musicPath(%q, %q) = %q, want %q", c.root, c.file, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectRefusedIsOneObjectNotTheWholeBucket(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{&Refusal{Status: http.StatusForbidden, Code: "AccessDenied"}, true},
|
||||
{&Refusal{Status: http.StatusConflict, Code: "ObjectLocked"}, true},
|
||||
{fmt.Errorf("put: %w", &Refusal{Status: http.StatusBadRequest, Code: "InvalidObjectName"}), true},
|
||||
{&Refusal{Status: http.StatusForbidden, Code: "SignatureDoesNotMatch"}, false},
|
||||
{&Refusal{Status: http.StatusNotFound, Code: "NoSuchBucket"}, false},
|
||||
{&Refusal{Status: http.StatusTooManyRequests}, false},
|
||||
{&Refusal{Status: http.StatusServiceUnavailable, Code: "SlowDown"}, false},
|
||||
{&Refusal{Status: http.StatusInternalServerError}, false},
|
||||
{errors.New("connection refused"), false},
|
||||
} {
|
||||
if got := ObjectRefused(c.err); got != c.want {
|
||||
t.Errorf("ObjectRefused(%v %+v) = %v, want %v", c.err, c.err, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
+21
-16
@@ -13,9 +13,8 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNotEmpty is a restore into an instance that already has accounts or
|
||||
// a library.
|
||||
ErrNotEmpty = errors.New("backup: this Minstrel already has accounts or a library; a backup restores into an empty one")
|
||||
// ErrNotEmpty is a restore into an instance that already has accounts.
|
||||
ErrNotEmpty = errors.New("backup: this Minstrel already has accounts; a backup restores into a new one")
|
||||
|
||||
// ErrNewerSchema is a backup written by a newer Minstrel than this one.
|
||||
ErrNewerSchema = errors.New("backup: the backup was written by a newer version of Minstrel; upgrade this one first")
|
||||
@@ -30,10 +29,12 @@ const batchRows = 500
|
||||
// Restore loads a backup into an empty instance, in one transaction: either
|
||||
// all of it lands or none of it does.
|
||||
//
|
||||
// The instance must have no accounts and no tracks. Its other rows in the
|
||||
// tables the backup holds are replaced, which is what a fresh instance's
|
||||
// seeded settings need. A table the backup doesn't hold, because it was
|
||||
// written before that table existed, keeps what the instance has.
|
||||
// The instance must have no accounts. Its rows in the tables the backup holds
|
||||
// are replaced: a fresh instance's seeded settings, and the library its
|
||||
// startup scan found before anyone signed up, which the scan after the
|
||||
// restore finds again under the backup's ids. A table the backup doesn't
|
||||
// hold, because it was written before that table existed, keeps what the
|
||||
// instance has.
|
||||
//
|
||||
// A row's columns are those the backup and the table both have: an older
|
||||
// backup's rows take the defaults of columns added since, and a column dropped
|
||||
@@ -52,15 +53,15 @@ func Restore(ctx context.Context, pool *pgxpool.Pool, c Contents) error {
|
||||
if c.Schema > schema {
|
||||
return ErrNewerSchema
|
||||
}
|
||||
// Taken before the emptiness check, so a first account or a scan starting
|
||||
// at the same moment waits for the restore, then finds the data.
|
||||
if _, err := tx.Exec(ctx, "LOCK TABLE users, tracks IN EXCLUSIVE MODE"); err != nil {
|
||||
// Taken before the emptiness check, so a first account made at the same
|
||||
// moment waits for the restore, then finds the data. A scan still running
|
||||
// waits too, rather than adding to the library while it is replaced.
|
||||
if _, err := tx.Exec(ctx, "LOCK TABLE users, artists, albums, tracks IN EXCLUSIVE MODE"); err != nil {
|
||||
return fmt.Errorf("backup: lock: %w", err)
|
||||
}
|
||||
var occupied bool
|
||||
if err := tx.QueryRow(ctx,
|
||||
"SELECT EXISTS (SELECT 1 FROM users) OR EXISTS (SELECT 1 FROM tracks)").Scan(&occupied); err != nil {
|
||||
return fmt.Errorf("backup: check the instance is empty: %w", err)
|
||||
if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM users)").Scan(&occupied); err != nil {
|
||||
return fmt.Errorf("backup: check the instance is new: %w", err)
|
||||
}
|
||||
if occupied {
|
||||
return ErrNotEmpty
|
||||
@@ -158,14 +159,18 @@ func restoreTable(ctx context.Context, tx pgx.Tx, table string, rows [][]byte) e
|
||||
}
|
||||
|
||||
// A sequence carries on after the highest restored value, so the next row
|
||||
// doesn't collide with one from the backup.
|
||||
// doesn't collide with one from the backup. It is left as having issued
|
||||
// that value, so the sequence itself says how far the restored history
|
||||
// goes: the library sync compares an app's cursor with it.
|
||||
for _, c := range cols {
|
||||
if !c.sequence {
|
||||
continue
|
||||
}
|
||||
col := pgx.Identifier{c.name}.Sanitize()
|
||||
if _, err := tx.Exec(ctx, fmt.Sprintf(
|
||||
"SELECT setval(pg_get_serial_sequence($1, $2), COALESCE((SELECT max(%s) FROM %s), 0) + 1, false)", col, ident),
|
||||
if _, err := tx.Exec(ctx, fmt.Sprintf(`
|
||||
SELECT CASE WHEN m > 0 THEN setval(pg_get_serial_sequence($1, $2), m, true)
|
||||
ELSE setval(pg_get_serial_sequence($1, $2), 1, false) END
|
||||
FROM (SELECT COALESCE(max(%s), 0) AS m FROM %s) AS top`, col, ident),
|
||||
table, c.name); err != nil {
|
||||
return fmt.Errorf("backup: carry on %s's sequence: %w", table, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrS3 is a request the bucket refused or never answered.
|
||||
var ErrS3 = errors.New("backup: the S3 bucket refused the request")
|
||||
|
||||
// Bucket is a path-style S3 bucket: an endpoint such as Fabled Archives'
|
||||
// backup port or MinIO, a bucket name, and a key pair. A backup only ever
|
||||
// adds objects and lists them, so an append-only key is enough.
|
||||
type Bucket struct {
|
||||
Endpoint string
|
||||
Name string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
// Region defaults to us-east-1, which S3-compatible servers accept.
|
||||
Region string
|
||||
Client *http.Client
|
||||
// now is the signing clock; tests pin it.
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// Object is one stored object.
|
||||
type Object struct {
|
||||
Key string
|
||||
Size int64
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
func (b Bucket) region() string {
|
||||
if b.Region == "" {
|
||||
return "us-east-1"
|
||||
}
|
||||
return b.Region
|
||||
}
|
||||
|
||||
func (b Bucket) client() *http.Client {
|
||||
if b.Client != nil {
|
||||
return b.Client
|
||||
}
|
||||
return http.DefaultClient
|
||||
}
|
||||
|
||||
func (b Bucket) clock() time.Time {
|
||||
if b.now != nil {
|
||||
return b.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// objectURL is the path-style URL of key, encoded exactly as it is signed.
|
||||
func (b Bucket) objectURL(key string) (*url.URL, error) {
|
||||
u, err := url.Parse(strings.TrimRight(b.Endpoint, "/"))
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return nil, fmt.Errorf("%w: the endpoint %q is not an http(s) address", ErrS3, b.Endpoint)
|
||||
}
|
||||
if b.Name == "" {
|
||||
return nil, fmt.Errorf("%w: no bucket name", ErrS3)
|
||||
}
|
||||
raw := "/" + uriEncode(b.Name)
|
||||
if key != "" {
|
||||
for s := range strings.SplitSeq(key, "/") {
|
||||
raw += "/" + uriEncode(s)
|
||||
}
|
||||
}
|
||||
u.RawPath = u.EscapedPath() + raw
|
||||
u.Path, _ = url.PathUnescape(u.RawPath)
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (b Bucket) do(req *http.Request, payloadSHA256 string) (*http.Response, error) {
|
||||
sign(req, payloadSHA256, b.AccessKey, b.SecretKey, b.region(), b.clock())
|
||||
resp, err := b.client().Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s is unreachable: %v", ErrS3, req.URL.Host, err)
|
||||
}
|
||||
if resp.StatusCode/100 != 2 {
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
return nil, refusal(resp)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Put stores body under key. size is the body's length and payloadSHA256
|
||||
// its hex SHA-256, which the request is signed over.
|
||||
func (b Bucket) Put(ctx context.Context, key string, body io.Reader, size int64, payloadSHA256 string) error {
|
||||
u, err := b.objectURL(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.URL = u
|
||||
req.ContentLength = size
|
||||
if size == 0 {
|
||||
req.Body = http.NoBody
|
||||
}
|
||||
resp, err := b.do(req, payloadSHA256)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
drain(resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns up to limit objects whose keys start with prefix, in key
|
||||
// order.
|
||||
func (b Bucket) List(ctx context.Context, prefix string, limit int) ([]Object, error) {
|
||||
if limit < 1 {
|
||||
limit = 1000
|
||||
}
|
||||
var out []Object
|
||||
token := ""
|
||||
for {
|
||||
u, err := b.objectURL("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := url.Values{"list-type": {"2"}, "prefix": {prefix}}
|
||||
if page := limit - len(out); page < 1000 {
|
||||
q.Set("max-keys", strconv.Itoa(page))
|
||||
}
|
||||
if token != "" {
|
||||
q.Set("continuation-token", token)
|
||||
}
|
||||
u.RawQuery = canonicalQuery(q)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.URL = u
|
||||
resp, err := b.do(req, emptySHA256)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var page struct {
|
||||
Contents []struct {
|
||||
Key string `xml:"Key"`
|
||||
Size int64 `xml:"Size"`
|
||||
LastModified string `xml:"LastModified"`
|
||||
} `xml:"Contents"`
|
||||
IsTruncated bool `xml:"IsTruncated"`
|
||||
NextContinuationToken string `xml:"NextContinuationToken"`
|
||||
}
|
||||
err = xml.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&page)
|
||||
drain(resp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: the listing is not S3 XML: %v", ErrS3, err)
|
||||
}
|
||||
for _, c := range page.Contents {
|
||||
modified, _ := time.Parse(time.RFC3339, c.LastModified)
|
||||
out = append(out, Object{Key: c.Key, Size: c.Size, LastModified: modified})
|
||||
}
|
||||
if !page.IsTruncated || page.NextContinuationToken == "" || len(out) >= limit {
|
||||
return out, nil
|
||||
}
|
||||
token = page.NextContinuationToken
|
||||
}
|
||||
}
|
||||
|
||||
// Refusal is an S3 error response: its HTTP status, S3's code, and the
|
||||
// reason in words. It matches ErrS3.
|
||||
type Refusal struct {
|
||||
Status int
|
||||
Code string
|
||||
text string
|
||||
}
|
||||
|
||||
func (r *Refusal) Error() string { return r.text }
|
||||
func (r *Refusal) Unwrap() error { return ErrS3 }
|
||||
|
||||
// refusal turns an S3 error response into words an admin can act on.
|
||||
func refusal(resp *http.Response) error {
|
||||
var e struct {
|
||||
Code string `xml:"Code"`
|
||||
Message string `xml:"Message"`
|
||||
}
|
||||
_ = xml.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&e)
|
||||
hint := map[string]string{
|
||||
"SignatureDoesNotMatch": "the secret key is wrong",
|
||||
"InvalidAccessKeyId": "the access key is not known to this server",
|
||||
"AccessDenied": "the key may not do this in this bucket",
|
||||
"NoSuchBucket": "there is no bucket by that name",
|
||||
"SlowDown": "the server asks to slow down",
|
||||
}[e.Code]
|
||||
r := &Refusal{Status: resp.StatusCode, Code: e.Code}
|
||||
switch {
|
||||
case hint != "":
|
||||
r.text = fmt.Sprintf("%v: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, hint)
|
||||
case e.Code != "":
|
||||
r.text = fmt.Sprintf("%v: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, e.Message)
|
||||
default:
|
||||
r.text = fmt.Sprintf("%v: HTTP %d", ErrS3, resp.StatusCode)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// ObjectRefused reports whether err is the bucket refusing one object (an
|
||||
// overwrite a write-once target won't take, a name it won't store) rather
|
||||
// than refusing everything: bad keys, a missing bucket, a request to slow
|
||||
// down, or a server error all stop a run instead.
|
||||
func ObjectRefused(err error) bool {
|
||||
var r *Refusal
|
||||
if !errors.As(err, &r) || r.Status/100 != 4 || r.Status == http.StatusTooManyRequests {
|
||||
return false
|
||||
}
|
||||
switch r.Code {
|
||||
case "SignatureDoesNotMatch", "InvalidAccessKeyId", "NoSuchBucket", "SlowDown",
|
||||
"RequestTimeTooSkewed", "ExpiredToken", "InvalidToken":
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func drain(resp *http.Response) {
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20))
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AWS's published examples ("Examples: Signature Calculations in AWS
|
||||
// Signature Version 4", Amazon S3 API reference).
|
||||
const (
|
||||
awsAccessKey = "AKIAIOSFODNN7EXAMPLE"
|
||||
awsSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
awsCred = "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"
|
||||
)
|
||||
|
||||
var awsTime = time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
func TestSignMatchesAWSsGetObjectExample(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/test.txt", nil)
|
||||
req.Header.Set("Range", "bytes=0-9")
|
||||
sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
|
||||
want := awsCred + ", SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
|
||||
if got := req.Header.Get("Authorization"); got != want {
|
||||
t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignMatchesAWSsPutObjectExample(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodPut, "http://examplebucket.s3.amazonaws.com/test$file.text", strings.NewReader("Welcome to Amazon S3."))
|
||||
req.Header.Set("Date", "Fri, 24 May 2013 00:00:00 GMT")
|
||||
req.Header.Set("X-Amz-Storage-Class", "REDUCED_REDUNDANCY")
|
||||
payload := hexSHA256([]byte("Welcome to Amazon S3."))
|
||||
if payload != "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072" {
|
||||
t.Fatalf("payload hash = %s", payload)
|
||||
}
|
||||
sign(req, payload, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
|
||||
want := awsCred + ", SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd"
|
||||
if got := req.Header.Get("Authorization"); got != want {
|
||||
t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignMatchesAWSsListObjectsExample(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
||||
sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
|
||||
want := awsCred + ", SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
|
||||
if got := req.Header.Get("Authorization"); got != want {
|
||||
t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUriEncodeKeepsOnlyUnreservedCharacters(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"01 - Song.flac": "01%20-%20Song.flac",
|
||||
"a+b=c&d": "a%2Bb%3Dc%26d",
|
||||
"Björk~_.": "Bj%C3%B6rk~_.",
|
||||
"a/b": "a%2Fb",
|
||||
} {
|
||||
if got := uriEncode(in); got != want {
|
||||
t.Errorf("uriEncode(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testBucket(url string) Bucket {
|
||||
return Bucket{Endpoint: url, Name: "minstrel", AccessKey: "ak", SecretKey: "sk",
|
||||
now: func() time.Time { return awsTime }}
|
||||
}
|
||||
|
||||
func TestPutSendsTheBodyToItsPathStyleKey(t *testing.T) {
|
||||
var gotPath, gotSHA, gotBody, gotAuth string
|
||||
var gotLen int64
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
|
||||
gotPath, gotSHA, gotAuth, gotLen = r.URL.EscapedPath(), r.Header.Get("X-Amz-Content-Sha256"), r.Header.Get("Authorization"), r.ContentLength
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
gotBody = string(b)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
body := "flac bytes"
|
||||
sum := hexSHA256([]byte(body))
|
||||
if err := testBucket(srv.URL).Put(context.Background(), "music/A B/01 + x.flac", strings.NewReader(body), int64(len(body)), sum); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gotPath != "/minstrel/music/A%20B/01%20%2B%20x.flac" {
|
||||
t.Errorf("path = %s", gotPath)
|
||||
}
|
||||
if gotSHA != sum || gotBody != body || gotLen != int64(len(body)) {
|
||||
t.Errorf("sha %s body %q length %d", gotSHA, gotBody, gotLen)
|
||||
}
|
||||
if !strings.HasPrefix(gotAuth, "AWS4-HMAC-SHA256 Credential=ak/20130524/us-east-1/s3/aws4_request") {
|
||||
t.Errorf("Authorization = %s", gotAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFollowsContinuationTokens(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
if r.URL.Path != "/minstrel" || q.Get("list-type") != "2" || q.Get("prefix") != "database/" {
|
||||
http.Error(w, "unexpected "+r.URL.String(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if q.Get("continuation-token") == "" {
|
||||
_, _ = fmt.Fprint(w, `<ListBucketResult><IsTruncated>true</IsTruncated><NextContinuationToken>next</NextContinuationToken>
|
||||
<Contents><Key>database/minstrel-20261101T030000Z.tar</Key><Size>10</Size><LastModified>2026-11-01T03:00:05.000Z</LastModified></Contents>
|
||||
<Contents><Key>database/minstrel-20261102T030000Z.tar</Key><Size>11</Size><LastModified>2026-11-02T03:00:05.000Z</LastModified></Contents>
|
||||
</ListBucketResult>`)
|
||||
return
|
||||
}
|
||||
_, _ = fmt.Fprint(w, `<ListBucketResult><IsTruncated>false</IsTruncated>
|
||||
<Contents><Key>database/minstrel-20261103T030000Z.tar</Key><Size>12</Size><LastModified>2026-11-03T03:00:05.000Z</LastModified></Contents>
|
||||
</ListBucketResult>`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
objs, err := testBucket(srv.URL).List(context.Background(), "database/", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(objs) != 3 || objs[2].Key != "database/minstrel-20261103T030000Z.tar" || objs[1].Size != 11 ||
|
||||
!objs[0].LastModified.Equal(time.Date(2026, 11, 1, 3, 0, 5, 0, time.UTC)) {
|
||||
t.Errorf("objects = %+v", objs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARefusalSaysWhatIsWrong(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = fmt.Fprint(w, `<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match</Message></Error>`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
_, err := testBucket(srv.URL).List(context.Background(), "", 1)
|
||||
if !errors.Is(err, ErrS3) || !strings.Contains(err.Error(), "HTTP 403 SignatureDoesNotMatch: the secret key is wrong") {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreachableOrMalformedEndpointIsAnS3Error(t *testing.T) {
|
||||
srv := httptest.NewServer(http.NotFoundHandler())
|
||||
srv.Close()
|
||||
for name, endpoint := range map[string]string{"closed": srv.URL, "not http": "ftp://archives", "empty": ""} {
|
||||
if _, err := testBucket(endpoint).List(context.Background(), "", 1); !errors.Is(err, ErrS3) {
|
||||
t.Errorf("%s: err = %v, want ErrS3", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,755 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// Where a backup can go.
|
||||
const (
|
||||
DestinationFolder = "folder"
|
||||
DestinationS3 = "s3"
|
||||
)
|
||||
|
||||
// What started a run.
|
||||
const (
|
||||
TriggerSchedule = "schedule"
|
||||
TriggerManual = "manual"
|
||||
)
|
||||
|
||||
// The verdict a Status gives, worst first in the order an admin acts on them.
|
||||
const (
|
||||
StateOff = "off" // the nightly backup is switched off
|
||||
StateIncomplete = "incomplete" // switched on, but the destination isn't set up
|
||||
StateFailing = "failing" // the latest run failed
|
||||
StateWaiting = "waiting" // set up, and no backup has finished yet
|
||||
StateOverdue = "overdue" // the latest good backup is more than a day old
|
||||
StateOK = "ok"
|
||||
)
|
||||
|
||||
const (
|
||||
// RunLimit bounds one run, snapshot and upload together. The web panel
|
||||
// stops polling a run a little after this, since the run is over by then.
|
||||
RunLimit = 30 * time.Minute
|
||||
// A nightly backup that hasn't succeeded for this long is overdue; the
|
||||
// slack past a day covers a run that starts late or takes a while.
|
||||
overdueAfter = 26 * time.Hour
|
||||
tickEvery = 5 * time.Minute
|
||||
// A failed run is tried again after at least this long, and after longer
|
||||
// each time it fails again, so a destination that is down isn't hammered.
|
||||
retryAfter = time.Hour
|
||||
recentRuns = 10
|
||||
// Run history older than this is pruned after each run.
|
||||
keepRunsFor = 90 * 24 * time.Hour
|
||||
testTimeout = 30 * time.Second
|
||||
// The database backups' folder inside the destination. The music, when
|
||||
// it is backed up, sits beside it in the same destination.
|
||||
databaseDir = "database"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrRunning is returned by Start while a backup is running.
|
||||
ErrRunning = errors.New("backup: a backup is already running")
|
||||
// ErrNotReady is returned by Start when the destination isn't set up.
|
||||
ErrNotReady = errors.New("backup: the destination isn't set up yet")
|
||||
// ErrClosed is returned by Start once the server is shutting down.
|
||||
ErrClosed = errors.New("backup: the server is shutting down")
|
||||
)
|
||||
|
||||
// FieldError is a setting that can't be saved, with the reason in words.
|
||||
type FieldError struct {
|
||||
Field string `json:"field"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (e *FieldError) Error() string { return e.Field + ": " + e.Message }
|
||||
|
||||
// Config is the backup's settings as an admin sees them. The bucket's secret
|
||||
// key is never sent back, only whether one is stored.
|
||||
type Config struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Destination string `json:"destination"`
|
||||
Folder string `json:"folder"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKey string `json:"access_key"`
|
||||
HasSecretKey bool `json:"has_secret_key"`
|
||||
RunHour int `json:"run_hour"`
|
||||
UploadLimitKiB int `json:"upload_limit_kib"`
|
||||
IncludeMusic bool `json:"include_music"`
|
||||
}
|
||||
|
||||
// Update is a save from the admin card. An empty SecretKey keeps the stored
|
||||
// one, so the key doesn't have to be typed again on every save.
|
||||
type Update struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Destination string `json:"destination"`
|
||||
Folder string `json:"folder"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKey string `json:"access_key"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
RunHour int `json:"run_hour"`
|
||||
UploadLimitKiB int `json:"upload_limit_kib"`
|
||||
IncludeMusic bool `json:"include_music"`
|
||||
}
|
||||
|
||||
// Run is one attempt at a backup.
|
||||
type Run struct {
|
||||
ID string `json:"id"`
|
||||
Trigger string `json:"trigger"`
|
||||
Status string `json:"status"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt *time.Time `json:"finished_at"`
|
||||
Location string `json:"location"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Error string `json:"error"`
|
||||
// What the run sent of the music, and why that stopped early, if it did.
|
||||
MusicFiles int `json:"music_files"`
|
||||
MusicBytes int64 `json:"music_bytes"`
|
||||
MusicNote string `json:"music_note"`
|
||||
}
|
||||
|
||||
// Status is everything the admin panel shows: a verdict, what stands in its
|
||||
// way, and the recent runs.
|
||||
type Status struct {
|
||||
Config Config `json:"config"`
|
||||
State string `json:"state"`
|
||||
Problems []string `json:"problems"`
|
||||
RunningSince *time.Time `json:"running_since"`
|
||||
LastSuccess *time.Time `json:"last_success"`
|
||||
FailingSince *time.Time `json:"failing_since"`
|
||||
// NextRun is when the schedule starts the next backup, or nil when it
|
||||
// won't: switched off, or not set up.
|
||||
NextRun *time.Time `json:"next_run"`
|
||||
RunLimitSeconds int `json:"run_limit_seconds"`
|
||||
Runs []Run `json:"runs"`
|
||||
// Music is how much of the library the backup holds, when it includes
|
||||
// the music.
|
||||
Music *MusicProgress `json:"music"`
|
||||
}
|
||||
|
||||
// TestResult is what a destination test found: how many backups are there
|
||||
// already and when the newest was made.
|
||||
type TestResult struct {
|
||||
Backups int `json:"backups"`
|
||||
Latest *time.Time `json:"latest"`
|
||||
}
|
||||
|
||||
// Service runs Minstrel's own backup: nightly when switched on, or when an
|
||||
// admin asks. One run at a time; each is recorded in backup_runs.
|
||||
type Service struct {
|
||||
pool *pgxpool.Pool
|
||||
logger *slog.Logger
|
||||
build string
|
||||
now func() time.Time
|
||||
// Roots lists the library's folders. The music's paths in the backup are
|
||||
// relative to the folder holding them all.
|
||||
Roots func() []string
|
||||
|
||||
base context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
|
||||
mu sync.Mutex
|
||||
runningSince time.Time
|
||||
closed bool
|
||||
}
|
||||
|
||||
// NewService returns a Service over pool. build is the server's version,
|
||||
// written into each backup's manifest.
|
||||
func NewService(pool *pgxpool.Pool, logger *slog.Logger, build string) *Service {
|
||||
base, cancel := context.WithCancel(context.Background())
|
||||
return &Service{pool: pool, logger: logger, build: build, now: time.Now, base: base, cancel: cancel}
|
||||
}
|
||||
|
||||
func (s *Service) settings(ctx context.Context) (dbq.BackupSetting, error) {
|
||||
st, err := dbq.New(s.pool).GetBackupSettings(ctx)
|
||||
if err != nil {
|
||||
return st, fmt.Errorf("backup: read settings: %w", err)
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
func view(st dbq.BackupSetting) Config {
|
||||
return Config{
|
||||
Enabled: st.Enabled,
|
||||
Destination: st.Destination,
|
||||
Folder: st.Folder,
|
||||
Endpoint: st.Endpoint,
|
||||
Bucket: st.Bucket,
|
||||
AccessKey: st.AccessKey,
|
||||
HasSecretKey: st.SecretKey != "",
|
||||
RunHour: int(st.RunHour),
|
||||
UploadLimitKiB: int(st.UploadLimitKib),
|
||||
IncludeMusic: st.IncludeMusic,
|
||||
}
|
||||
}
|
||||
|
||||
// Config returns the settings as an admin sees them.
|
||||
func (s *Service) Config(ctx context.Context) (Config, error) {
|
||||
st, err := s.settings(ctx)
|
||||
return view(st), err
|
||||
}
|
||||
|
||||
// merge applies u over the stored settings: it checks each field, and seals
|
||||
// a new secret key. It returns a *FieldError for a value that can't be kept.
|
||||
func merge(st dbq.BackupSetting, u Update) (dbq.BackupSetting, error) {
|
||||
if u.Destination != DestinationFolder && u.Destination != DestinationS3 {
|
||||
return st, &FieldError{"destination", "Choose a folder or an S3 bucket."}
|
||||
}
|
||||
if u.RunHour < 0 || u.RunHour > 23 {
|
||||
return st, &FieldError{"run_hour", "The hour is from 0 to 23."}
|
||||
}
|
||||
if u.UploadLimitKiB < 0 || u.UploadLimitKiB > 1<<30 {
|
||||
return st, &FieldError{"upload_limit_kib", "The upload limit is 0 (no limit) or more."}
|
||||
}
|
||||
folder := strings.TrimSpace(u.Folder)
|
||||
if folder != "" && !filepath.IsAbs(folder) {
|
||||
return st, &FieldError{"folder", "Give the folder's full path, starting with /."}
|
||||
}
|
||||
endpoint := strings.TrimRight(strings.TrimSpace(u.Endpoint), "/")
|
||||
if endpoint != "" {
|
||||
e, err := url.Parse(endpoint)
|
||||
if err != nil || (e.Scheme != "http" && e.Scheme != "https") || e.Host == "" {
|
||||
return st, &FieldError{"endpoint", "Give the endpoint as a URL, such as https://s3.example.com."}
|
||||
}
|
||||
}
|
||||
st.Enabled = u.Enabled
|
||||
st.Destination = u.Destination
|
||||
st.Folder = filepath.Clean(folder)
|
||||
if folder == "" {
|
||||
st.Folder = ""
|
||||
}
|
||||
st.Endpoint = endpoint
|
||||
st.Bucket = strings.TrimSpace(u.Bucket)
|
||||
st.AccessKey = strings.TrimSpace(u.AccessKey)
|
||||
st.RunHour = int16(u.RunHour)
|
||||
st.UploadLimitKib = int32(u.UploadLimitKiB)
|
||||
st.IncludeMusic = u.IncludeMusic
|
||||
if u.SecretKey != "" {
|
||||
sealed, err := secret.Seal(secret.BackupSecretKey.Purpose(), "", u.SecretKey)
|
||||
if err != nil {
|
||||
return st, err
|
||||
}
|
||||
st.SecretKey = sealed
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// Save stores u. A value that can't be kept is a *FieldError and nothing is
|
||||
// stored. Settings that are incomplete still save; Status lists what's missing.
|
||||
func (s *Service) Save(ctx context.Context, u Update) error {
|
||||
st, err := s.settings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
st, err = merge(st, u)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return dbq.New(s.pool).SaveBackupSettings(ctx, dbq.SaveBackupSettingsParams{
|
||||
Enabled: st.Enabled,
|
||||
Destination: st.Destination,
|
||||
Folder: st.Folder,
|
||||
Endpoint: st.Endpoint,
|
||||
Bucket: st.Bucket,
|
||||
AccessKey: st.AccessKey,
|
||||
SecretKey: st.SecretKey,
|
||||
RunHour: st.RunHour,
|
||||
UploadLimitKib: st.UploadLimitKib,
|
||||
IncludeMusic: st.IncludeMusic,
|
||||
})
|
||||
}
|
||||
|
||||
// problems is what stops a backup from running with these settings, in words.
|
||||
func problems(st dbq.BackupSetting) []string {
|
||||
var out []string
|
||||
switch st.Destination {
|
||||
case DestinationFolder:
|
||||
if st.Folder == "" {
|
||||
out = append(out, "Choose the folder backups are written to.")
|
||||
}
|
||||
case DestinationS3:
|
||||
if st.Endpoint == "" {
|
||||
out = append(out, "Enter the bucket's endpoint.")
|
||||
}
|
||||
if st.Bucket == "" {
|
||||
out = append(out, "Enter the bucket's name.")
|
||||
}
|
||||
if st.AccessKey == "" {
|
||||
out = append(out, "Enter the access key.")
|
||||
}
|
||||
if st.SecretKey == "" {
|
||||
out = append(out, "Enter the secret key.")
|
||||
}
|
||||
default:
|
||||
out = append(out, "Choose where backups go.")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func timeOf(t pgtype.Timestamptz) time.Time {
|
||||
if !t.Valid || t.Time.Unix() <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return t.Time
|
||||
}
|
||||
|
||||
func ptr(t time.Time) *time.Time {
|
||||
if t.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return &t
|
||||
}
|
||||
|
||||
// runTimes is what the verdict and the schedule turn on. A zero time is a run
|
||||
// that never happened.
|
||||
type runTimes struct {
|
||||
lastSuccess, lastAttempt, failingSince time.Time
|
||||
}
|
||||
|
||||
func (s *Service) times(ctx context.Context) (runTimes, error) {
|
||||
row, err := dbq.New(s.pool).BackupRunTimes(ctx)
|
||||
if err != nil {
|
||||
return runTimes{}, fmt.Errorf("backup: read run times: %w", err)
|
||||
}
|
||||
return runTimes{timeOf(row.LastSuccess), timeOf(row.LastAttempt), timeOf(row.FailingSince)}, nil
|
||||
}
|
||||
|
||||
// slotBefore is the latest scheduled start at or before now: today at hour
|
||||
// UTC, or yesterday's if that hour hasn't come yet.
|
||||
func slotBefore(now time.Time, hour int) time.Time {
|
||||
now = now.UTC()
|
||||
slot := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, time.UTC)
|
||||
if slot.After(now) {
|
||||
slot = slot.AddDate(0, 0, -1)
|
||||
}
|
||||
return slot
|
||||
}
|
||||
|
||||
// due reports when the schedule next starts a backup: now or earlier means
|
||||
// the next tick starts one. A slot that passed with no attempt since is due,
|
||||
// so a server that was down at the hour catches up. A run that failed is
|
||||
// tried again, waiting at least retryAfter and then as long again as it has
|
||||
// been failing, until it succeeds or the next slot comes.
|
||||
func due(now time.Time, hour int, t runTimes) time.Time {
|
||||
slot := slotBefore(now, hour)
|
||||
if t.lastAttempt.Before(slot) {
|
||||
return slot
|
||||
}
|
||||
next := slot.AddDate(0, 0, 1)
|
||||
if !t.failingSince.IsZero() {
|
||||
wait := max(retryAfter, t.lastAttempt.Sub(t.failingSince))
|
||||
if retry := t.lastAttempt.Add(wait); retry.Before(next) {
|
||||
return retry
|
||||
}
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
func verdict(st dbq.BackupSetting, probs []string, t runTimes, now time.Time) string {
|
||||
switch {
|
||||
case !st.Enabled:
|
||||
return StateOff
|
||||
case len(probs) > 0:
|
||||
return StateIncomplete
|
||||
case !t.failingSince.IsZero():
|
||||
return StateFailing
|
||||
case t.lastSuccess.IsZero():
|
||||
return StateWaiting
|
||||
case now.Sub(t.lastSuccess) > overdueAfter:
|
||||
return StateOverdue
|
||||
}
|
||||
return StateOK
|
||||
}
|
||||
|
||||
func uuidText(u pgtype.UUID) string {
|
||||
b := u.Bytes
|
||||
return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
|
||||
}
|
||||
|
||||
// Status is the verdict, what stands in its way, and the recent runs.
|
||||
func (s *Service) Status(ctx context.Context) (Status, error) {
|
||||
st, err := s.settings(ctx)
|
||||
if err != nil {
|
||||
return Status{}, err
|
||||
}
|
||||
t, err := s.times(ctx)
|
||||
if err != nil {
|
||||
return Status{}, err
|
||||
}
|
||||
rows, err := dbq.New(s.pool).ListBackupRuns(ctx, recentRuns)
|
||||
if err != nil {
|
||||
return Status{}, fmt.Errorf("backup: list runs: %w", err)
|
||||
}
|
||||
now := s.now()
|
||||
probs := problems(st)
|
||||
out := Status{
|
||||
Config: view(st),
|
||||
State: verdict(st, probs, t, now),
|
||||
Problems: probs,
|
||||
LastSuccess: ptr(t.lastSuccess),
|
||||
FailingSince: ptr(t.failingSince),
|
||||
RunLimitSeconds: int(runLimit(st) / time.Second),
|
||||
Runs: make([]Run, 0, len(rows)),
|
||||
}
|
||||
if out.Problems == nil {
|
||||
out.Problems = []string{}
|
||||
}
|
||||
if st.IncludeMusic {
|
||||
if out.Music, err = s.musicProgress(ctx); err != nil {
|
||||
return Status{}, err
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
out.RunningSince = ptr(s.runningSince)
|
||||
s.mu.Unlock()
|
||||
if st.Enabled && len(probs) == 0 {
|
||||
next := due(now, int(st.RunHour), t)
|
||||
if next.Before(now) {
|
||||
// Due now: the next tick starts it.
|
||||
next = now
|
||||
}
|
||||
out.NextRun = &next
|
||||
}
|
||||
for _, r := range rows {
|
||||
out.Runs = append(out.Runs, Run{
|
||||
ID: uuidText(r.ID),
|
||||
Trigger: r.Trigger,
|
||||
Status: r.Status,
|
||||
StartedAt: r.StartedAt.Time,
|
||||
FinishedAt: ptr(timeOf(r.FinishedAt)),
|
||||
Location: r.Location,
|
||||
Bytes: r.Bytes,
|
||||
Error: r.Error,
|
||||
MusicFiles: int(r.MusicFiles),
|
||||
MusicBytes: r.MusicBytes,
|
||||
MusicNote: r.MusicNote,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func bucketOf(st dbq.BackupSetting) (Bucket, error) {
|
||||
key, err := secret.Open(secret.BackupSecretKey.Purpose(), "", st.SecretKey)
|
||||
if err != nil {
|
||||
return Bucket{}, fmt.Errorf("backup: the stored secret key doesn't open; enter it again: %w", err)
|
||||
}
|
||||
return Bucket{Endpoint: st.Endpoint, Name: st.Bucket, AccessKey: st.AccessKey, SecretKey: key}, nil
|
||||
}
|
||||
|
||||
// Test checks the destination u describes, falling back to the stored
|
||||
// secret key when u leaves it empty. It only reads: it lists the backups
|
||||
// already there and writes nothing.
|
||||
func (s *Service) Test(ctx context.Context, u Update) (TestResult, error) {
|
||||
st, err := s.settings(ctx)
|
||||
if err != nil {
|
||||
return TestResult{}, err
|
||||
}
|
||||
st, err = merge(st, u)
|
||||
if err != nil {
|
||||
return TestResult{}, err
|
||||
}
|
||||
if probs := problems(st); len(probs) > 0 {
|
||||
return TestResult{}, fmt.Errorf("%w: %s", ErrNotReady, strings.Join(probs, " "))
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, testTimeout)
|
||||
defer cancel()
|
||||
|
||||
var res TestResult
|
||||
note := func(name string, at time.Time) {
|
||||
if !IsFileName(name) {
|
||||
return
|
||||
}
|
||||
res.Backups++
|
||||
if res.Latest == nil || at.After(*res.Latest) {
|
||||
res.Latest = &at
|
||||
}
|
||||
}
|
||||
if st.Destination == DestinationFolder {
|
||||
if err := isFolder(st.Folder); err != nil {
|
||||
return res, err
|
||||
}
|
||||
entries, err := os.ReadDir(filepath.Join(st.Folder, databaseDir))
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return res, fmt.Errorf("backup: read %s: %w", filepath.Join(st.Folder, databaseDir), err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if info, err := e.Info(); err == nil && info.Mode().IsRegular() {
|
||||
note(e.Name(), info.ModTime())
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
b, err := bucketOf(st)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
objs, err := b.List(ctx, databaseDir+"/", 1000)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
for _, o := range objs {
|
||||
note(path.Base(o.Key), o.LastModified)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// isFolder refuses a folder that isn't there. A backup never creates the
|
||||
// destination folder itself: a missing one usually means a volume that
|
||||
// isn't mounted, and writing beside it would fill the server's own disk.
|
||||
func isFolder(dir string) error {
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: the folder %s isn't there; is its volume mounted? %w", dir, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("backup: %s is a file, not a folder", dir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Start begins a backup in the background and returns at once. It returns
|
||||
// ErrRunning while one runs, ErrNotReady when the destination isn't set up,
|
||||
// and ErrClosed once the server is shutting down.
|
||||
func (s *Service) Start(ctx context.Context, trigger string) error {
|
||||
st, err := s.settings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(problems(st)) > 0 {
|
||||
return ErrNotReady
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
return ErrClosed
|
||||
}
|
||||
if !s.runningSince.IsZero() {
|
||||
return ErrRunning
|
||||
}
|
||||
s.runningSince = s.now()
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
s.runningSince = time.Time{}
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
s.run(trigger, st)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Running reports when the running backup started, or the zero time.
|
||||
func (s *Service) Running() time.Time {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.runningSince
|
||||
}
|
||||
|
||||
// runLimit is the longest a run may take: the database file, and the
|
||||
// music's nightly budget when the backup includes the music.
|
||||
func runLimit(st dbq.BackupSetting) time.Duration {
|
||||
if st.IncludeMusic {
|
||||
return RunLimit + MusicBudget
|
||||
}
|
||||
return RunLimit
|
||||
}
|
||||
|
||||
func (s *Service) run(trigger string, st dbq.BackupSetting) {
|
||||
ctx, cancel := context.WithTimeout(s.base, RunLimit)
|
||||
defer cancel()
|
||||
q := dbq.New(s.pool)
|
||||
started, err := q.StartBackupRun(ctx, trigger)
|
||||
if err != nil {
|
||||
s.logger.Error("backup: record the run's start", "err", err)
|
||||
return
|
||||
}
|
||||
location, size, err := s.write(ctx, st, started.StartedAt.Time)
|
||||
// The music goes only after the database file has landed, which also
|
||||
// proves the destination takes writes.
|
||||
var music musicSent
|
||||
if err == nil && st.IncludeMusic {
|
||||
mctx, mcancel := context.WithTimeout(s.base, MusicBudget)
|
||||
music = s.sendMusic(mctx, st)
|
||||
mcancel()
|
||||
}
|
||||
|
||||
// The finish is recorded on its own context, so a run stopped by a
|
||||
// shutdown or its time limit still says how it ended.
|
||||
fctx, fcancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer fcancel()
|
||||
fin := dbq.FinishBackupRunParams{
|
||||
ID: started.ID, Status: "succeeded", Location: location, Bytes: size,
|
||||
MusicFiles: int32(music.files), MusicBytes: music.bytes, MusicNote: music.note,
|
||||
}
|
||||
if err != nil {
|
||||
fin.Status, fin.Error = "failed", describe(err)
|
||||
s.logger.Warn("backup failed", "trigger", trigger, "err", err)
|
||||
} else {
|
||||
s.logger.Info("backup written", "trigger", trigger, "location", location, "bytes", size,
|
||||
"music_files", music.files, "music_bytes", music.bytes, "music_note", music.note)
|
||||
}
|
||||
if err := q.FinishBackupRun(fctx, fin); err != nil {
|
||||
s.logger.Error("backup: record the run's end", "err", err)
|
||||
}
|
||||
if _, err := q.PruneBackupRuns(fctx, pgtype.Timestamptz{Time: s.now().Add(-keepRunsFor), Valid: true}); err != nil {
|
||||
s.logger.Warn("backup: prune old runs", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// describe is a run's error as the panel shows it.
|
||||
func describe(err error) string {
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
return fmt.Sprintf("The backup took longer than %d minutes and was stopped.", int(RunLimit/time.Minute))
|
||||
case errors.Is(err, context.Canceled):
|
||||
return "The server stopped while the backup was running."
|
||||
}
|
||||
return strings.TrimPrefix(err.Error(), "backup: ")
|
||||
}
|
||||
|
||||
// write snapshots the database into one backup file at the destination and
|
||||
// returns where it went and its size.
|
||||
func (s *Service) write(ctx context.Context, st dbq.BackupSetting, at time.Time) (string, int64, error) {
|
||||
if st.Destination == DestinationFolder {
|
||||
// Checked before the snapshot, so a missing volume costs nothing.
|
||||
if err := isFolder(st.Folder); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
}
|
||||
c, err := Snapshot(ctx, s.pool)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if st.Destination == DestinationFolder {
|
||||
dir := filepath.Join(st.Folder, databaseDir)
|
||||
if err := os.Mkdir(dir, 0o700); err != nil && !errors.Is(err, os.ErrExist) {
|
||||
return "", 0, fmt.Errorf("backup: create %s: %w", dir, err)
|
||||
}
|
||||
file, _, err := WriteFile(dir, c, at, s.build)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
info, err := os.Stat(file)
|
||||
if err != nil {
|
||||
return file, 0, err
|
||||
}
|
||||
return file, info.Size(), nil
|
||||
}
|
||||
|
||||
b, err := bucketOf(st)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
// The file is written to a temporary file first: S3 signs over the
|
||||
// body's hash and needs its length before the upload starts.
|
||||
tmp, err := os.CreateTemp("", ".minstrel-backup-*.tar")
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("backup: create a temporary file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmp.Name())
|
||||
}()
|
||||
h := sha256.New()
|
||||
if _, err := Write(io.MultiWriter(tmp, h), c, at, s.build); err != nil {
|
||||
return "", 0, fmt.Errorf("backup: write the temporary file: %w", err)
|
||||
}
|
||||
size, err := tmp.Seek(0, io.SeekCurrent)
|
||||
if err == nil {
|
||||
_, err = tmp.Seek(0, io.SeekStart)
|
||||
}
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("backup: rewind the temporary file: %w", err)
|
||||
}
|
||||
key := databaseDir + "/" + FileName(at)
|
||||
body := Throttle(ctx, tmp, int64(st.UploadLimitKib)*1024)
|
||||
if err := b.Put(ctx, key, body, size, hex.EncodeToString(h.Sum(nil))); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
return key, size, nil
|
||||
}
|
||||
|
||||
// Tick starts a scheduled backup when one is due. A backup that is already
|
||||
// running, or settings that aren't ready, mean there's nothing to do.
|
||||
func (s *Service) Tick(ctx context.Context) {
|
||||
st, err := s.settings(ctx)
|
||||
if err != nil {
|
||||
s.logger.Warn("backup: tick", "err", err)
|
||||
return
|
||||
}
|
||||
if !st.Enabled || len(problems(st)) > 0 {
|
||||
return
|
||||
}
|
||||
t, err := s.times(ctx)
|
||||
if err != nil {
|
||||
s.logger.Warn("backup: tick", "err", err)
|
||||
return
|
||||
}
|
||||
if due(s.now(), int(st.RunHour), t).After(s.now()) {
|
||||
return
|
||||
}
|
||||
if err := s.Start(ctx, TriggerSchedule); err != nil && !errors.Is(err, ErrRunning) && !errors.Is(err, ErrClosed) {
|
||||
s.logger.Warn("backup: start the scheduled backup", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Recover marks runs a stopped server left running as failed. It runs once
|
||||
// at startup, before the first tick.
|
||||
func (s *Service) Recover(ctx context.Context) {
|
||||
n, err := dbq.New(s.pool).FailUnfinishedBackupRuns(ctx)
|
||||
if err != nil {
|
||||
s.logger.Warn("backup: recover unfinished runs", "err", err)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
s.logger.Warn("backup: a run was cut off by the last shutdown", "runs", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Loop recovers, then ticks until ctx ends. The first tick waits a full
|
||||
// interval, so a restart isn't followed straight away by a backup.
|
||||
func (s *Service) Loop(ctx context.Context) {
|
||||
s.Recover(ctx)
|
||||
t := time.NewTicker(tickEvery)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
s.Tick(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close stops a running backup and waits for it to record how it ended.
|
||||
func (s *Service) Close() {
|
||||
s.mu.Lock()
|
||||
s.closed = true
|
||||
s.mu.Unlock()
|
||||
s.cancel()
|
||||
s.wg.Wait()
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
package backup_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||
)
|
||||
|
||||
func newService(t *testing.T) (*backup.Service, *pgxpool.Pool) {
|
||||
t.Helper()
|
||||
pool := newPool(t)
|
||||
svc := backup.NewService(pool, slog.New(slog.NewTextHandler(io.Discard, nil)), "test-build")
|
||||
t.Cleanup(svc.Close)
|
||||
return svc, pool
|
||||
}
|
||||
|
||||
// waitIdle waits for the running backup to finish, for at most 30 seconds.
|
||||
func waitIdle(t *testing.T, svc *backup.Service) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for !svc.Running().IsZero() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the backup is still running after 30s")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func status(t *testing.T, svc *backup.Service) backup.Status {
|
||||
t.Helper()
|
||||
st, err := svc.Status(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
func TestABackupIsOffAndRefusesToStartUntilSetUp_Integration(t *testing.T) {
|
||||
svc, _ := newService(t)
|
||||
st := status(t, svc)
|
||||
if st.State != backup.StateOff || st.Config.Enabled || st.NextRun != nil || len(st.Runs) != 0 {
|
||||
t.Errorf("a fresh server's backup = %+v, want off with no runs", st)
|
||||
}
|
||||
if err := svc.Start(context.Background(), backup.TriggerManual); !errors.Is(err, backup.ErrNotReady) {
|
||||
t.Errorf("Start = %v, want ErrNotReady", err)
|
||||
}
|
||||
if err := svc.Save(context.Background(), backup.Update{Enabled: true, Destination: backup.DestinationS3, RunHour: 3}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := status(t, svc); st.State != backup.StateIncomplete || len(st.Problems) != 4 {
|
||||
t.Errorf("switched on with no bucket: %s %v", st.State, st.Problems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABackupRunsIntoAFolder_Integration(t *testing.T) {
|
||||
svc, _ := newService(t)
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
if err := svc.Save(ctx, backup.Update{Enabled: true, Destination: backup.DestinationFolder, Folder: dir, RunHour: 3}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := status(t, svc); st.State != backup.StateWaiting || st.NextRun == nil {
|
||||
t.Errorf("set up, never run: %s next %v", st.State, st.NextRun)
|
||||
}
|
||||
if err := svc.Start(ctx, backup.TriggerManual); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitIdle(t, svc)
|
||||
|
||||
st := status(t, svc)
|
||||
if st.State != backup.StateOK || len(st.Runs) != 1 || st.LastSuccess == nil {
|
||||
t.Fatalf("after a run: %+v", st)
|
||||
}
|
||||
run := st.Runs[0]
|
||||
if run.Status != "succeeded" || run.Trigger != backup.TriggerManual || run.FinishedAt == nil ||
|
||||
filepath.Dir(run.Location) != filepath.Join(dir, "database") || run.Error != "" {
|
||||
t.Fatalf("run = %+v", run)
|
||||
}
|
||||
f, err := os.Open(run.Location)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
info, _ := f.Stat()
|
||||
if info.Size() != run.Bytes {
|
||||
t.Errorf("recorded %d bytes, the file holds %d", run.Bytes, info.Size())
|
||||
}
|
||||
m, _, err := backup.Read(f)
|
||||
if err != nil || m.Build != "test-build" {
|
||||
t.Errorf("the backup reads back as %+v, %v", m, err)
|
||||
}
|
||||
|
||||
res, err := svc.Test(ctx, backup.Update{Destination: backup.DestinationFolder, Folder: dir, RunHour: 3})
|
||||
if err != nil || res.Backups != 1 || res.Latest == nil {
|
||||
t.Errorf("Test = %+v, %v; want the one backup", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMissingFolderFailsTheRunAndSaysSo_Integration(t *testing.T) {
|
||||
svc, _ := newService(t)
|
||||
ctx := context.Background()
|
||||
missing := filepath.Join(t.TempDir(), "not-mounted")
|
||||
if err := svc.Save(ctx, backup.Update{Enabled: true, Destination: backup.DestinationFolder, Folder: missing, RunHour: 3}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := svc.Test(ctx, backup.Update{Destination: backup.DestinationFolder, Folder: missing}); err == nil {
|
||||
t.Error("Test passed on a missing folder")
|
||||
}
|
||||
if err := svc.Start(ctx, backup.TriggerManual); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitIdle(t, svc)
|
||||
st := status(t, svc)
|
||||
if st.State != backup.StateFailing || st.FailingSince == nil || len(st.Runs) != 1 ||
|
||||
!strings.Contains(st.Runs[0].Error, "isn't there") {
|
||||
t.Errorf("after a failed run: %s %+v", st.State, st.Runs)
|
||||
}
|
||||
if _, err := os.Stat(missing); !os.IsNotExist(err) {
|
||||
t.Error("the run created the missing folder")
|
||||
}
|
||||
}
|
||||
|
||||
// fakeBucket answers PUTs and lists like an S3 bucket. While hold is open a
|
||||
// PUT waits on it.
|
||||
type fakeBucket struct {
|
||||
mu sync.Mutex
|
||||
puts map[string][]byte
|
||||
hold chan struct{}
|
||||
}
|
||||
|
||||
func (b *fakeBucket) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 Credential=ak/") {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodPut:
|
||||
if b.hold != nil {
|
||||
<-b.hold
|
||||
}
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
b.mu.Lock()
|
||||
b.puts[r.URL.Path] = body
|
||||
b.mu.Unlock()
|
||||
case http.MethodGet:
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
_, _ = fmt.Fprint(w, `<ListBucketResult><IsTruncated>false</IsTruncated>`)
|
||||
for p, body := range b.puts {
|
||||
_, _ = fmt.Fprintf(w, `<Contents><Key>%s</Key><Size>%d</Size><LastModified>2026-11-02T03:00:05.000Z</LastModified></Contents>`,
|
||||
strings.TrimPrefix(p, "/minstrel/"), len(body))
|
||||
}
|
||||
_, _ = fmt.Fprint(w, `</ListBucketResult>`)
|
||||
}
|
||||
}
|
||||
|
||||
func bucketUpdate(endpoint string) backup.Update {
|
||||
return backup.Update{Enabled: true, Destination: backup.DestinationS3, Endpoint: endpoint,
|
||||
Bucket: "minstrel", AccessKey: "ak", SecretKey: "sk", RunHour: 3}
|
||||
}
|
||||
|
||||
func TestABackupRunsIntoABucket_Integration(t *testing.T) {
|
||||
svc, pool := newService(t)
|
||||
fake := &fakeBucket{puts: map[string][]byte{}}
|
||||
srv := httptest.NewServer(fake)
|
||||
defer srv.Close()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := svc.Save(ctx, bucketUpdate(srv.URL)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stored string
|
||||
if err := pool.QueryRow(ctx, "SELECT secret_key FROM backup_settings").Scan(&stored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(stored, "v1.") {
|
||||
t.Fatalf("the secret key is stored as %q, want it sealed", stored)
|
||||
}
|
||||
if cfg, _ := svc.Config(ctx); !cfg.HasSecretKey {
|
||||
t.Error("the config doesn't say a secret key is stored")
|
||||
}
|
||||
if err := svc.Start(ctx, backup.TriggerManual); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitIdle(t, svc)
|
||||
|
||||
st := status(t, svc)
|
||||
if st.State != backup.StateOK || len(st.Runs) != 1 || st.Runs[0].Status != "succeeded" {
|
||||
t.Fatalf("after a run: %s %+v", st.State, st.Runs)
|
||||
}
|
||||
key := st.Runs[0].Location
|
||||
if !strings.HasPrefix(key, "database/minstrel-") || !backup.IsFileName(filepath.Base(key)) {
|
||||
t.Errorf("stored under %q", key)
|
||||
}
|
||||
fake.mu.Lock()
|
||||
body := fake.puts["/minstrel/"+key]
|
||||
fake.mu.Unlock()
|
||||
if int64(len(body)) != st.Runs[0].Bytes {
|
||||
t.Errorf("the bucket got %d bytes, the run recorded %d", len(body), st.Runs[0].Bytes)
|
||||
}
|
||||
if _, _, err := backup.Read(bytes.NewReader(body)); err != nil {
|
||||
t.Errorf("the uploaded backup doesn't read: %v", err)
|
||||
}
|
||||
|
||||
// The test sends no secret key: it uses the stored one, and only lists.
|
||||
u := bucketUpdate(srv.URL)
|
||||
u.SecretKey = ""
|
||||
res, err := svc.Test(ctx, u)
|
||||
if err != nil || res.Backups != 1 {
|
||||
t.Errorf("Test = %+v, %v; want the one backup", res, err)
|
||||
}
|
||||
fake.mu.Lock()
|
||||
defer fake.mu.Unlock()
|
||||
if len(fake.puts) != 1 {
|
||||
t.Errorf("the test wrote to the bucket: %d objects", len(fake.puts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneBackupAtATime_Integration(t *testing.T) {
|
||||
svc, _ := newService(t)
|
||||
fake := &fakeBucket{puts: map[string][]byte{}, hold: make(chan struct{})}
|
||||
srv := httptest.NewServer(fake)
|
||||
defer srv.Close()
|
||||
ctx := context.Background()
|
||||
if err := svc.Save(ctx, bucketUpdate(srv.URL)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := svc.Start(ctx, backup.TriggerManual); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := svc.Start(ctx, backup.TriggerManual); !errors.Is(err, backup.ErrRunning) {
|
||||
t.Errorf("a second Start = %v, want ErrRunning", err)
|
||||
}
|
||||
if st := status(t, svc); st.RunningSince == nil {
|
||||
t.Error("the status doesn't say a backup is running")
|
||||
}
|
||||
close(fake.hold)
|
||||
waitIdle(t, svc)
|
||||
}
|
||||
|
||||
func TestRecoverFailsARunTheLastShutdownCutOff_Integration(t *testing.T) {
|
||||
svc, pool := newService(t)
|
||||
ctx := context.Background()
|
||||
if _, err := pool.Exec(ctx, "INSERT INTO backup_runs (trigger) VALUES ('schedule')"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc.Recover(ctx)
|
||||
st := status(t, svc)
|
||||
if len(st.Runs) != 1 || st.Runs[0].Status != "failed" || st.Runs[0].FinishedAt == nil ||
|
||||
!strings.Contains(st.Runs[0].Error, "server stopped") {
|
||||
t.Errorf("runs = %+v", st.Runs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveRefusesABadValueAndKeepsTheOldSettings_Integration(t *testing.T) {
|
||||
svc, _ := newService(t)
|
||||
ctx := context.Background()
|
||||
err := svc.Save(ctx, backup.Update{Enabled: true, Destination: backup.DestinationFolder, Folder: "relative/path"})
|
||||
var fe *backup.FieldError
|
||||
if !errors.As(err, &fe) || fe.Field != "folder" {
|
||||
t.Fatalf("err = %v, want a FieldError on folder", err)
|
||||
}
|
||||
if cfg, _ := svc.Config(ctx); cfg.Enabled || cfg.Folder != "" {
|
||||
t.Errorf("a refused save changed the settings: %+v", cfg)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
)
|
||||
|
||||
func at(s string) time.Time {
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func TestDueRunsAtTheHourCatchesUpAndBacksOffAfterAFailure(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
now string
|
||||
t runTimes
|
||||
want string
|
||||
}{
|
||||
{"never run: the slot that passed is due", "2026-11-02T10:00:00Z",
|
||||
runTimes{}, "2026-11-02T03:00:00Z"},
|
||||
{"ran at the hour: tomorrow's slot", "2026-11-02T10:00:00Z",
|
||||
runTimes{lastSuccess: at("2026-11-02T03:00:05Z"), lastAttempt: at("2026-11-02T03:00:05Z")}, "2026-11-03T03:00:00Z"},
|
||||
{"before today's hour, yesterday's ran: today's slot", "2026-11-02T02:00:00Z",
|
||||
runTimes{lastSuccess: at("2026-11-01T03:00:05Z"), lastAttempt: at("2026-11-01T03:00:05Z")}, "2026-11-02T03:00:00Z"},
|
||||
{"server was down at the hour: catches up", "2026-11-02T10:00:00Z",
|
||||
runTimes{lastSuccess: at("2026-11-01T03:00:05Z"), lastAttempt: at("2026-11-01T03:00:05Z")}, "2026-11-02T03:00:00Z"},
|
||||
{"first failure: an hour later", "2026-11-02T03:30:00Z",
|
||||
runTimes{lastAttempt: at("2026-11-02T03:00:00Z"), failingSince: at("2026-11-02T03:00:00Z")}, "2026-11-02T04:00:00Z"},
|
||||
{"failing for two hours: two hours later", "2026-11-02T05:30:00Z",
|
||||
runTimes{lastAttempt: at("2026-11-02T05:00:00Z"), failingSince: at("2026-11-02T03:00:00Z")}, "2026-11-02T07:00:00Z"},
|
||||
{"a retry past the next slot waits for the slot", "2026-11-02T19:30:00Z",
|
||||
runTimes{lastAttempt: at("2026-11-02T19:00:00Z"), failingSince: at("2026-11-02T03:00:00Z")}, "2026-11-03T03:00:00Z"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := due(at(c.now), 3, c.t); !got.Equal(at(c.want)) {
|
||||
t.Errorf("%s: due = %s, want %s", c.name, got.Format(time.RFC3339), c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheHourIsUTC(t *testing.T) {
|
||||
// 22:00 at UTC-5 is 03:00 UTC the next day, which is that day's slot.
|
||||
now := time.Date(2026, 11, 1, 22, 0, 0, 0, time.FixedZone("x", -5*3600))
|
||||
if got := slotBefore(now, 3); !got.Equal(at("2026-11-02T03:00:00Z")) {
|
||||
t.Errorf("slot = %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerdict(t *testing.T) {
|
||||
now := at("2026-11-02T10:00:00Z")
|
||||
on := dbq.BackupSetting{Enabled: true}
|
||||
good := runTimes{lastSuccess: at("2026-11-02T03:00:00Z"), lastAttempt: at("2026-11-02T03:00:00Z")}
|
||||
cases := []struct {
|
||||
name string
|
||||
st dbq.BackupSetting
|
||||
probs []string
|
||||
t runTimes
|
||||
want string
|
||||
}{
|
||||
{"switched off, even with problems and failures", dbq.BackupSetting{}, []string{"x"}, runTimes{failingSince: now}, StateOff},
|
||||
{"on but not set up", on, []string{"x"}, good, StateIncomplete},
|
||||
{"the latest run failed", on, nil, runTimes{lastSuccess: good.lastSuccess, failingSince: now}, StateFailing},
|
||||
{"nothing has run yet", on, nil, runTimes{}, StateWaiting},
|
||||
{"the latest backup is old", on, nil, runTimes{lastSuccess: now.Add(-27 * time.Hour)}, StateOverdue},
|
||||
{"a day and a bit is still fine", on, nil, runTimes{lastSuccess: now.Add(-25 * time.Hour)}, StateOK},
|
||||
{"fine", on, nil, good, StateOK},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := verdict(c.st, c.probs, c.t, now); got != c.want {
|
||||
t.Errorf("%s: %s, want %s", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProblemsNameWhatIsMissing(t *testing.T) {
|
||||
if p := problems(dbq.BackupSetting{Destination: DestinationS3}); len(p) != 4 {
|
||||
t.Errorf("an empty bucket: %v, want endpoint, name, access key and secret key", p)
|
||||
}
|
||||
if p := problems(dbq.BackupSetting{Destination: DestinationS3, Endpoint: "https://s3.example.com", Bucket: "b", AccessKey: "a", SecretKey: "v1.x"}); len(p) != 0 {
|
||||
t.Errorf("a whole bucket: %v", p)
|
||||
}
|
||||
if p := problems(dbq.BackupSetting{Destination: DestinationFolder}); len(p) != 1 {
|
||||
t.Errorf("an empty folder: %v", p)
|
||||
}
|
||||
if p := problems(dbq.BackupSetting{Destination: DestinationFolder, Folder: "/backups"}); len(p) != 0 {
|
||||
t.Errorf("a folder: %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeRefusesWhatCannotBeKept(t *testing.T) {
|
||||
ok := Update{Destination: DestinationS3, RunHour: 3}
|
||||
for field, u := range map[string]func(Update) Update{
|
||||
"destination": func(u Update) Update { u.Destination = "ftp"; return u },
|
||||
"run_hour": func(u Update) Update { u.RunHour = 24; return u },
|
||||
"upload_limit_kib": func(u Update) Update { u.UploadLimitKiB = -1; return u },
|
||||
"folder": func(u Update) Update { u.Folder = "backups"; return u },
|
||||
"endpoint": func(u Update) Update { u.Endpoint = "s3.example.com"; return u },
|
||||
} {
|
||||
_, err := merge(dbq.BackupSetting{}, u(ok))
|
||||
var fe *FieldError
|
||||
if !errors.As(err, &fe) || fe.Field != field {
|
||||
t.Errorf("%s: err = %v, want a FieldError on it", field, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeTidiesAndKeepsTheStoredSecret(t *testing.T) {
|
||||
st, err := merge(dbq.BackupSetting{SecretKey: "v1.stored"}, Update{
|
||||
Enabled: true, Destination: DestinationS3, Endpoint: " https://s3.example.com/ ",
|
||||
Bucket: " minstrel ", AccessKey: " ak ", RunHour: 4, UploadLimitKiB: 512, Folder: "/srv/backups/",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.Endpoint != "https://s3.example.com" || st.Bucket != "minstrel" || st.AccessKey != "ak" ||
|
||||
st.SecretKey != "v1.stored" || st.RunHour != 4 || st.UploadLimitKib != 512 || st.Folder != "/srv/backups" || !st.Enabled {
|
||||
t.Errorf("merged = %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThrottleKeepsToItsRate(t *testing.T) {
|
||||
data := bytes.Repeat([]byte("x"), 20*1024)
|
||||
start := time.Now()
|
||||
got, err := io.ReadAll(Throttle(context.Background(), bytes.NewReader(data), 100*1024))
|
||||
if err != nil || !bytes.Equal(got, data) {
|
||||
t.Fatalf("read %d bytes, err %v", len(got), err)
|
||||
}
|
||||
// 20 KiB at 100 KiB a second takes at least a fifth of a second.
|
||||
if took := time.Since(start); took < 180*time.Millisecond {
|
||||
t.Errorf("took %s, want at least 200ms", took)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThrottleWithNoLimitIsTheReaderItself(t *testing.T) {
|
||||
r := strings.NewReader("x")
|
||||
if Throttle(context.Background(), r, 0) != io.Reader(r) {
|
||||
t.Error("a zero limit wrapped the reader")
|
||||
}
|
||||
}
|
||||
|
||||
func TestThrottleStopsWhenItsContextEnds(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err := io.ReadAll(Throttle(ctx, bytes.NewReader(make([]byte, 4096)), 1024))
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("err = %v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescribeSaysWhyARunStopped(t *testing.T) {
|
||||
if got := describe(context.DeadlineExceeded); !strings.Contains(got, "longer than 30 minutes") {
|
||||
t.Errorf("deadline: %q", got)
|
||||
}
|
||||
if got := describe(context.Canceled); !strings.Contains(got, "server stopped") {
|
||||
t.Errorf("canceled: %q", got)
|
||||
}
|
||||
if got := describe(errors.New("backup: the folder /x isn't there")); got != "the folder /x isn't there" {
|
||||
t.Errorf("other: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// emptySHA256 is the payload hash of a request with no body.
|
||||
const emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
|
||||
// sign adds AWS Signature Version 4 headers for S3 to req: X-Amz-Date,
|
||||
// X-Amz-Content-Sha256 and Authorization. Every header already on the
|
||||
// request is signed, along with the host.
|
||||
func sign(req *http.Request, payloadSHA256, accessKey, secretKey, region string, at time.Time) {
|
||||
at = at.UTC()
|
||||
stamp := at.Format("20060102T150405Z")
|
||||
date := at.Format("20060102")
|
||||
req.Header.Set("X-Amz-Date", stamp)
|
||||
req.Header.Set("X-Amz-Content-Sha256", payloadSHA256)
|
||||
|
||||
host := req.Host
|
||||
if host == "" {
|
||||
host = req.URL.Host
|
||||
}
|
||||
headers := map[string]string{"host": host}
|
||||
for name, values := range req.Header {
|
||||
headers[strings.ToLower(name)] = strings.TrimSpace(strings.Join(values, ","))
|
||||
}
|
||||
names := make([]string, 0, len(headers))
|
||||
for name := range headers {
|
||||
names = append(names, name)
|
||||
}
|
||||
slices.Sort(names)
|
||||
var canonHeaders strings.Builder
|
||||
for _, name := range names {
|
||||
canonHeaders.WriteString(name + ":" + headers[name] + "\n")
|
||||
}
|
||||
signed := strings.Join(names, ";")
|
||||
|
||||
canonical := strings.Join([]string{
|
||||
req.Method,
|
||||
canonicalPath(req.URL),
|
||||
canonicalQuery(req.URL.Query()),
|
||||
canonHeaders.String(),
|
||||
signed,
|
||||
payloadSHA256,
|
||||
}, "\n")
|
||||
scope := date + "/" + region + "/s3/aws4_request"
|
||||
toSign := "AWS4-HMAC-SHA256\n" + stamp + "\n" + scope + "\n" + hexSHA256([]byte(canonical))
|
||||
|
||||
key := hmacSHA256([]byte("AWS4"+secretKey), date)
|
||||
key = hmacSHA256(key, region)
|
||||
key = hmacSHA256(key, "s3")
|
||||
key = hmacSHA256(key, "aws4_request")
|
||||
signature := hex.EncodeToString(hmacSHA256(key, toSign))
|
||||
|
||||
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+accessKey+"/"+scope+
|
||||
", SignedHeaders="+signed+", Signature="+signature)
|
||||
}
|
||||
|
||||
// canonicalPath is the request path with each segment URI-encoded once.
|
||||
func canonicalPath(u *url.URL) string {
|
||||
p := u.EscapedPath()
|
||||
if p == "" {
|
||||
return "/"
|
||||
}
|
||||
if unescaped, err := url.PathUnescape(p); err == nil {
|
||||
p = unescaped
|
||||
}
|
||||
segments := strings.Split(p, "/")
|
||||
for i, s := range segments {
|
||||
segments[i] = uriEncode(s)
|
||||
}
|
||||
return strings.Join(segments, "/")
|
||||
}
|
||||
|
||||
// canonicalQuery is the query sorted by name, names and values URI-encoded.
|
||||
func canonicalQuery(q url.Values) string {
|
||||
pairs := make([]string, 0, len(q))
|
||||
for name, values := range q {
|
||||
for _, v := range values {
|
||||
pairs = append(pairs, uriEncode(name)+"="+uriEncode(v))
|
||||
}
|
||||
}
|
||||
slices.Sort(pairs)
|
||||
return strings.Join(pairs, "&")
|
||||
}
|
||||
|
||||
// uriEncode percent-encodes everything but the unreserved characters, as
|
||||
// SigV4 requires; url.QueryEscape differs on space and '~'.
|
||||
func uriEncode(s string) string {
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' || c == '_' || c == '.' || c == '~' {
|
||||
b.WriteByte(c)
|
||||
continue
|
||||
}
|
||||
b.WriteString("%" + strings.ToUpper(hex.EncodeToString([]byte{c})))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func hmacSHA256(key []byte, msg string) []byte {
|
||||
m := hmac.New(sha256.New, key)
|
||||
m.Write([]byte(msg))
|
||||
return m.Sum(nil)
|
||||
}
|
||||
|
||||
func hexSHA256(b []byte) string {
|
||||
sum := sha256.Sum256(b)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -111,6 +111,7 @@ var Tables = []Table{
|
||||
// The library change feed is how the apps sync, including a first sync,
|
||||
// so a restored server carries it on from where the backup left it.
|
||||
{Name: "library_changes"},
|
||||
{Name: "library_changes_trimmed"},
|
||||
{Name: "audit_log"},
|
||||
{Name: "playback_errors"},
|
||||
}
|
||||
@@ -146,6 +147,10 @@ var Excluded = map[string]string{
|
||||
"want_candidates": "the last search's results; the next search finds them again",
|
||||
"scan_runs": "a log of past library scans",
|
||||
"diagnostic_events": "debug-mode diagnostics",
|
||||
|
||||
"backup_settings": "holds the bucket's secret key; point the restored server at its destination again",
|
||||
"backup_runs": "this server's own backup history",
|
||||
"backup_music_files": "which music files this server has sent; the files themselves are beside the database in the destination",
|
||||
}
|
||||
|
||||
// carried reports whether a table is in Tables.
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Throttle returns a reader that reads r no faster than perSecond bytes a
|
||||
// second, so an upload leaves room on the connection for everything else.
|
||||
// perSecond of 0 or less returns r unchanged. A wait ends early when ctx does,
|
||||
// and the read then returns ctx's error.
|
||||
func Throttle(ctx context.Context, r io.Reader, perSecond int64) io.Reader {
|
||||
if perSecond <= 0 {
|
||||
return r
|
||||
}
|
||||
return &throttled{ctx: ctx, r: r, perSecond: perSecond}
|
||||
}
|
||||
|
||||
type throttled struct {
|
||||
ctx context.Context
|
||||
r io.Reader
|
||||
perSecond int64
|
||||
start time.Time
|
||||
sent int64
|
||||
}
|
||||
|
||||
func (t *throttled) Read(p []byte) (int, error) {
|
||||
if t.start.IsZero() {
|
||||
t.start = time.Now()
|
||||
}
|
||||
// Reads are cut to a tenth of a second's worth, so the pace is even
|
||||
// rather than a burst followed by a long pause.
|
||||
if chunk := max(t.perSecond/10, 1); int64(len(p)) > chunk {
|
||||
p = p[:chunk]
|
||||
}
|
||||
n, err := t.r.Read(p)
|
||||
t.sent += int64(n)
|
||||
at := t.start.Add(time.Duration(float64(t.sent) / float64(t.perSecond) * float64(time.Second)))
|
||||
if wait := time.Until(at); wait > 0 {
|
||||
timer := time.NewTimer(wait)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-t.ctx.Done():
|
||||
return n, t.ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: backups.sql
|
||||
|
||||
package dbq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const backupMusicProgress = `-- name: BackupMusicProgress :one
|
||||
SELECT count(*)::bigint AS total_files,
|
||||
COALESCE(sum(t.file_size), 0)::bigint AS total_bytes,
|
||||
count(*) FILTER (WHERE m.error = '' AND m.size = t.file_size AND t.updated_at <= m.checked_at)::bigint AS sent_files,
|
||||
COALESCE(sum(t.file_size) FILTER (WHERE m.error = '' AND m.size = t.file_size AND t.updated_at <= m.checked_at), 0)::bigint AS sent_bytes,
|
||||
count(*) FILTER (WHERE m.error <> '' AND m.size = t.file_size AND t.updated_at <= m.checked_at)::bigint AS refused_files
|
||||
FROM tracks t
|
||||
LEFT JOIN backup_music_files m ON m.track_id = t.id
|
||||
WHERE t.missing_since IS NULL
|
||||
`
|
||||
|
||||
type BackupMusicProgressRow struct {
|
||||
TotalFiles int64
|
||||
TotalBytes int64
|
||||
SentFiles int64
|
||||
SentBytes int64
|
||||
RefusedFiles int64
|
||||
}
|
||||
|
||||
// How much of the library the backup holds as it is now.
|
||||
func (q *Queries) BackupMusicProgress(ctx context.Context) (BackupMusicProgressRow, error) {
|
||||
row := q.db.QueryRow(ctx, backupMusicProgress)
|
||||
var i BackupMusicProgressRow
|
||||
err := row.Scan(
|
||||
&i.TotalFiles,
|
||||
&i.TotalBytes,
|
||||
&i.SentFiles,
|
||||
&i.SentBytes,
|
||||
&i.RefusedFiles,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const backupRunTimes = `-- name: BackupRunTimes :one
|
||||
SELECT
|
||||
COALESCE((SELECT max(started_at) FROM backup_runs WHERE status = 'succeeded'),
|
||||
'epoch'::timestamptz)::timestamptz AS last_success,
|
||||
COALESCE((SELECT max(started_at) FROM backup_runs),
|
||||
'epoch'::timestamptz)::timestamptz AS last_attempt,
|
||||
COALESCE((SELECT min(started_at) FROM backup_runs
|
||||
WHERE status = 'failed'
|
||||
AND started_at > COALESCE((SELECT max(started_at) FROM backup_runs WHERE status = 'succeeded'),
|
||||
'epoch'::timestamptz)),
|
||||
'epoch'::timestamptz)::timestamptz AS failing_since
|
||||
`
|
||||
|
||||
type BackupRunTimesRow struct {
|
||||
LastSuccess pgtype.Timestamptz
|
||||
LastAttempt pgtype.Timestamptz
|
||||
FailingSince pgtype.Timestamptz
|
||||
}
|
||||
|
||||
// The times the verdict turns on. A run that never happened is the epoch,
|
||||
// so the row always exists. failing_since is the first failure after the last
|
||||
// success, and the epoch when the latest finished run succeeded.
|
||||
func (q *Queries) BackupRunTimes(ctx context.Context) (BackupRunTimesRow, error) {
|
||||
row := q.db.QueryRow(ctx, backupRunTimes)
|
||||
var i BackupRunTimesRow
|
||||
err := row.Scan(&i.LastSuccess, &i.LastAttempt, &i.FailingSince)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const failUnfinishedBackupRuns = `-- name: FailUnfinishedBackupRuns :execrows
|
||||
UPDATE backup_runs
|
||||
SET status = 'failed', finished_at = now(),
|
||||
error = 'The server stopped while the backup was running.'
|
||||
WHERE status = 'running'
|
||||
`
|
||||
|
||||
func (q *Queries) FailUnfinishedBackupRuns(ctx context.Context) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, failUnfinishedBackupRuns)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const finishBackupRun = `-- name: FinishBackupRun :exec
|
||||
UPDATE backup_runs
|
||||
SET status = $2,
|
||||
finished_at = now(),
|
||||
location = $3,
|
||||
bytes = $4,
|
||||
error = $5,
|
||||
music_files = $6,
|
||||
music_bytes = $7,
|
||||
music_note = $8
|
||||
WHERE id = $1
|
||||
`
|
||||
|
||||
type FinishBackupRunParams struct {
|
||||
ID pgtype.UUID
|
||||
Status string
|
||||
Location string
|
||||
Bytes int64
|
||||
Error string
|
||||
MusicFiles int32
|
||||
MusicBytes int64
|
||||
MusicNote string
|
||||
}
|
||||
|
||||
func (q *Queries) FinishBackupRun(ctx context.Context, arg FinishBackupRunParams) error {
|
||||
_, err := q.db.Exec(ctx, finishBackupRun,
|
||||
arg.ID,
|
||||
arg.Status,
|
||||
arg.Location,
|
||||
arg.Bytes,
|
||||
arg.Error,
|
||||
arg.MusicFiles,
|
||||
arg.MusicBytes,
|
||||
arg.MusicNote,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const getBackupSettings = `-- name: GetBackupSettings :one
|
||||
|
||||
SELECT id, enabled, destination, folder, endpoint, bucket, access_key, secret_key, run_hour, upload_limit_kib, updated_at, include_music FROM backup_settings WHERE id = true
|
||||
`
|
||||
|
||||
// Minstrel's own backup: its settings and its runs.
|
||||
func (q *Queries) GetBackupSettings(ctx context.Context) (BackupSetting, error) {
|
||||
row := q.db.QueryRow(ctx, getBackupSettings)
|
||||
var i BackupSetting
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Enabled,
|
||||
&i.Destination,
|
||||
&i.Folder,
|
||||
&i.Endpoint,
|
||||
&i.Bucket,
|
||||
&i.AccessKey,
|
||||
&i.SecretKey,
|
||||
&i.RunHour,
|
||||
&i.UploadLimitKib,
|
||||
&i.UpdatedAt,
|
||||
&i.IncludeMusic,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const listBackupRuns = `-- name: ListBackupRuns :many
|
||||
SELECT id, trigger, status, started_at, finished_at, location, bytes, error,
|
||||
music_files, music_bytes, music_note
|
||||
FROM backup_runs
|
||||
ORDER BY started_at DESC
|
||||
LIMIT $1
|
||||
`
|
||||
|
||||
func (q *Queries) ListBackupRuns(ctx context.Context, limit int32) ([]BackupRun, error) {
|
||||
rows, err := q.db.Query(ctx, listBackupRuns, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []BackupRun
|
||||
for rows.Next() {
|
||||
var i BackupRun
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Trigger,
|
||||
&i.Status,
|
||||
&i.StartedAt,
|
||||
&i.FinishedAt,
|
||||
&i.Location,
|
||||
&i.Bytes,
|
||||
&i.Error,
|
||||
&i.MusicFiles,
|
||||
&i.MusicBytes,
|
||||
&i.MusicNote,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const pendingBackupMusic = `-- name: PendingBackupMusic :many
|
||||
SELECT t.id, t.file_path, t.file_size,
|
||||
m.size AS sent_size, m.mtime AS sent_mtime, m.error AS sent_error
|
||||
FROM tracks t
|
||||
LEFT JOIN backup_music_files m ON m.track_id = t.id
|
||||
WHERE t.missing_since IS NULL
|
||||
AND t.file_path > $1::text
|
||||
AND (m.track_id IS NULL OR m.size <> t.file_size OR t.updated_at > m.checked_at
|
||||
OR (m.error <> '' AND m.checked_at < (SELECT updated_at FROM backup_settings)))
|
||||
ORDER BY t.file_path
|
||||
LIMIT $2
|
||||
`
|
||||
|
||||
type PendingBackupMusicParams struct {
|
||||
AfterPath string
|
||||
MaxRows int32
|
||||
}
|
||||
|
||||
type PendingBackupMusicRow struct {
|
||||
ID pgtype.UUID
|
||||
FilePath string
|
||||
FileSize int64
|
||||
SentSize *int64
|
||||
SentMtime pgtype.Timestamptz
|
||||
SentError *string
|
||||
}
|
||||
|
||||
// Library files the backup may not hold as they are now, in path order from
|
||||
// after_path, so a night that runs out of time carries on where it stopped:
|
||||
// never sent, a different size, touched by a scan since they were last
|
||||
// checked, or refused before the settings were last saved (a fixed key gets
|
||||
// another go). The caller compares the file's own size and mtime before
|
||||
// sending.
|
||||
func (q *Queries) PendingBackupMusic(ctx context.Context, arg PendingBackupMusicParams) ([]PendingBackupMusicRow, error) {
|
||||
rows, err := q.db.Query(ctx, pendingBackupMusic, arg.AfterPath, arg.MaxRows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []PendingBackupMusicRow
|
||||
for rows.Next() {
|
||||
var i PendingBackupMusicRow
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.FilePath,
|
||||
&i.FileSize,
|
||||
&i.SentSize,
|
||||
&i.SentMtime,
|
||||
&i.SentError,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const pruneBackupRuns = `-- name: PruneBackupRuns :execrows
|
||||
DELETE FROM backup_runs WHERE status <> 'running' AND started_at < $1
|
||||
`
|
||||
|
||||
func (q *Queries) PruneBackupRuns(ctx context.Context, startedAt pgtype.Timestamptz) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, pruneBackupRuns, startedAt)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const recordBackupMusic = `-- name: RecordBackupMusic :exec
|
||||
INSERT INTO backup_music_files (track_id, size, mtime, location, error)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (track_id) DO UPDATE
|
||||
SET size = EXCLUDED.size, mtime = EXCLUDED.mtime, location = EXCLUDED.location,
|
||||
error = EXCLUDED.error, checked_at = now()
|
||||
`
|
||||
|
||||
type RecordBackupMusicParams struct {
|
||||
TrackID pgtype.UUID
|
||||
Size int64
|
||||
Mtime pgtype.Timestamptz
|
||||
Location string
|
||||
Error string
|
||||
}
|
||||
|
||||
func (q *Queries) RecordBackupMusic(ctx context.Context, arg RecordBackupMusicParams) error {
|
||||
_, err := q.db.Exec(ctx, recordBackupMusic,
|
||||
arg.TrackID,
|
||||
arg.Size,
|
||||
arg.Mtime,
|
||||
arg.Location,
|
||||
arg.Error,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const refusedBackupMusic = `-- name: RefusedBackupMusic :many
|
||||
SELECT t.file_path, m.error, m.checked_at
|
||||
FROM backup_music_files m
|
||||
JOIN tracks t ON t.id = m.track_id
|
||||
WHERE m.error <> '' AND m.size = t.file_size AND t.updated_at <= m.checked_at
|
||||
ORDER BY m.checked_at DESC
|
||||
LIMIT $1
|
||||
`
|
||||
|
||||
type RefusedBackupMusicRow struct {
|
||||
FilePath string
|
||||
Error string
|
||||
CheckedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
func (q *Queries) RefusedBackupMusic(ctx context.Context, limit int32) ([]RefusedBackupMusicRow, error) {
|
||||
rows, err := q.db.Query(ctx, refusedBackupMusic, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []RefusedBackupMusicRow
|
||||
for rows.Next() {
|
||||
var i RefusedBackupMusicRow
|
||||
if err := rows.Scan(&i.FilePath, &i.Error, &i.CheckedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const saveBackupSettings = `-- name: SaveBackupSettings :exec
|
||||
UPDATE backup_settings
|
||||
SET enabled = $1,
|
||||
destination = $2,
|
||||
folder = $3,
|
||||
endpoint = $4,
|
||||
bucket = $5,
|
||||
access_key = $6,
|
||||
secret_key = $7,
|
||||
run_hour = $8,
|
||||
upload_limit_kib = $9,
|
||||
include_music = $10,
|
||||
updated_at = now()
|
||||
WHERE id = true
|
||||
`
|
||||
|
||||
type SaveBackupSettingsParams struct {
|
||||
Enabled bool
|
||||
Destination string
|
||||
Folder string
|
||||
Endpoint string
|
||||
Bucket string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
RunHour int16
|
||||
UploadLimitKib int32
|
||||
IncludeMusic bool
|
||||
}
|
||||
|
||||
func (q *Queries) SaveBackupSettings(ctx context.Context, arg SaveBackupSettingsParams) error {
|
||||
_, err := q.db.Exec(ctx, saveBackupSettings,
|
||||
arg.Enabled,
|
||||
arg.Destination,
|
||||
arg.Folder,
|
||||
arg.Endpoint,
|
||||
arg.Bucket,
|
||||
arg.AccessKey,
|
||||
arg.SecretKey,
|
||||
arg.RunHour,
|
||||
arg.UploadLimitKib,
|
||||
arg.IncludeMusic,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const startBackupRun = `-- name: StartBackupRun :one
|
||||
INSERT INTO backup_runs (trigger) VALUES ($1)
|
||||
RETURNING id, started_at
|
||||
`
|
||||
|
||||
type StartBackupRunRow struct {
|
||||
ID pgtype.UUID
|
||||
StartedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
func (q *Queries) StartBackupRun(ctx context.Context, trigger string) (StartBackupRunRow, error) {
|
||||
row := q.db.QueryRow(ctx, startBackupRun, trigger)
|
||||
var i StartBackupRunRow
|
||||
err := row.Scan(&i.ID, &i.StartedAt)
|
||||
return i, err
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: genre_mixes.sql
|
||||
|
||||
package dbq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const listGenreMixCandidates = `-- name: ListGenreMixCandidates :many
|
||||
|
||||
WITH library_genres AS (
|
||||
SELECT lower(trim(g.g)) AS genre_key,
|
||||
min(trim(g.g))::text AS label,
|
||||
COUNT(DISTINCT t.id) AS track_count,
|
||||
array_agg(DISTINCT t.id)::uuid[] AS track_ids
|
||||
FROM tracks t
|
||||
CROSS JOIN LATERAL regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g(g)
|
||||
WHERE t.missing_since IS NULL
|
||||
AND trim(g.g) <> ''
|
||||
GROUP BY lower(trim(g.g))
|
||||
HAVING COUNT(DISTINCT t.id) >= 25
|
||||
AND COUNT(DISTINCT t.artist_id) >= 3
|
||||
),
|
||||
taste AS (
|
||||
SELECT lower(trim(tpt.tag)) AS genre_key, SUM(tpt.weight) AS weight
|
||||
FROM taste_profile_tags tpt
|
||||
WHERE tpt.user_id = $1
|
||||
GROUP BY lower(trim(tpt.tag))
|
||||
HAVING SUM(tpt.weight) > 0
|
||||
),
|
||||
listened AS (
|
||||
SELECT t.id
|
||||
FROM play_events pe
|
||||
JOIN tracks t ON t.id = pe.track_id
|
||||
WHERE pe.user_id = $1 AND pe.was_skipped = false
|
||||
UNION ALL
|
||||
SELECT gl.track_id
|
||||
FROM general_likes gl
|
||||
WHERE gl.user_id = $1
|
||||
),
|
||||
history AS (
|
||||
SELECT lower(trim(g.g)) AS genre_key, COUNT(*) AS plays
|
||||
FROM listened l
|
||||
JOIN tracks t ON t.id = l.id
|
||||
CROSS JOIN LATERAL regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g(g)
|
||||
WHERE trim(g.g) <> ''
|
||||
GROUP BY lower(trim(g.g))
|
||||
)
|
||||
SELECT lg.genre_key::text AS genre_key,
|
||||
lg.label AS label,
|
||||
(CASE WHEN ta.genre_key IS NOT NULL THEN 1
|
||||
WHEN h.genre_key IS NOT NULL THEN 2
|
||||
ELSE 3 END)::int AS tier,
|
||||
(CASE WHEN ta.genre_key IS NOT NULL THEN ta.weight
|
||||
WHEN h.genre_key IS NOT NULL THEN h.plays::float8
|
||||
ELSE lg.track_count::float8 END)::float8 AS score,
|
||||
lg.track_ids
|
||||
FROM library_genres lg
|
||||
LEFT JOIN taste ta ON ta.genre_key = lg.genre_key
|
||||
LEFT JOIN history h ON h.genre_key = lg.genre_key
|
||||
ORDER BY tier, score DESC, lg.genre_key
|
||||
LIMIT 60
|
||||
`
|
||||
|
||||
type ListGenreMixCandidatesRow struct {
|
||||
GenreKey string
|
||||
Label string
|
||||
Tier int32
|
||||
Score float64
|
||||
TrackIds []pgtype.UUID
|
||||
}
|
||||
|
||||
// Candidate queries for the genre mixes: one daily system playlist per genre
|
||||
// the user listens to. Genres live inline on tracks.genre as a delimited
|
||||
// string and are split on [;,] the same way the Discover and radio tag arms
|
||||
// split them. Matching is case-insensitive, so the ID3 "Post-Punk" and the
|
||||
// folksonomy "post-punk" in the taste profile count as one genre.
|
||||
// Library genres big enough to carry a mix (>= 25 playable tracks across
|
||||
// >= 3 artists), each tagged with how close it is to the user's listening:
|
||||
//
|
||||
// tier 1: a positive taste-profile genre, scored by its taste weight
|
||||
// tier 2: a genre of tracks the user has played through or liked, scored
|
||||
// by how many — catches a profile that has not been built yet or
|
||||
// has decayed past its window
|
||||
// tier 3: any other library genre, scored by its size
|
||||
//
|
||||
// track_ids lets the caller tell how much music two genres share: a genre
|
||||
// whose songs mostly sit in genres already picked adds no new channel, and a
|
||||
// broad genre ("Rock") whose songs mostly carry narrower ones ("Pop Punk",
|
||||
// "Ska Punk") gives way to them.
|
||||
// $1 user_id.
|
||||
func (q *Queries) ListGenreMixCandidates(ctx context.Context, userID pgtype.UUID) ([]ListGenreMixCandidatesRow, error) {
|
||||
rows, err := q.db.Query(ctx, listGenreMixCandidates, userID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListGenreMixCandidatesRow
|
||||
for rows.Next() {
|
||||
var i ListGenreMixCandidatesRow
|
||||
if err := rows.Scan(
|
||||
&i.GenreKey,
|
||||
&i.Label,
|
||||
&i.Tier,
|
||||
&i.Score,
|
||||
&i.TrackIds,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listGenreMixTracks = `-- name: ListGenreMixTracks :many
|
||||
WITH play_counts AS (
|
||||
SELECT pe.track_id, COUNT(*) AS c
|
||||
FROM play_events pe
|
||||
WHERE pe.user_id = $1 AND pe.was_skipped = false
|
||||
GROUP BY pe.track_id
|
||||
),
|
||||
skip_counts AS (
|
||||
SELECT pe.track_id, COUNT(*) AS c
|
||||
FROM play_events pe
|
||||
WHERE pe.user_id = $1 AND pe.was_skipped = true
|
||||
GROUP BY pe.track_id
|
||||
),
|
||||
artist_plays AS (
|
||||
SELECT t.artist_id,
|
||||
COUNT(*) FILTER (WHERE NOT pe.was_skipped) AS plays,
|
||||
COUNT(*) FILTER (WHERE pe.was_skipped) AS skips
|
||||
FROM play_events pe
|
||||
JOIN tracks t ON t.id = pe.track_id
|
||||
WHERE pe.user_id = $1
|
||||
GROUP BY t.artist_id
|
||||
),
|
||||
artist_likes AS (
|
||||
SELECT t.artist_id, COUNT(*) AS likes
|
||||
FROM general_likes gl
|
||||
JOIN tracks t ON t.id = gl.track_id
|
||||
WHERE gl.user_id = $1
|
||||
GROUP BY t.artist_id
|
||||
),
|
||||
own_artists AS (
|
||||
SELECT ap.artist_id FROM artist_plays ap WHERE ap.plays > 0
|
||||
UNION
|
||||
SELECT al.artist_id FROM artist_likes al
|
||||
),
|
||||
artist_sim AS (
|
||||
SELECT asim.artist_b_id AS artist_id,
|
||||
MAX(LEAST(GREATEST(asim.score, 0), 1)) AS sim
|
||||
FROM artist_similarity asim
|
||||
JOIN own_artists oa ON oa.artist_id = asim.artist_a_id
|
||||
GROUP BY asim.artist_b_id
|
||||
),
|
||||
eligible AS (
|
||||
SELECT t.id, t.album_id, t.artist_id,
|
||||
(COALESCE(pc.c, 0) > 0 OR gl.track_id IS NOT NULL) AS familiar,
|
||||
CASE
|
||||
WHEN COALESCE(pc.c, 0) > 0 OR gl.track_id IS NOT NULL THEN
|
||||
(1 + ln(1 + COALESCE(pc.c, 0)::float8)
|
||||
+ CASE WHEN gl.track_id IS NOT NULL THEN 1 ELSE 0 END)
|
||||
* ((COALESCE(pc.c, 0) + 1)::float8 / (COALESCE(pc.c, 0) + COALESCE(sc.c, 0) + 1))
|
||||
ELSE
|
||||
(1 + ln(1 + COALESCE(ap.plays, 0)::float8 + 2 * COALESCE(al.likes, 0)::float8)
|
||||
+ 2 * COALESCE(asm.sim, 0))
|
||||
* ((COALESCE(ap.plays, 0) + 1)::float8 / (COALESCE(ap.plays, 0) + COALESCE(ap.skips, 0) + 1))
|
||||
END::float8 AS weight,
|
||||
-- u in (0, 1): the first 32 bits of md5(track, date), shifted off 0.
|
||||
((('x' || substr(md5(t.id::text || $2::text), 1, 8))::bit(32)::bigint)::float8 + 1)
|
||||
/ 4294967297.0 AS u
|
||||
FROM tracks t
|
||||
LEFT JOIN play_counts pc ON pc.track_id = t.id
|
||||
LEFT JOIN skip_counts sc ON sc.track_id = t.id
|
||||
LEFT JOIN general_likes gl ON gl.user_id = $1 AND gl.track_id = t.id
|
||||
LEFT JOIN artist_plays ap ON ap.artist_id = t.artist_id
|
||||
LEFT JOIN artist_likes al ON al.artist_id = t.artist_id
|
||||
LEFT JOIN artist_sim asm ON asm.artist_id = t.artist_id
|
||||
WHERE t.missing_since IS NULL -- never offer a file that is gone
|
||||
AND NOT (COALESCE(sc.c, 0) >= 2 AND COALESCE(sc.c, 0) > COALESCE(pc.c, 0))
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g(g)
|
||||
WHERE lower(trim(g.g)) = $3::text
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g(g)
|
||||
WHERE lower(trim(g.g)) = ANY($4::text[])
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM quarantine q
|
||||
WHERE q.user_id = $1 AND q.track_id = t.id
|
||||
)
|
||||
),
|
||||
ranked AS (
|
||||
SELECT e.id, e.album_id, e.artist_id, e.familiar,
|
||||
-ln(e.u) / e.weight AS draw,
|
||||
row_number() OVER (PARTITION BY e.familiar ORDER BY -ln(e.u) / e.weight, e.id) AS rn
|
||||
FROM eligible e
|
||||
)
|
||||
SELECT r.id, r.album_id, r.artist_id, r.familiar::bool AS familiar
|
||||
FROM ranked r
|
||||
WHERE r.rn <= 200
|
||||
ORDER BY r.familiar DESC, r.draw, r.id
|
||||
`
|
||||
|
||||
type ListGenreMixTracksParams struct {
|
||||
UserID pgtype.UUID
|
||||
DateSeed string
|
||||
GenreKey string
|
||||
ExcludeGenres []string
|
||||
}
|
||||
|
||||
type ListGenreMixTracksRow struct {
|
||||
ID pgtype.UUID
|
||||
AlbumID pgtype.UUID
|
||||
ArtistID pgtype.UUID
|
||||
Familiar bool
|
||||
}
|
||||
|
||||
// Playable tracks carrying one genre, split into the two halves a genre mix
|
||||
// blends: familiar (the user has played it through or liked it) and unheard.
|
||||
// Up to 200 of each, drawn by a weighted lottery seeded with the date, so the
|
||||
// mix changes day to day, holds still within a day, and leans toward what the
|
||||
// user likes without becoming the same 50 songs every day.
|
||||
//
|
||||
// The lottery is the Efraimidis-Spirakis draw: each track gets
|
||||
// -ln(u) / weight with u uniform in (0, 1) from md5(track, date), and the
|
||||
// smallest keys win. A track of weight 3 is three times as likely as one of
|
||||
// weight 1 to come before it, and every track keeps some chance.
|
||||
//
|
||||
// Familiar weight: 1 + ln(1 + plays through) + 1 if liked, times the share of
|
||||
// its plays that were not skips. A song played 40 times weighs about 4.7, one
|
||||
// played once about 1.7.
|
||||
//
|
||||
// Unheard weight: 1 + ln(1 + the artist's plays through + 2 × its liked
|
||||
// tracks) + 2 × how similar the artist is to the user's own artists (the best
|
||||
// artist_similarity score from any artist they have played through or liked,
|
||||
// 0 to 1), times the share of the artist's plays that were not skips. An
|
||||
// artist the user has never met weighs 1, so new artists still come up; an
|
||||
// artist they keep skipping weighs well under 1.
|
||||
//
|
||||
// A track the user keeps skipping stays out: twice-skipped and skipped more
|
||||
// often than played through. The second half of that test keeps a favourite
|
||||
// with a couple of stray skips in.
|
||||
//
|
||||
// exclude_genres keeps each song in one mix: a track carrying a genre that
|
||||
// already has a mix today is left to that mix.
|
||||
func (q *Queries) ListGenreMixTracks(ctx context.Context, arg ListGenreMixTracksParams) ([]ListGenreMixTracksRow, error) {
|
||||
rows, err := q.db.Query(ctx, listGenreMixTracks,
|
||||
arg.UserID,
|
||||
arg.DateSeed,
|
||||
arg.GenreKey,
|
||||
arg.ExcludeGenres,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListGenreMixTracksRow
|
||||
for rows.Next() {
|
||||
var i ListGenreMixTracksRow
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.AlbumID,
|
||||
&i.ArtistID,
|
||||
&i.Familiar,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
@@ -11,21 +11,58 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const deleteOldLibraryChanges = `-- name: DeleteOldLibraryChanges :execrows
|
||||
DELETE FROM library_changes WHERE changed_at < $1
|
||||
const deleteOldLibraryDeletes = `-- name: DeleteOldLibraryDeletes :one
|
||||
WITH gone AS (
|
||||
DELETE FROM library_changes WHERE op = 'delete' AND changed_at < $1 RETURNING id
|
||||
), mark AS (
|
||||
UPDATE library_changes_trimmed
|
||||
SET through = GREATEST(through, (SELECT COALESCE(max(id), 0) FROM gone))
|
||||
)
|
||||
SELECT count(*)::BIGINT FROM gone
|
||||
`
|
||||
|
||||
// Removes library_changes rows older than the given cutoff. Run from
|
||||
// the sync compactor goroutine on a daily tick. Returns the number of
|
||||
// rows deleted so the worker can log meaningful progress.
|
||||
func (q *Queries) DeleteOldLibraryChanges(ctx context.Context, changedAt pgtype.Timestamptz) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, deleteOldLibraryChanges, changedAt)
|
||||
// Removes deletes older than the cutoff and raises the trimmed mark past the
|
||||
// highest one, so an app whose cursor is below it starts over. Returns how
|
||||
// many went.
|
||||
func (q *Queries) DeleteOldLibraryDeletes(ctx context.Context, changedAt pgtype.Timestamptz) (int64, error) {
|
||||
row := q.db.QueryRow(ctx, deleteOldLibraryDeletes, changedAt)
|
||||
var column_1 int64
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const deleteSupersededLibraryChanges = `-- name: DeleteSupersededLibraryChanges :execrows
|
||||
DELETE FROM library_changes old
|
||||
WHERE EXISTS (SELECT 1 FROM library_changes newer
|
||||
WHERE newer.entity_type = old.entity_type
|
||||
AND newer.entity_id = old.entity_id
|
||||
AND newer.id > old.id)
|
||||
`
|
||||
|
||||
// Removes every row a newer row for the same entity has superseded. An app
|
||||
// reading from any cursor still meets the newer row, so it loses nothing.
|
||||
func (q *Queries) DeleteSupersededLibraryChanges(ctx context.Context) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, deleteSupersededLibraryChanges)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const getLastIssuedLibraryChangeCursor = `-- name: GetLastIssuedLibraryChangeCursor :one
|
||||
SELECT COALESCE(pg_sequence_last_value(pg_get_serial_sequence('library_changes', 'id')::regclass), 0)::BIGINT
|
||||
`
|
||||
|
||||
// The highest cursor this server has handed out, even once the compactor has
|
||||
// trimmed the rows: the id sequence's last value. A restored server's
|
||||
// sequence stands at the backup's last change.
|
||||
func (q *Queries) GetLastIssuedLibraryChangeCursor(ctx context.Context) (int64, error) {
|
||||
row := q.db.QueryRow(ctx, getLastIssuedLibraryChangeCursor)
|
||||
var column_1 int64
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const getLibraryChangesSince = `-- name: GetLibraryChangesSince :many
|
||||
SELECT id, entity_type, entity_id, op, changed_at
|
||||
FROM library_changes
|
||||
@@ -65,6 +102,19 @@ func (q *Queries) GetLibraryChangesSince(ctx context.Context, arg GetLibraryChan
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const getLibraryChangesTrimmedThrough = `-- name: GetLibraryChangesTrimmedThrough :one
|
||||
SELECT through FROM library_changes_trimmed
|
||||
`
|
||||
|
||||
// The highest id of a delete the compactor has dropped. A cursor below it may
|
||||
// have missed that delete.
|
||||
func (q *Queries) GetLibraryChangesTrimmedThrough(ctx context.Context) (int64, error) {
|
||||
row := q.db.QueryRow(ctx, getLibraryChangesTrimmedThrough)
|
||||
var through int64
|
||||
err := row.Scan(&through)
|
||||
return through, err
|
||||
}
|
||||
|
||||
const getMaxLibraryChangeCursor = `-- name: GetMaxLibraryChangeCursor :one
|
||||
SELECT COALESCE(MAX(id), 0)::BIGINT FROM library_changes
|
||||
`
|
||||
@@ -76,17 +126,6 @@ func (q *Queries) GetMaxLibraryChangeCursor(ctx context.Context) (int64, error)
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const getMinLibraryChangeCursor = `-- name: GetMinLibraryChangeCursor :one
|
||||
SELECT COALESCE(MIN(id), 0)::BIGINT FROM library_changes
|
||||
`
|
||||
|
||||
func (q *Queries) GetMinLibraryChangeCursor(ctx context.Context) (int64, error) {
|
||||
row := q.db.QueryRow(ctx, getMinLibraryChangeCursor)
|
||||
var column_1 int64
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const insertLibraryChange = `-- name: InsertLibraryChange :exec
|
||||
INSERT INTO library_changes (entity_type, entity_id, op)
|
||||
VALUES ($1, $2, $3)
|
||||
|
||||
@@ -305,6 +305,44 @@ type AuditLog struct {
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type BackupMusicFile struct {
|
||||
TrackID pgtype.UUID
|
||||
Size int64
|
||||
Mtime pgtype.Timestamptz
|
||||
Location string
|
||||
CheckedAt pgtype.Timestamptz
|
||||
Error string
|
||||
}
|
||||
|
||||
type BackupRun struct {
|
||||
ID pgtype.UUID
|
||||
Trigger string
|
||||
Status string
|
||||
StartedAt pgtype.Timestamptz
|
||||
FinishedAt pgtype.Timestamptz
|
||||
Location string
|
||||
Bytes int64
|
||||
Error string
|
||||
MusicFiles int32
|
||||
MusicBytes int64
|
||||
MusicNote string
|
||||
}
|
||||
|
||||
type BackupSetting struct {
|
||||
ID bool
|
||||
Enabled bool
|
||||
Destination string
|
||||
Folder string
|
||||
Endpoint string
|
||||
Bucket string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
RunHour int16
|
||||
UploadLimitKib int32
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
IncludeMusic bool
|
||||
}
|
||||
|
||||
type CandidateArtistTag struct {
|
||||
CandidateMbid string
|
||||
Tag string
|
||||
@@ -506,6 +544,11 @@ type LibraryChange struct {
|
||||
ChangedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type LibraryChangesTrimmed struct {
|
||||
ID bool
|
||||
Through int64
|
||||
}
|
||||
|
||||
type LibrarySetting struct {
|
||||
ID bool
|
||||
ScanPaths []string
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
DROP TABLE IF EXISTS backup_runs;
|
||||
DROP TABLE IF EXISTS backup_settings;
|
||||
@@ -0,0 +1,44 @@
|
||||
-- 0099_backups.up.sql — Minstrel backing itself up.
|
||||
--
|
||||
-- backup_settings one row: whether the nightly backup is on, where it goes
|
||||
-- (a folder, or an S3 bucket such as a Fabled Archives
|
||||
-- target), and the hour it runs, in UTC. Off until an
|
||||
-- admin switches it on.
|
||||
-- backup_runs one row per attempt: when, what started it, how it ended,
|
||||
-- where the file went, its size, and the error in words.
|
||||
CREATE TABLE backup_settings (
|
||||
id boolean PRIMARY KEY DEFAULT true CONSTRAINT backup_settings_one_row CHECK (id),
|
||||
enabled boolean NOT NULL DEFAULT false,
|
||||
destination text NOT NULL DEFAULT 's3'
|
||||
CONSTRAINT backup_settings_destination_check CHECK (destination IN ('folder', 's3')),
|
||||
folder text NOT NULL DEFAULT '',
|
||||
endpoint text NOT NULL DEFAULT '',
|
||||
bucket text NOT NULL DEFAULT '',
|
||||
access_key text NOT NULL DEFAULT '',
|
||||
-- Sealed under the stream secret, as every stored credential is.
|
||||
secret_key text NOT NULL DEFAULT '',
|
||||
run_hour smallint NOT NULL DEFAULT 3
|
||||
CONSTRAINT backup_settings_run_hour_check CHECK (run_hour BETWEEN 0 AND 23),
|
||||
-- The most an upload to the bucket may send, in KiB a second; 0 is no limit.
|
||||
upload_limit_kib integer NOT NULL DEFAULT 4096
|
||||
CONSTRAINT backup_settings_upload_limit_check CHECK (upload_limit_kib >= 0),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
INSERT INTO backup_settings DEFAULT VALUES;
|
||||
|
||||
CREATE TABLE backup_runs (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
trigger text NOT NULL
|
||||
CONSTRAINT backup_runs_trigger_check CHECK (trigger IN ('schedule', 'manual')),
|
||||
status text NOT NULL DEFAULT 'running'
|
||||
CONSTRAINT backup_runs_status_check CHECK (status IN ('running', 'succeeded', 'failed')),
|
||||
started_at timestamptz NOT NULL DEFAULT now(),
|
||||
finished_at timestamptz,
|
||||
-- Where the file went: the object key, or the file's path in the folder.
|
||||
location text NOT NULL DEFAULT '',
|
||||
bytes bigint NOT NULL DEFAULT 0,
|
||||
error text NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE INDEX backup_runs_started_at_idx ON backup_runs (started_at DESC);
|
||||
@@ -0,0 +1,6 @@
|
||||
DROP TABLE IF EXISTS backup_music_files;
|
||||
ALTER TABLE backup_runs
|
||||
DROP COLUMN IF EXISTS music_note,
|
||||
DROP COLUMN IF EXISTS music_bytes,
|
||||
DROP COLUMN IF EXISTS music_files;
|
||||
ALTER TABLE backup_settings DROP COLUMN IF EXISTS include_music;
|
||||
@@ -0,0 +1,36 @@
|
||||
-- 0100_backup_music.up.sql — the music files in Minstrel's own backup.
|
||||
--
|
||||
-- Until now a backup held the database only. With include_music on, each
|
||||
-- library file is also sent once to music/<path> in the same destination,
|
||||
-- and sent again only when it changes.
|
||||
--
|
||||
-- backup_settings.include_music off until an admin switches it on.
|
||||
-- backup_music_files one row per track sent (or refused): the
|
||||
-- file's size and mtime when it was sent, so
|
||||
-- a later night sends only what is new or
|
||||
-- changed. A track the scan has touched since
|
||||
-- checked_at is looked at again; it goes up
|
||||
-- again only if the file itself changed, so a
|
||||
-- full rescan doesn't resend the library. A
|
||||
-- refusal is kept with its reason and tried
|
||||
-- again once the file changes.
|
||||
-- backup_runs.music_* how much a run sent, and why it stopped.
|
||||
ALTER TABLE backup_settings ADD COLUMN include_music boolean NOT NULL DEFAULT false;
|
||||
|
||||
ALTER TABLE backup_runs
|
||||
ADD COLUMN music_files integer NOT NULL DEFAULT 0,
|
||||
ADD COLUMN music_bytes bigint NOT NULL DEFAULT 0,
|
||||
ADD COLUMN music_note text NOT NULL DEFAULT '';
|
||||
|
||||
CREATE TABLE backup_music_files (
|
||||
track_id uuid PRIMARY KEY REFERENCES tracks (id) ON DELETE CASCADE,
|
||||
-- The file as it was sent.
|
||||
size bigint NOT NULL,
|
||||
mtime timestamptz NOT NULL,
|
||||
-- Where it went: the object key, or the file's path in the folder.
|
||||
location text NOT NULL,
|
||||
-- When the file was last sent or found unchanged.
|
||||
checked_at timestamptz NOT NULL DEFAULT now(),
|
||||
-- Empty when it was sent; otherwise why the destination refused it.
|
||||
error text NOT NULL DEFAULT ''
|
||||
);
|
||||
@@ -0,0 +1,2 @@
|
||||
DROP TABLE IF EXISTS library_changes_trimmed;
|
||||
DROP INDEX IF EXISTS idx_library_changes_entity;
|
||||
@@ -0,0 +1,56 @@
|
||||
-- 0101_library_changes_keep_latest.up.sql — the change feed keeps every
|
||||
-- entity's latest change, so an app syncing from nothing gets the whole
|
||||
-- library.
|
||||
--
|
||||
-- Until now the compactor dropped every row older than 30 days. An app
|
||||
-- syncing from cursor 0 (a fresh install, cleared data, or a reset after a
|
||||
-- 410) builds its library from the feed, so it got only what had changed in
|
||||
-- the last 30 days. From here the compactor drops a row only once a newer row
|
||||
-- for the same entity exists, and drops a delete only once it is 30 days old.
|
||||
--
|
||||
-- idx_library_changes_entity finds an entity's rows, newest first, for
|
||||
-- the compactor and the fill below.
|
||||
-- library_changes_trimmed one row: the highest id of a delete the
|
||||
-- compactor has dropped. An app whose cursor
|
||||
-- is below it may have missed that delete, so
|
||||
-- it is told to start over (410). It starts
|
||||
-- where the old compactor left off: everything
|
||||
-- below the oldest remaining row is gone.
|
||||
--
|
||||
-- Then every artist, album and track whose latest row is not an upsert, or
|
||||
-- which has no row at all (the old compactor dropped it), gets one, artists
|
||||
-- first so an album never arrives before its artist. Likes and playlists are
|
||||
-- left as they are: the apps fetch those from their own endpoints.
|
||||
CREATE INDEX idx_library_changes_entity ON library_changes (entity_type, entity_id, id DESC);
|
||||
|
||||
CREATE TABLE library_changes_trimmed (
|
||||
id boolean PRIMARY KEY DEFAULT true CONSTRAINT library_changes_trimmed_one_row CHECK (id),
|
||||
through bigint NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
INSERT INTO library_changes_trimmed (through)
|
||||
SELECT COALESCE(min(id), 1) - 1 FROM library_changes;
|
||||
|
||||
INSERT INTO library_changes (entity_type, entity_id, op)
|
||||
SELECT 'artist', a.id::text, 'upsert'
|
||||
FROM artists a
|
||||
WHERE COALESCE((SELECT c.op FROM library_changes c
|
||||
WHERE c.entity_type = 'artist' AND c.entity_id = a.id::text
|
||||
ORDER BY c.id DESC LIMIT 1), 'delete') <> 'upsert'
|
||||
ORDER BY a.id;
|
||||
|
||||
INSERT INTO library_changes (entity_type, entity_id, op)
|
||||
SELECT 'album', al.id::text, 'upsert'
|
||||
FROM albums al
|
||||
WHERE COALESCE((SELECT c.op FROM library_changes c
|
||||
WHERE c.entity_type = 'album' AND c.entity_id = al.id::text
|
||||
ORDER BY c.id DESC LIMIT 1), 'delete') <> 'upsert'
|
||||
ORDER BY al.id;
|
||||
|
||||
INSERT INTO library_changes (entity_type, entity_id, op)
|
||||
SELECT 'track', t.id::text, 'upsert'
|
||||
FROM tracks t
|
||||
WHERE COALESCE((SELECT c.op FROM library_changes c
|
||||
WHERE c.entity_type = 'track' AND c.entity_id = t.id::text
|
||||
ORDER BY c.id DESC LIMIT 1), 'delete') <> 'upsert'
|
||||
ORDER BY t.id;
|
||||
@@ -0,0 +1,37 @@
|
||||
-- Restores the 0028 constraint set. Genre-mix rows would violate it, so they
|
||||
-- go first.
|
||||
DELETE FROM playlists WHERE system_variant = 'genre_mix';
|
||||
|
||||
ALTER TABLE playlists DROP CONSTRAINT IF EXISTS playlists_kind_variant_consistent;
|
||||
ALTER TABLE playlists
|
||||
ADD CONSTRAINT playlists_kind_variant_consistent CHECK (
|
||||
(kind = 'user' AND system_variant IS NULL)
|
||||
OR
|
||||
(kind = 'system' AND system_variant IN (
|
||||
'for_you', 'songs_like_artist', 'discover',
|
||||
'deep_cuts', 'rediscover', 'new_for_you',
|
||||
'on_this_day', 'first_listens'
|
||||
))
|
||||
);
|
||||
|
||||
ALTER TABLE playlists DROP CONSTRAINT IF EXISTS playlists_seed_consistent;
|
||||
ALTER TABLE playlists
|
||||
ADD CONSTRAINT playlists_seed_consistent CHECK (
|
||||
(system_variant IS NULL AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'for_you' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'discover' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'deep_cuts' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'rediscover' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'new_for_you' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'on_this_day' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'first_listens' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'songs_like_artist' AND seed_artist_id IS NOT NULL)
|
||||
);
|
||||
@@ -0,0 +1,43 @@
|
||||
-- Genre mixes: one daily system playlist per genre the user listens to
|
||||
-- ("Post-Punk mix", "Jazz mix", ...). Several per user, like
|
||||
-- songs_like_artist, but seedless: the genre lives in the playlist name and
|
||||
-- seed_artist_id is NULL.
|
||||
--
|
||||
-- Postgres has no ALTER CONSTRAINT for CHECK, so both constraints are
|
||||
-- dropped and re-added with 'genre_mix' in the accepted set.
|
||||
|
||||
ALTER TABLE playlists DROP CONSTRAINT IF EXISTS playlists_kind_variant_consistent;
|
||||
ALTER TABLE playlists
|
||||
ADD CONSTRAINT playlists_kind_variant_consistent CHECK (
|
||||
(kind = 'user' AND system_variant IS NULL)
|
||||
OR
|
||||
(kind = 'system' AND system_variant IN (
|
||||
'for_you', 'songs_like_artist', 'discover',
|
||||
'deep_cuts', 'rediscover', 'new_for_you',
|
||||
'on_this_day', 'first_listens', 'genre_mix'
|
||||
))
|
||||
);
|
||||
|
||||
ALTER TABLE playlists DROP CONSTRAINT IF EXISTS playlists_seed_consistent;
|
||||
ALTER TABLE playlists
|
||||
ADD CONSTRAINT playlists_seed_consistent CHECK (
|
||||
(system_variant IS NULL AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'for_you' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'discover' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'deep_cuts' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'rediscover' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'new_for_you' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'on_this_day' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'first_listens' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'genre_mix' AND seed_artist_id IS NULL)
|
||||
OR
|
||||
(system_variant = 'songs_like_artist' AND seed_artist_id IS NOT NULL)
|
||||
);
|
||||
@@ -0,0 +1,110 @@
|
||||
-- Minstrel's own backup: its settings and its runs.
|
||||
|
||||
-- name: GetBackupSettings :one
|
||||
SELECT * FROM backup_settings WHERE id = true;
|
||||
|
||||
-- name: SaveBackupSettings :exec
|
||||
UPDATE backup_settings
|
||||
SET enabled = $1,
|
||||
destination = $2,
|
||||
folder = $3,
|
||||
endpoint = $4,
|
||||
bucket = $5,
|
||||
access_key = $6,
|
||||
secret_key = $7,
|
||||
run_hour = $8,
|
||||
upload_limit_kib = $9,
|
||||
include_music = $10,
|
||||
updated_at = now()
|
||||
WHERE id = true;
|
||||
|
||||
-- name: StartBackupRun :one
|
||||
INSERT INTO backup_runs (trigger) VALUES ($1)
|
||||
RETURNING id, started_at;
|
||||
|
||||
-- name: FinishBackupRun :exec
|
||||
UPDATE backup_runs
|
||||
SET status = $2,
|
||||
finished_at = now(),
|
||||
location = $3,
|
||||
bytes = $4,
|
||||
error = $5,
|
||||
music_files = $6,
|
||||
music_bytes = $7,
|
||||
music_note = $8
|
||||
WHERE id = $1;
|
||||
|
||||
-- name: ListBackupRuns :many
|
||||
SELECT id, trigger, status, started_at, finished_at, location, bytes, error,
|
||||
music_files, music_bytes, music_note
|
||||
FROM backup_runs
|
||||
ORDER BY started_at DESC
|
||||
LIMIT $1;
|
||||
|
||||
-- name: BackupRunTimes :one
|
||||
-- The times the verdict turns on. A run that never happened is the epoch,
|
||||
-- so the row always exists. failing_since is the first failure after the last
|
||||
-- success, and the epoch when the latest finished run succeeded.
|
||||
SELECT
|
||||
COALESCE((SELECT max(started_at) FROM backup_runs WHERE status = 'succeeded'),
|
||||
'epoch'::timestamptz)::timestamptz AS last_success,
|
||||
COALESCE((SELECT max(started_at) FROM backup_runs),
|
||||
'epoch'::timestamptz)::timestamptz AS last_attempt,
|
||||
COALESCE((SELECT min(started_at) FROM backup_runs
|
||||
WHERE status = 'failed'
|
||||
AND started_at > COALESCE((SELECT max(started_at) FROM backup_runs WHERE status = 'succeeded'),
|
||||
'epoch'::timestamptz)),
|
||||
'epoch'::timestamptz)::timestamptz AS failing_since;
|
||||
|
||||
-- name: FailUnfinishedBackupRuns :execrows
|
||||
UPDATE backup_runs
|
||||
SET status = 'failed', finished_at = now(),
|
||||
error = 'The server stopped while the backup was running.'
|
||||
WHERE status = 'running';
|
||||
|
||||
-- name: PruneBackupRuns :execrows
|
||||
DELETE FROM backup_runs WHERE status <> 'running' AND started_at < $1;
|
||||
|
||||
-- name: PendingBackupMusic :many
|
||||
-- Library files the backup may not hold as they are now, in path order from
|
||||
-- after_path, so a night that runs out of time carries on where it stopped:
|
||||
-- never sent, a different size, touched by a scan since they were last
|
||||
-- checked, or refused before the settings were last saved (a fixed key gets
|
||||
-- another go). The caller compares the file's own size and mtime before
|
||||
-- sending.
|
||||
SELECT t.id, t.file_path, t.file_size,
|
||||
m.size AS sent_size, m.mtime AS sent_mtime, m.error AS sent_error
|
||||
FROM tracks t
|
||||
LEFT JOIN backup_music_files m ON m.track_id = t.id
|
||||
WHERE t.missing_since IS NULL
|
||||
AND t.file_path > sqlc.arg(after_path)::text
|
||||
AND (m.track_id IS NULL OR m.size <> t.file_size OR t.updated_at > m.checked_at
|
||||
OR (m.error <> '' AND m.checked_at < (SELECT updated_at FROM backup_settings)))
|
||||
ORDER BY t.file_path
|
||||
LIMIT sqlc.arg(max_rows);
|
||||
|
||||
-- name: RecordBackupMusic :exec
|
||||
INSERT INTO backup_music_files (track_id, size, mtime, location, error)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (track_id) DO UPDATE
|
||||
SET size = EXCLUDED.size, mtime = EXCLUDED.mtime, location = EXCLUDED.location,
|
||||
error = EXCLUDED.error, checked_at = now();
|
||||
|
||||
-- name: BackupMusicProgress :one
|
||||
-- How much of the library the backup holds as it is now.
|
||||
SELECT count(*)::bigint AS total_files,
|
||||
COALESCE(sum(t.file_size), 0)::bigint AS total_bytes,
|
||||
count(*) FILTER (WHERE m.error = '' AND m.size = t.file_size AND t.updated_at <= m.checked_at)::bigint AS sent_files,
|
||||
COALESCE(sum(t.file_size) FILTER (WHERE m.error = '' AND m.size = t.file_size AND t.updated_at <= m.checked_at), 0)::bigint AS sent_bytes,
|
||||
count(*) FILTER (WHERE m.error <> '' AND m.size = t.file_size AND t.updated_at <= m.checked_at)::bigint AS refused_files
|
||||
FROM tracks t
|
||||
LEFT JOIN backup_music_files m ON m.track_id = t.id
|
||||
WHERE t.missing_since IS NULL;
|
||||
|
||||
-- name: RefusedBackupMusic :many
|
||||
SELECT t.file_path, m.error, m.checked_at
|
||||
FROM backup_music_files m
|
||||
JOIN tracks t ON t.id = m.track_id
|
||||
WHERE m.error <> '' AND m.size = t.file_size AND t.updated_at <= m.checked_at
|
||||
ORDER BY m.checked_at DESC
|
||||
LIMIT $1;
|
||||
@@ -0,0 +1,193 @@
|
||||
-- Candidate queries for the genre mixes: one daily system playlist per genre
|
||||
-- the user listens to. Genres live inline on tracks.genre as a delimited
|
||||
-- string and are split on [;,] the same way the Discover and radio tag arms
|
||||
-- split them. Matching is case-insensitive, so the ID3 "Post-Punk" and the
|
||||
-- folksonomy "post-punk" in the taste profile count as one genre.
|
||||
|
||||
-- name: ListGenreMixCandidates :many
|
||||
-- Library genres big enough to carry a mix (>= 25 playable tracks across
|
||||
-- >= 3 artists), each tagged with how close it is to the user's listening:
|
||||
--
|
||||
-- tier 1: a positive taste-profile genre, scored by its taste weight
|
||||
-- tier 2: a genre of tracks the user has played through or liked, scored
|
||||
-- by how many — catches a profile that has not been built yet or
|
||||
-- has decayed past its window
|
||||
-- tier 3: any other library genre, scored by its size
|
||||
--
|
||||
-- track_ids lets the caller tell how much music two genres share: a genre
|
||||
-- whose songs mostly sit in genres already picked adds no new channel, and a
|
||||
-- broad genre ("Rock") whose songs mostly carry narrower ones ("Pop Punk",
|
||||
-- "Ska Punk") gives way to them.
|
||||
-- $1 user_id.
|
||||
WITH library_genres AS (
|
||||
SELECT lower(trim(g.g)) AS genre_key,
|
||||
min(trim(g.g))::text AS label,
|
||||
COUNT(DISTINCT t.id) AS track_count,
|
||||
array_agg(DISTINCT t.id)::uuid[] AS track_ids
|
||||
FROM tracks t
|
||||
CROSS JOIN LATERAL regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g(g)
|
||||
WHERE t.missing_since IS NULL
|
||||
AND trim(g.g) <> ''
|
||||
GROUP BY lower(trim(g.g))
|
||||
HAVING COUNT(DISTINCT t.id) >= 25
|
||||
AND COUNT(DISTINCT t.artist_id) >= 3
|
||||
),
|
||||
taste AS (
|
||||
SELECT lower(trim(tpt.tag)) AS genre_key, SUM(tpt.weight) AS weight
|
||||
FROM taste_profile_tags tpt
|
||||
WHERE tpt.user_id = $1
|
||||
GROUP BY lower(trim(tpt.tag))
|
||||
HAVING SUM(tpt.weight) > 0
|
||||
),
|
||||
listened AS (
|
||||
SELECT t.id
|
||||
FROM play_events pe
|
||||
JOIN tracks t ON t.id = pe.track_id
|
||||
WHERE pe.user_id = $1 AND pe.was_skipped = false
|
||||
UNION ALL
|
||||
SELECT gl.track_id
|
||||
FROM general_likes gl
|
||||
WHERE gl.user_id = $1
|
||||
),
|
||||
history AS (
|
||||
SELECT lower(trim(g.g)) AS genre_key, COUNT(*) AS plays
|
||||
FROM listened l
|
||||
JOIN tracks t ON t.id = l.id
|
||||
CROSS JOIN LATERAL regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g(g)
|
||||
WHERE trim(g.g) <> ''
|
||||
GROUP BY lower(trim(g.g))
|
||||
)
|
||||
SELECT lg.genre_key::text AS genre_key,
|
||||
lg.label AS label,
|
||||
(CASE WHEN ta.genre_key IS NOT NULL THEN 1
|
||||
WHEN h.genre_key IS NOT NULL THEN 2
|
||||
ELSE 3 END)::int AS tier,
|
||||
(CASE WHEN ta.genre_key IS NOT NULL THEN ta.weight
|
||||
WHEN h.genre_key IS NOT NULL THEN h.plays::float8
|
||||
ELSE lg.track_count::float8 END)::float8 AS score,
|
||||
lg.track_ids
|
||||
FROM library_genres lg
|
||||
LEFT JOIN taste ta ON ta.genre_key = lg.genre_key
|
||||
LEFT JOIN history h ON h.genre_key = lg.genre_key
|
||||
ORDER BY tier, score DESC, lg.genre_key
|
||||
LIMIT 60;
|
||||
|
||||
-- name: ListGenreMixTracks :many
|
||||
-- Playable tracks carrying one genre, split into the two halves a genre mix
|
||||
-- blends: familiar (the user has played it through or liked it) and unheard.
|
||||
-- Up to 200 of each, drawn by a weighted lottery seeded with the date, so the
|
||||
-- mix changes day to day, holds still within a day, and leans toward what the
|
||||
-- user likes without becoming the same 50 songs every day.
|
||||
--
|
||||
-- The lottery is the Efraimidis-Spirakis draw: each track gets
|
||||
-- -ln(u) / weight with u uniform in (0, 1) from md5(track, date), and the
|
||||
-- smallest keys win. A track of weight 3 is three times as likely as one of
|
||||
-- weight 1 to come before it, and every track keeps some chance.
|
||||
--
|
||||
-- Familiar weight: 1 + ln(1 + plays through) + 1 if liked, times the share of
|
||||
-- its plays that were not skips. A song played 40 times weighs about 4.7, one
|
||||
-- played once about 1.7.
|
||||
--
|
||||
-- Unheard weight: 1 + ln(1 + the artist's plays through + 2 × its liked
|
||||
-- tracks) + 2 × how similar the artist is to the user's own artists (the best
|
||||
-- artist_similarity score from any artist they have played through or liked,
|
||||
-- 0 to 1), times the share of the artist's plays that were not skips. An
|
||||
-- artist the user has never met weighs 1, so new artists still come up; an
|
||||
-- artist they keep skipping weighs well under 1.
|
||||
--
|
||||
-- A track the user keeps skipping stays out: twice-skipped and skipped more
|
||||
-- often than played through. The second half of that test keeps a favourite
|
||||
-- with a couple of stray skips in.
|
||||
--
|
||||
-- exclude_genres keeps each song in one mix: a track carrying a genre that
|
||||
-- already has a mix today is left to that mix.
|
||||
WITH play_counts AS (
|
||||
SELECT pe.track_id, COUNT(*) AS c
|
||||
FROM play_events pe
|
||||
WHERE pe.user_id = sqlc.arg(user_id) AND pe.was_skipped = false
|
||||
GROUP BY pe.track_id
|
||||
),
|
||||
skip_counts AS (
|
||||
SELECT pe.track_id, COUNT(*) AS c
|
||||
FROM play_events pe
|
||||
WHERE pe.user_id = sqlc.arg(user_id) AND pe.was_skipped = true
|
||||
GROUP BY pe.track_id
|
||||
),
|
||||
artist_plays AS (
|
||||
SELECT t.artist_id,
|
||||
COUNT(*) FILTER (WHERE NOT pe.was_skipped) AS plays,
|
||||
COUNT(*) FILTER (WHERE pe.was_skipped) AS skips
|
||||
FROM play_events pe
|
||||
JOIN tracks t ON t.id = pe.track_id
|
||||
WHERE pe.user_id = sqlc.arg(user_id)
|
||||
GROUP BY t.artist_id
|
||||
),
|
||||
artist_likes AS (
|
||||
SELECT t.artist_id, COUNT(*) AS likes
|
||||
FROM general_likes gl
|
||||
JOIN tracks t ON t.id = gl.track_id
|
||||
WHERE gl.user_id = sqlc.arg(user_id)
|
||||
GROUP BY t.artist_id
|
||||
),
|
||||
own_artists AS (
|
||||
SELECT ap.artist_id FROM artist_plays ap WHERE ap.plays > 0
|
||||
UNION
|
||||
SELECT al.artist_id FROM artist_likes al
|
||||
),
|
||||
artist_sim AS (
|
||||
SELECT asim.artist_b_id AS artist_id,
|
||||
MAX(LEAST(GREATEST(asim.score, 0), 1)) AS sim
|
||||
FROM artist_similarity asim
|
||||
JOIN own_artists oa ON oa.artist_id = asim.artist_a_id
|
||||
GROUP BY asim.artist_b_id
|
||||
),
|
||||
eligible AS (
|
||||
SELECT t.id, t.album_id, t.artist_id,
|
||||
(COALESCE(pc.c, 0) > 0 OR gl.track_id IS NOT NULL) AS familiar,
|
||||
CASE
|
||||
WHEN COALESCE(pc.c, 0) > 0 OR gl.track_id IS NOT NULL THEN
|
||||
(1 + ln(1 + COALESCE(pc.c, 0)::float8)
|
||||
+ CASE WHEN gl.track_id IS NOT NULL THEN 1 ELSE 0 END)
|
||||
* ((COALESCE(pc.c, 0) + 1)::float8 / (COALESCE(pc.c, 0) + COALESCE(sc.c, 0) + 1))
|
||||
ELSE
|
||||
(1 + ln(1 + COALESCE(ap.plays, 0)::float8 + 2 * COALESCE(al.likes, 0)::float8)
|
||||
+ 2 * COALESCE(asm.sim, 0))
|
||||
* ((COALESCE(ap.plays, 0) + 1)::float8 / (COALESCE(ap.plays, 0) + COALESCE(ap.skips, 0) + 1))
|
||||
END::float8 AS weight,
|
||||
-- u in (0, 1): the first 32 bits of md5(track, date), shifted off 0.
|
||||
((('x' || substr(md5(t.id::text || sqlc.arg(date_seed)::text), 1, 8))::bit(32)::bigint)::float8 + 1)
|
||||
/ 4294967297.0 AS u
|
||||
FROM tracks t
|
||||
LEFT JOIN play_counts pc ON pc.track_id = t.id
|
||||
LEFT JOIN skip_counts sc ON sc.track_id = t.id
|
||||
LEFT JOIN general_likes gl ON gl.user_id = sqlc.arg(user_id) AND gl.track_id = t.id
|
||||
LEFT JOIN artist_plays ap ON ap.artist_id = t.artist_id
|
||||
LEFT JOIN artist_likes al ON al.artist_id = t.artist_id
|
||||
LEFT JOIN artist_sim asm ON asm.artist_id = t.artist_id
|
||||
WHERE t.missing_since IS NULL -- never offer a file that is gone
|
||||
AND NOT (COALESCE(sc.c, 0) >= 2 AND COALESCE(sc.c, 0) > COALESCE(pc.c, 0))
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g(g)
|
||||
WHERE lower(trim(g.g)) = sqlc.arg(genre_key)::text
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g(g)
|
||||
WHERE lower(trim(g.g)) = ANY(sqlc.arg(exclude_genres)::text[])
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM quarantine q
|
||||
WHERE q.user_id = sqlc.arg(user_id) AND q.track_id = t.id
|
||||
)
|
||||
),
|
||||
ranked AS (
|
||||
SELECT e.id, e.album_id, e.artist_id, e.familiar,
|
||||
-ln(e.u) / e.weight AS draw,
|
||||
row_number() OVER (PARTITION BY e.familiar ORDER BY -ln(e.u) / e.weight, e.id) AS rn
|
||||
FROM eligible e
|
||||
)
|
||||
SELECT r.id, r.album_id, r.artist_id, r.familiar::bool AS familiar
|
||||
FROM ranked r
|
||||
WHERE r.rn <= 200
|
||||
ORDER BY r.familiar DESC, r.draw, r.id;
|
||||
@@ -22,11 +22,34 @@ LIMIT $2;
|
||||
-- name: GetMaxLibraryChangeCursor :one
|
||||
SELECT COALESCE(MAX(id), 0)::BIGINT FROM library_changes;
|
||||
|
||||
-- name: GetMinLibraryChangeCursor :one
|
||||
SELECT COALESCE(MIN(id), 0)::BIGINT FROM library_changes;
|
||||
-- name: GetLastIssuedLibraryChangeCursor :one
|
||||
-- The highest cursor this server has handed out, even once the compactor has
|
||||
-- trimmed the rows: the id sequence's last value. A restored server's
|
||||
-- sequence stands at the backup's last change.
|
||||
SELECT COALESCE(pg_sequence_last_value(pg_get_serial_sequence('library_changes', 'id')::regclass), 0)::BIGINT;
|
||||
|
||||
-- name: DeleteOldLibraryChanges :execrows
|
||||
-- Removes library_changes rows older than the given cutoff. Run from
|
||||
-- the sync compactor goroutine on a daily tick. Returns the number of
|
||||
-- rows deleted so the worker can log meaningful progress.
|
||||
DELETE FROM library_changes WHERE changed_at < $1;
|
||||
-- name: GetLibraryChangesTrimmedThrough :one
|
||||
-- The highest id of a delete the compactor has dropped. A cursor below it may
|
||||
-- have missed that delete.
|
||||
SELECT through FROM library_changes_trimmed;
|
||||
|
||||
-- name: DeleteSupersededLibraryChanges :execrows
|
||||
-- Removes every row a newer row for the same entity has superseded. An app
|
||||
-- reading from any cursor still meets the newer row, so it loses nothing.
|
||||
DELETE FROM library_changes old
|
||||
WHERE EXISTS (SELECT 1 FROM library_changes newer
|
||||
WHERE newer.entity_type = old.entity_type
|
||||
AND newer.entity_id = old.entity_id
|
||||
AND newer.id > old.id);
|
||||
|
||||
-- name: DeleteOldLibraryDeletes :one
|
||||
-- Removes deletes older than the cutoff and raises the trimmed mark past the
|
||||
-- highest one, so an app whose cursor is below it starts over. Returns how
|
||||
-- many went.
|
||||
WITH gone AS (
|
||||
DELETE FROM library_changes WHERE op = 'delete' AND changed_at < $1 RETURNING id
|
||||
), mark AS (
|
||||
UPDATE library_changes_trimmed
|
||||
SET through = GREATEST(through, (SELECT COALESCE(max(id), 0) FROM gone))
|
||||
)
|
||||
SELECT count(*)::BIGINT FROM gone;
|
||||
|
||||
@@ -184,6 +184,21 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) {
|
||||
); err != nil {
|
||||
t.Fatalf("dbtest.ResetDB reset release-type profiles: %v", err)
|
||||
}
|
||||
// The change log's trimmed mark, a singleton: nothing has been trimmed.
|
||||
if _, err := pool.Exec(ctx, "UPDATE library_changes_trimmed SET through = 0"); err != nil {
|
||||
t.Fatalf("dbtest.ResetDB reset library_changes_trimmed: %v", err)
|
||||
}
|
||||
// The server's own backup: no runs, and settings back to off.
|
||||
if _, err := pool.Exec(ctx, `
|
||||
DELETE FROM backup_runs;
|
||||
DELETE FROM backup_music_files;
|
||||
UPDATE backup_settings
|
||||
SET enabled = DEFAULT, destination = DEFAULT, folder = DEFAULT, endpoint = DEFAULT,
|
||||
bucket = DEFAULT, access_key = DEFAULT, secret_key = DEFAULT, run_hour = DEFAULT,
|
||||
upload_limit_kib = DEFAULT, include_music = DEFAULT, updated_at = DEFAULT`,
|
||||
); err != nil {
|
||||
t.Fatalf("dbtest.ResetDB reset backup settings: %v", err)
|
||||
}
|
||||
// Search and grab settings, a singleton reset the same way.
|
||||
// The library listing reads as done, so a test's albums are not
|
||||
// put on the wanted list behind its back; its own tests clear it.
|
||||
|
||||
@@ -93,6 +93,7 @@ var systemPlaylistSources = map[string]bool{
|
||||
"on_this_day": true,
|
||||
"first_listens": true,
|
||||
"songs_like_artist": true,
|
||||
"genre_mix": true,
|
||||
}
|
||||
|
||||
// lookupSystemPickKind resolves a system-playlist play's pick_kind
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
package playlists
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
)
|
||||
|
||||
// Genre mixes: one daily playlist per genre the user listens to, shown on
|
||||
// their own Home row. Each mix is a blend of the user's familiar tracks in
|
||||
// that genre and tracks in it they have not heard yet.
|
||||
//
|
||||
// Which genres: ListGenreMixCandidates ranks library genres in three tiers
|
||||
// (taste profile → listening history → the rest of the library). Tier 3
|
||||
// only fills a shelf the user's own history cannot: a user with a handful of
|
||||
// listened genres gets those, topped up to genreMixMinShelf, rather than
|
||||
// ten channels of music they never play.
|
||||
//
|
||||
// The shelf is a set of channels that sound different from one another. Tags
|
||||
// overlap heavily (a lo-fi beat is often tagged Lo-Fi, Hip Hop, Chillwave and
|
||||
// Instrumental at once), so the pick is made on songs, not names: a genre
|
||||
// whose songs mostly sit in a genre already picked adds no new channel, and a
|
||||
// broad genre whose songs mostly carry narrower ones gives way to them, so
|
||||
// the shelf holds Pop Punk and Ska Punk rather than one Rock mix. Each song
|
||||
// belongs to at most one mix.
|
||||
//
|
||||
// The set of genres is stable from day to day, like a list of channels; the
|
||||
// tracks inside each mix change daily. ListGenreMixTracks draws them by a
|
||||
// date-seeded weighted lottery: familiar tracks lean toward the ones played
|
||||
// most and liked, unheard ones toward the user's artists and artists similar
|
||||
// to them, and every track keeps some chance.
|
||||
|
||||
const (
|
||||
// genreMixCount is the most genre mixes a user gets.
|
||||
genreMixCount = 10
|
||||
// genreMixMinShelf is how many mixes tier-3 genres may fill up to when
|
||||
// the user's history yields fewer.
|
||||
genreMixMinShelf = 4
|
||||
// genreMixLen is each mix's length.
|
||||
genreMixLen = 50
|
||||
// genreMixMinLen withholds a genre whose eligible tracks can't make a
|
||||
// real mix; the next candidate genre takes its place.
|
||||
genreMixMinLen = 15
|
||||
// A mix is genreMixFamiliarParts in every genreMixParts tracks the user
|
||||
// already knows (3 in 5); the rest is unheard tracks in the genre.
|
||||
// Either side running short is filled from the other.
|
||||
genreMixFamiliarParts = 3
|
||||
genreMixParts = 5
|
||||
// genreMixMinNewShare is the share of a genre's songs that must not be
|
||||
// in a genre already picked for it to get a mix of its own.
|
||||
genreMixMinNewShare = 0.5
|
||||
// A genre is broad when at least genreMixBroadShare of its songs also
|
||||
// carry a clearly narrower candidate genre: one at most
|
||||
// genreMixNarrowerSize times its size with at least half of its songs
|
||||
// inside it. Broad genres are considered after the specific ones of
|
||||
// their tier.
|
||||
genreMixBroadShare = 0.5
|
||||
genreMixNarrowerSize = 0.5
|
||||
// genreMixTier3 is the tier of genres the user has no history with.
|
||||
genreMixTier3 = 3
|
||||
)
|
||||
|
||||
// genreCandidate is one row of ListGenreMixCandidates.
|
||||
type genreCandidate struct {
|
||||
Key string
|
||||
Label string
|
||||
Tier int32
|
||||
TrackIDs []pgtype.UUID
|
||||
}
|
||||
|
||||
// genreTrack is one row of ListGenreMixTracks.
|
||||
type genreTrack struct {
|
||||
discoverTrack
|
||||
Familiar bool
|
||||
}
|
||||
|
||||
// produceGenreMixes builds up to genreMixCount "{Genre} mix" playlists.
|
||||
// The candidate query failing is fatal to the build, like the Songs-like
|
||||
// seed query; a single genre's track query failing skips that genre.
|
||||
func produceGenreMixes(
|
||||
ctx context.Context, q *dbq.Queries, logger *slog.Logger,
|
||||
userID pgtype.UUID, dateStr string, _ time.Time,
|
||||
) ([]builtPlaylist, error) {
|
||||
rows, err := q.ListGenreMixCandidates(ctx, userID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list genre mix candidates: %w", err)
|
||||
}
|
||||
cands := make([]genreCandidate, len(rows))
|
||||
for i, r := range rows {
|
||||
cands[i] = genreCandidate{Key: r.GenreKey, Label: r.Label, Tier: r.Tier, TrackIDs: r.TrackIds}
|
||||
}
|
||||
build := func(c genreCandidate, exclude []string) []rankedCandidate {
|
||||
trackRows, terr := q.ListGenreMixTracks(ctx, dbq.ListGenreMixTracksParams{
|
||||
UserID: userID,
|
||||
GenreKey: c.Key,
|
||||
DateSeed: dateStr,
|
||||
ExcludeGenres: exclude,
|
||||
})
|
||||
if terr != nil {
|
||||
logger.Warn("system playlist: genre mix tracks query failed; skipping genre",
|
||||
"user_id", uuidStringPL(userID), "genre", c.Label, "err", terr)
|
||||
return nil
|
||||
}
|
||||
tracks := make([]genreTrack, len(trackRows))
|
||||
for i, r := range trackRows {
|
||||
tracks[i] = genreTrack{
|
||||
discoverTrack: discoverTrack{ID: r.ID, AlbumID: r.AlbumID, ArtistID: r.ArtistID},
|
||||
Familiar: r.Familiar,
|
||||
}
|
||||
}
|
||||
return composeGenreMix(tracks, genreMixLen)
|
||||
}
|
||||
return selectGenreMixes(cands, build), nil
|
||||
}
|
||||
|
||||
// selectGenreMixes walks the candidates, specific genres before broad ones
|
||||
// within each tier, and keeps the ones that make a real mix of music the
|
||||
// shelf does not have yet. It takes tier-3 genres only while the shelf is
|
||||
// below genreMixMinShelf. build is handed the genres kept so far, whose songs
|
||||
// the new mix leaves out.
|
||||
func selectGenreMixes(
|
||||
cands []genreCandidate, build func(genreCandidate, []string) []rankedCandidate,
|
||||
) []builtPlaylist {
|
||||
sets := make([]map[pgtype.UUID]struct{}, len(cands))
|
||||
for i, c := range cands {
|
||||
sets[i] = make(map[pgtype.UUID]struct{}, len(c.TrackIDs))
|
||||
for _, id := range c.TrackIDs {
|
||||
sets[i][id] = struct{}{}
|
||||
}
|
||||
}
|
||||
owned := map[pgtype.UUID]struct{}{}
|
||||
kept := []string{}
|
||||
var out []builtPlaylist
|
||||
for _, i := range specificFirst(cands, sets) {
|
||||
c := cands[i]
|
||||
if len(out) >= genreMixCount {
|
||||
break
|
||||
}
|
||||
if c.Tier >= genreMixTier3 && len(out) >= genreMixMinShelf {
|
||||
break
|
||||
}
|
||||
if !mostlyNew(sets[i], owned) {
|
||||
continue
|
||||
}
|
||||
tracks := build(c, kept)
|
||||
if len(tracks) < genreMixMinLen {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, c.Key)
|
||||
for id := range sets[i] {
|
||||
owned[id] = struct{}{}
|
||||
}
|
||||
out = append(out, builtPlaylist{
|
||||
Name: c.Label + " mix",
|
||||
Variant: "genre_mix",
|
||||
Tracks: tracks,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// specificFirst returns the candidates' indexes in rank order, except that
|
||||
// within each tier the broad genres come after the specific ones.
|
||||
func specificFirst(cands []genreCandidate, sets []map[pgtype.UUID]struct{}) []int {
|
||||
broad := make([]bool, len(cands))
|
||||
for i := range cands {
|
||||
broad[i] = isBroad(i, sets)
|
||||
}
|
||||
order := make([]int, 0, len(cands))
|
||||
for start := 0; start < len(cands); {
|
||||
end := start
|
||||
for end < len(cands) && cands[end].Tier == cands[start].Tier {
|
||||
end++
|
||||
}
|
||||
for _, wantBroad := range []bool{false, true} {
|
||||
for i := start; i < end; i++ {
|
||||
if broad[i] == wantBroad {
|
||||
order = append(order, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
start = end
|
||||
}
|
||||
return order
|
||||
}
|
||||
|
||||
// isBroad reports whether at least genreMixBroadShare of genre g's songs
|
||||
// also carry a clearly narrower candidate genre.
|
||||
func isBroad(g int, sets []map[pgtype.UUID]struct{}) bool {
|
||||
if len(sets[g]) == 0 {
|
||||
return false
|
||||
}
|
||||
var narrower []map[pgtype.UUID]struct{}
|
||||
for h, set := range sets {
|
||||
if h == g || float64(len(set)) > genreMixNarrowerSize*float64(len(sets[g])) {
|
||||
continue
|
||||
}
|
||||
if 2*sharedCount(set, sets[g]) >= len(set) {
|
||||
narrower = append(narrower, set)
|
||||
}
|
||||
}
|
||||
covered := 0
|
||||
for id := range sets[g] {
|
||||
for _, set := range narrower {
|
||||
if _, ok := set[id]; ok {
|
||||
covered++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return float64(covered) >= genreMixBroadShare*float64(len(sets[g]))
|
||||
}
|
||||
|
||||
// mostlyNew reports whether at least genreMixMinNewShare of set's songs are
|
||||
// not yet owned by a kept genre.
|
||||
func mostlyNew(set, owned map[pgtype.UUID]struct{}) bool {
|
||||
return float64(len(set)-sharedCount(set, owned)) >= genreMixMinNewShare*float64(len(set))
|
||||
}
|
||||
|
||||
// sharedCount counts the songs in both a and b.
|
||||
func sharedCount(a, b map[pgtype.UUID]struct{}) int {
|
||||
if len(b) < len(a) {
|
||||
a, b = b, a
|
||||
}
|
||||
n := 0
|
||||
for id := range a {
|
||||
if _, ok := b[id]; ok {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// composeGenreMix interleaves familiar and unheard tracks 3 to 5
|
||||
// (familiar, familiar, unheard, familiar, unheard, …),
|
||||
// continuing with whichever side remains once the other runs out, then
|
||||
// applies the per-album / per-artist caps and truncates to n. A genre too
|
||||
// narrow for the caps is topped up from the uncapped pool, as the discovery
|
||||
// mixes are. Familiar tracks are stamped pick_kind 'taste' and unheard ones
|
||||
// 'taste_unheard', so skips can be attributed to each half.
|
||||
func composeGenreMix(tracks []genreTrack, n int) []rankedCandidate {
|
||||
var familiar, unheard []discoverTrack
|
||||
for _, t := range tracks {
|
||||
d := t.discoverTrack
|
||||
if t.Familiar {
|
||||
d.PickKind = pickKindTaste
|
||||
familiar = append(familiar, d)
|
||||
} else {
|
||||
d.PickKind = pickKindTasteUnheard
|
||||
unheard = append(unheard, d)
|
||||
}
|
||||
}
|
||||
pool := make([]discoverTrack, 0, len(familiar)+len(unheard))
|
||||
fi, ui := 0, 0
|
||||
for fi < len(familiar) || ui < len(unheard) {
|
||||
takeFamiliar := fi < len(familiar) &&
|
||||
(ui >= len(unheard) || fi*genreMixParts < genreMixFamiliarParts*(fi+ui+1))
|
||||
if takeFamiliar {
|
||||
pool = append(pool, familiar[fi])
|
||||
fi++
|
||||
} else {
|
||||
pool = append(pool, unheard[ui])
|
||||
ui++
|
||||
}
|
||||
}
|
||||
capped := topUpFromRaw(capByAlbumAndArtist(pool), pool, n)
|
||||
if len(capped) > n {
|
||||
capped = capped[:n]
|
||||
}
|
||||
out := make([]rankedCandidate, len(capped))
|
||||
for i, t := range capped {
|
||||
out[i] = rankedCandidate{TrackID: t.ID, PickKind: t.PickKind}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
package playlists_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/playlists"
|
||||
)
|
||||
|
||||
// seedGenre creates artists × perArtist tracks tagged genre, each artist on
|
||||
// its own album, and returns the tracks. seedTrack mints a new artist on
|
||||
// every call (an artist without an mbid has no conflict key), so each
|
||||
// artist's first track creates it and the rest join that track's album.
|
||||
func seedGenre(t *testing.T, pool *pgxpool.Pool, genre string, artists, perArtist int) []dbq.Track {
|
||||
t.Helper()
|
||||
var out []dbq.Track
|
||||
for a := 0; a < artists; a++ {
|
||||
first := seedTrack(t, pool, fmt.Sprintf("%s %d-0", genre, a), fmt.Sprintf("%s Artist %d", genre, a))
|
||||
setGenre(t, pool, first.ID, genre)
|
||||
out = append(out, first)
|
||||
for k := 1; k < perArtist; k++ {
|
||||
tr := seedTrackForArtist(t, pool, fmt.Sprintf("%s %d-%d", genre, a, k), first.AlbumID, first.ArtistID)
|
||||
setGenre(t, pool, tr.ID, genre)
|
||||
out = append(out, tr)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestListGenreMixCandidates_TiersAndSizeFloor: a taste-profile genre is
|
||||
// tier 1 (matched case-insensitively), a genre the user has only played is
|
||||
// tier 2, an untouched one is tier 3, and a genre too small for a mix is not
|
||||
// offered at all.
|
||||
func TestListGenreMixCandidates_TiersAndSizeFloor(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
ctx := context.Background()
|
||||
u := seedUser(t, pool, "gmixcand")
|
||||
now := time.Now().UTC()
|
||||
|
||||
seedGenre(t, pool, "Shoegaze", 6, 5)
|
||||
kraut := seedGenre(t, pool, "Krautrock", 6, 5)
|
||||
seedGenre(t, pool, "Dub", 6, 5)
|
||||
seedGenre(t, pool, "Zydeco", 2, 5) // 10 tracks, 2 artists: below the floor
|
||||
|
||||
seedTasteTag(t, pool, u.ID, "shoegaze", 4)
|
||||
seedPlayEvent(t, pool, u.ID, kraut[0].ID, now.Add(-time.Hour), false)
|
||||
|
||||
rows, err := dbq.New(pool).ListGenreMixCandidates(ctx, u.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("list candidates: %v", err)
|
||||
}
|
||||
want := map[string]int32{"Shoegaze": 1, "Krautrock": 2, "Dub": 3}
|
||||
tier := map[string]int32{}
|
||||
order := []string{}
|
||||
for _, r := range rows {
|
||||
tier[r.Label] = r.Tier
|
||||
if _, mine := want[r.Label]; mine {
|
||||
order = append(order, r.Label)
|
||||
}
|
||||
if r.Label == "Shoegaze" && len(r.TrackIds) != 30 {
|
||||
t.Errorf("Shoegaze track_ids = %d, want 30", len(r.TrackIds))
|
||||
}
|
||||
}
|
||||
for label, w := range want {
|
||||
if got, ok := tier[label]; !ok || got != w {
|
||||
t.Errorf("%s tier = %d (present %v), want %d", label, got, ok, w)
|
||||
}
|
||||
}
|
||||
if _, ok := tier["Zydeco"]; ok {
|
||||
t.Error("Zydeco (10 tracks, 2 artists) offered as a genre mix")
|
||||
}
|
||||
if strings.Join(order, ",") != "Shoegaze,Krautrock,Dub" {
|
||||
t.Errorf("order = %v, want tier order Shoegaze, Krautrock, Dub", order)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListGenreMixTracks_LeavesOutSongsOfGenresAlreadyPicked: a track that
|
||||
// also carries an excluded genre (matched case-insensitively) belongs to that
|
||||
// genre's mix and is left out of this one.
|
||||
func TestListGenreMixTracks_LeavesOutSongsOfGenresAlreadyPicked(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
ctx := context.Background()
|
||||
u := seedUser(t, pool, "gmixexcl")
|
||||
|
||||
punk := seedGenre(t, pool, "Ska Punk", 4, 5)
|
||||
shared := punk[:5]
|
||||
for _, tr := range shared {
|
||||
setGenre(t, pool, tr.ID, "Ska Punk; Pop Punk")
|
||||
}
|
||||
|
||||
rows, err := dbq.New(pool).ListGenreMixTracks(ctx, dbq.ListGenreMixTracksParams{
|
||||
UserID: u.ID, DateSeed: "2026-11-07", GenreKey: "ska punk", ExcludeGenres: []string{"pop punk"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list tracks: %v", err)
|
||||
}
|
||||
if len(rows) != len(punk)-len(shared) {
|
||||
t.Fatalf("rows = %d, want %d (the Ska Punk tracks not also Pop Punk)", len(rows), len(punk)-len(shared))
|
||||
}
|
||||
got := map[pgtype.UUID]bool{}
|
||||
for _, r := range rows {
|
||||
got[r.ID] = true
|
||||
}
|
||||
for _, tr := range shared {
|
||||
if got[tr.ID] {
|
||||
t.Fatal("a track also tagged Pop Punk was offered after Pop Punk was excluded")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestListGenreMixTracks_SplitsFamiliarAndDropsRejectedTracks: played and
|
||||
// liked tracks come back familiar, the rest unheard; a track skipped more
|
||||
// often than played through is left out, while a favourite with stray skips
|
||||
// stays; tracks of other genres never appear.
|
||||
func TestListGenreMixTracks_SplitsFamiliarAndDropsRejectedTracks(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
ctx := context.Background()
|
||||
u := seedUser(t, pool, "gmixtracks")
|
||||
now := time.Now().UTC()
|
||||
|
||||
shoegaze := seedGenre(t, pool, "Shoegaze", 6, 5)
|
||||
other := seedGenre(t, pool, "Dub", 3, 2)
|
||||
played, liked, rejected, favourite := shoegaze[0], shoegaze[1], shoegaze[2], shoegaze[3]
|
||||
|
||||
seedPlayEvent(t, pool, u.ID, played.ID, now.Add(-time.Hour), false)
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO general_likes (user_id, track_id) VALUES ($1, $2)`, u.ID, liked.ID); err != nil {
|
||||
t.Fatalf("like: %v", err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
seedPlayEvent(t, pool, u.ID, rejected.ID, now.Add(-time.Duration(i+2)*time.Hour), true)
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
seedPlayEvent(t, pool, u.ID, favourite.ID, now.Add(-time.Duration(i+10)*time.Hour), false)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
seedPlayEvent(t, pool, u.ID, favourite.ID, now.Add(-time.Duration(i+20)*time.Hour), true)
|
||||
}
|
||||
|
||||
rows, err := dbq.New(pool).ListGenreMixTracks(ctx, dbq.ListGenreMixTracksParams{
|
||||
UserID: u.ID, DateSeed: "2026-11-05", GenreKey: "shoegaze",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list tracks: %v", err)
|
||||
}
|
||||
familiar := map[pgtype.UUID]bool{}
|
||||
for _, r := range rows {
|
||||
familiar[r.ID] = r.Familiar
|
||||
}
|
||||
if len(rows) != len(shoegaze)-1 {
|
||||
t.Errorf("rows = %d, want %d (every Shoegaze track but the rejected one)", len(rows), len(shoegaze)-1)
|
||||
}
|
||||
for _, tr := range other {
|
||||
if _, in := familiar[tr.ID]; in {
|
||||
t.Fatalf("a Dub track appeared in the Shoegaze pool")
|
||||
}
|
||||
}
|
||||
if _, in := familiar[rejected.ID]; in {
|
||||
t.Error("a track skipped twice and never played through was offered")
|
||||
}
|
||||
for name, tr := range map[string]dbq.Track{"played": played, "liked": liked, "favourite": favourite} {
|
||||
if f, in := familiar[tr.ID]; !in || !f {
|
||||
t.Errorf("%s track: present %v familiar %v, want present and familiar", name, in, f)
|
||||
}
|
||||
}
|
||||
if f := familiar[shoegaze[4].ID]; f {
|
||||
t.Error("an untouched track came back familiar")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildSystemPlaylists_GenreMixes: a full build gives the user a mix for
|
||||
// the genre they listen to, holding only that genre's tracks, with both
|
||||
// halves of the blend stamped.
|
||||
func TestBuildSystemPlaylists_GenreMixes(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
ctx := context.Background()
|
||||
u := seedUser(t, pool, "gmixbuild")
|
||||
now := time.Now().UTC()
|
||||
|
||||
shoegaze := seedGenre(t, pool, "Shoegaze", 8, 5)
|
||||
seedGenre(t, pool, "Dub", 6, 5)
|
||||
seedTasteTag(t, pool, u.ID, "Shoegaze", 5)
|
||||
for i, tr := range shoegaze[:12] {
|
||||
seedPlayEvent(t, pool, u.ID, tr.ID, now.Add(-time.Duration(i+1)*time.Hour), false)
|
||||
}
|
||||
|
||||
if err := playlists.BuildSystemPlaylists(ctx, pool, discardLogger(), u.ID, now, t.TempDir()); err != nil {
|
||||
t.Fatalf("build: %v", err)
|
||||
}
|
||||
|
||||
var plID pgtype.UUID
|
||||
if err := pool.QueryRow(ctx, `
|
||||
SELECT id FROM playlists
|
||||
WHERE user_id = $1 AND system_variant = 'genre_mix' AND name = 'Shoegaze mix'`,
|
||||
u.ID).Scan(&plID); err != nil {
|
||||
t.Fatalf("no Shoegaze mix built: %v", err)
|
||||
}
|
||||
|
||||
inGenre := map[pgtype.UUID]bool{}
|
||||
for _, tr := range shoegaze {
|
||||
inGenre[tr.ID] = true
|
||||
}
|
||||
rows, err := pool.Query(ctx, `SELECT track_id, pick_kind FROM playlist_tracks WHERE playlist_id = $1`, plID)
|
||||
if err != nil {
|
||||
t.Fatalf("list mix tracks: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
kinds := map[string]int{}
|
||||
n := 0
|
||||
for rows.Next() {
|
||||
var id pgtype.UUID
|
||||
var kind *string
|
||||
if err := rows.Scan(&id, &kind); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
n++
|
||||
if !inGenre[id] {
|
||||
t.Errorf("track %s in the Shoegaze mix is not a Shoegaze track", uuidString(id))
|
||||
}
|
||||
if kind != nil {
|
||||
kinds[*kind]++
|
||||
}
|
||||
}
|
||||
if n < 15 {
|
||||
t.Fatalf("Shoegaze mix holds %d tracks, want at least 15", n)
|
||||
}
|
||||
if kinds["taste"] == 0 || kinds["taste_unheard"] == 0 {
|
||||
t.Errorf("pick kinds = %v, want both taste and taste_unheard", kinds)
|
||||
}
|
||||
}
|
||||
|
||||
// genreDrawPositions runs ListGenreMixTracks for ten different days and
|
||||
// returns, per track, the sum of its positions within its own half
|
||||
// (familiar or unheard). The draw is random per day, so a lean shows up as
|
||||
// an earlier average position across days, not in any single one.
|
||||
func genreDrawPositions(t *testing.T, pool *pgxpool.Pool, userID pgtype.UUID, genreKey string) map[pgtype.UUID]int {
|
||||
t.Helper()
|
||||
sum := map[pgtype.UUID]int{}
|
||||
for day := 1; day <= 10; day++ {
|
||||
rows, err := dbq.New(pool).ListGenreMixTracks(context.Background(), dbq.ListGenreMixTracksParams{
|
||||
UserID: userID, DateSeed: fmt.Sprintf("2026-11-%02d", day), GenreKey: genreKey,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list tracks: %v", err)
|
||||
}
|
||||
pos := map[bool]int{}
|
||||
for _, r := range rows {
|
||||
sum[r.ID] += pos[r.Familiar]
|
||||
pos[r.Familiar]++
|
||||
}
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
func meanPosition(sum map[pgtype.UUID]int, tracks []dbq.Track) float64 {
|
||||
total := 0
|
||||
for _, tr := range tracks {
|
||||
total += sum[tr.ID]
|
||||
}
|
||||
return float64(total) / float64(len(tracks))
|
||||
}
|
||||
|
||||
// TestListGenreMixTracks_FamiliarLeansTowardFavourites: among the tracks the
|
||||
// user knows, ones played often and liked come up earlier than ones played
|
||||
// once.
|
||||
func TestListGenreMixTracks_FamiliarLeansTowardFavourites(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
ctx := context.Background()
|
||||
u := seedUser(t, pool, "gmixfav")
|
||||
now := time.Now().UTC()
|
||||
|
||||
tracks := seedGenre(t, pool, "Krautrock", 4, 5)
|
||||
favourites, onceOnly := tracks[:10], tracks[10:]
|
||||
for i, tr := range favourites {
|
||||
for k := 0; k < 5; k++ {
|
||||
seedPlayEvent(t, pool, u.ID, tr.ID, now.Add(-time.Duration(i*10+k+1)*time.Hour), false)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO general_likes (user_id, track_id) VALUES ($1, $2)`, u.ID, tr.ID); err != nil {
|
||||
t.Fatalf("like: %v", err)
|
||||
}
|
||||
}
|
||||
for i, tr := range onceOnly {
|
||||
seedPlayEvent(t, pool, u.ID, tr.ID, now.Add(-time.Duration(200+i)*time.Hour), false)
|
||||
}
|
||||
|
||||
sum := genreDrawPositions(t, pool, u.ID, "krautrock")
|
||||
fav, once := meanPosition(sum, favourites), meanPosition(sum, onceOnly)
|
||||
if fav >= once {
|
||||
t.Errorf("favourites' summed position %.1f, played-once %.1f: want favourites earlier", fav, once)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListGenreMixTracks_UnheardLeansTowardTheUsersArtists: among unheard
|
||||
// tracks, an artist the user plays and likes comes up earlier than one they
|
||||
// have never met, and so does an artist similar to theirs. The stranger
|
||||
// still appears: the lean tilts the draw, it does not exclude.
|
||||
func TestListGenreMixTracks_UnheardLeansTowardTheUsersArtists(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
ctx := context.Background()
|
||||
u := seedUser(t, pool, "gmixartist")
|
||||
now := time.Now().UTC()
|
||||
|
||||
tracks := seedGenre(t, pool, "Shoegaze", 3, 12)
|
||||
loved, similar, stranger := tracks[:12], tracks[12:24], tracks[24:]
|
||||
for k := 0; k < 15; k++ {
|
||||
seedPlayEvent(t, pool, u.ID, loved[0].ID, now.Add(-time.Duration(k+1)*time.Hour), false)
|
||||
}
|
||||
for _, tr := range loved[1:3] {
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO general_likes (user_id, track_id) VALUES ($1, $2)`, u.ID, tr.ID); err != nil {
|
||||
t.Fatalf("like: %v", err)
|
||||
}
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO artist_similarity (artist_a_id, artist_b_id, score, source)
|
||||
VALUES ($1, $2, 0.9, 'listenbrainz')`, loved[0].ArtistID, similar[0].ArtistID); err != nil {
|
||||
t.Fatalf("seed similarity: %v", err)
|
||||
}
|
||||
|
||||
sum := genreDrawPositions(t, pool, u.ID, "shoegaze")
|
||||
l, s, x := meanPosition(sum, loved[3:]), meanPosition(sum, similar), meanPosition(sum, stranger)
|
||||
if l >= x {
|
||||
t.Errorf("loved artist's unheard tracks at %.1f, stranger's at %.1f: want the loved artist earlier", l, x)
|
||||
}
|
||||
if s >= x {
|
||||
t.Errorf("similar artist's tracks at %.1f, stranger's at %.1f: want the similar artist earlier", s, x)
|
||||
}
|
||||
for _, tr := range stranger {
|
||||
if _, in := sum[tr.ID]; !in {
|
||||
t.Fatalf("a stranger's track was left out of the pool; the lean must not exclude")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package playlists
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
// genreTracks builds n tracks, each on its own album and artist (so the
|
||||
// caps never bind), numbered from start.
|
||||
func genreTracks(start, n int, familiar bool) []genreTrack {
|
||||
out := make([]genreTrack, n)
|
||||
for i := range out {
|
||||
id := uuidN(start + i)
|
||||
out[i] = genreTrack{
|
||||
discoverTrack: discoverTrack{ID: id, AlbumID: id, ArtistID: id},
|
||||
Familiar: familiar,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func kindsOf(tracks []rankedCandidate) string {
|
||||
s := make([]byte, len(tracks))
|
||||
for i, t := range tracks {
|
||||
switch t.PickKind {
|
||||
case pickKindTaste:
|
||||
s[i] = 'F'
|
||||
case pickKindTasteUnheard:
|
||||
s[i] = 'U'
|
||||
default:
|
||||
s[i] = '?'
|
||||
}
|
||||
}
|
||||
return string(s)
|
||||
}
|
||||
|
||||
func TestComposeGenreMix_InterleavesThreeFamiliarToTwoUnheard(t *testing.T) {
|
||||
tracks := append(genreTracks(1, 20, true), genreTracks(100, 20, false)...)
|
||||
got := composeGenreMix(tracks, 10)
|
||||
if want := "FFUFUFFUFU"; kindsOf(got) != want {
|
||||
t.Fatalf("pick kinds = %s, want %s", kindsOf(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposeGenreMix_ShortSideIsFilledFromTheOther(t *testing.T) {
|
||||
// Two familiar tracks only: the rest of the mix is unheard, not short.
|
||||
tracks := append(genreTracks(1, 2, true), genreTracks(100, 20, false)...)
|
||||
got := composeGenreMix(tracks, 10)
|
||||
if len(got) != 10 {
|
||||
t.Fatalf("len = %d, want 10", len(got))
|
||||
}
|
||||
if want := "FFUUUUUUUU"; kindsOf(got) != want {
|
||||
t.Fatalf("pick kinds = %s, want %s", kindsOf(got), want)
|
||||
}
|
||||
|
||||
// No unheard tracks at all: an all-familiar mix.
|
||||
got = composeGenreMix(genreTracks(1, 12, true), 10)
|
||||
if want := "FFFFFFFFFF"; kindsOf(got) != want {
|
||||
t.Fatalf("pick kinds = %s, want %s", kindsOf(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposeGenreMix_CapsOneArtistThenTopsUpANarrowGenre(t *testing.T) {
|
||||
// One artist, one album, 8 tracks, plus 4 single-track artists. The cap
|
||||
// keeps 2 of the big album; the top-up refills from it so the mix still
|
||||
// reaches 8.
|
||||
big := genreTracks(1, 8, false)
|
||||
for i := range big {
|
||||
big[i].AlbumID = uuidN(500)
|
||||
big[i].ArtistID = uuidN(500)
|
||||
}
|
||||
tracks := append(big, genreTracks(100, 4, false)...)
|
||||
got := composeGenreMix(tracks, 8)
|
||||
if len(got) != 8 {
|
||||
t.Fatalf("len = %d, want 8", len(got))
|
||||
}
|
||||
// The first six are the capped pool: 2 from the big album, then the 4
|
||||
// singles — the big album does not lead the mix wholesale.
|
||||
bigID := map[pgtype.UUID]bool{}
|
||||
for _, b := range big {
|
||||
bigID[b.ID] = true
|
||||
}
|
||||
inHead := 0
|
||||
for _, c := range got[:6] {
|
||||
if bigID[c.TrackID] {
|
||||
inHead++
|
||||
}
|
||||
}
|
||||
if inHead != 2 {
|
||||
t.Fatalf("big-album tracks in the capped head = %d, want 2", inHead)
|
||||
}
|
||||
}
|
||||
|
||||
// songs returns the track ids from..to inclusive.
|
||||
func songs(from, to int) []pgtype.UUID {
|
||||
out := make([]pgtype.UUID, 0, to-from+1)
|
||||
for n := from; n <= to; n++ {
|
||||
out = append(out, uuidN(n))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func genreOf(label string, tier int32, ids ...[]pgtype.UUID) genreCandidate {
|
||||
var all []pgtype.UUID
|
||||
for _, part := range ids {
|
||||
all = append(all, part...)
|
||||
}
|
||||
return genreCandidate{Key: label, Label: label, Tier: tier, TrackIDs: all}
|
||||
}
|
||||
|
||||
// A genre whose songs mostly sit in one already picked is the same channel
|
||||
// under another name: Lo-Fi Hip Hop after Lo-Fi.
|
||||
func TestSelectGenreMixes_AGenreOfMostlyPickedSongsGivesWay(t *testing.T) {
|
||||
cands := []genreCandidate{
|
||||
genreOf("Lo-Fi", 1, songs(1, 100)),
|
||||
genreOf("Lo-Fi Hip Hop", 1, songs(1, 80), songs(500, 510)),
|
||||
genreOf("Jazz", 1, songs(1000, 1100)),
|
||||
}
|
||||
got := labels(selectGenreMixes(cands, buildN(30)))
|
||||
if len(got) != 2 || got[0] != "Lo-Fi mix" || got[1] != "Jazz mix" {
|
||||
t.Fatalf("mixes = %v, want [Lo-Fi mix Jazz mix]", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A broad genre whose songs mostly carry narrower genres gives way to them,
|
||||
// even when it ranks higher, and each later mix leaves out the songs of the
|
||||
// genres picked before it.
|
||||
func TestSelectGenreMixes_ABroadGenreGivesWayToItsSpecificOnes(t *testing.T) {
|
||||
cands := []genreCandidate{
|
||||
genreOf("Rock", 1, songs(1, 1000)),
|
||||
genreOf("Pop Punk", 1, songs(1, 300)),
|
||||
genreOf("Ska Punk", 1, songs(250, 600)),
|
||||
}
|
||||
excluded := map[string][]string{}
|
||||
build := func(c genreCandidate, exclude []string) []rankedCandidate {
|
||||
excluded[c.Label] = append([]string(nil), exclude...)
|
||||
return make([]rankedCandidate, 30)
|
||||
}
|
||||
got := labels(selectGenreMixes(cands, build))
|
||||
if len(got) != 2 || got[0] != "Pop Punk mix" || got[1] != "Ska Punk mix" {
|
||||
t.Fatalf("mixes = %v, want [Pop Punk mix Ska Punk mix]", got)
|
||||
}
|
||||
if ex := excluded["Ska Punk"]; len(ex) != 1 || ex[0] != "Pop Punk" {
|
||||
t.Fatalf("Ska Punk mix excludes %v, want [Pop Punk]", ex)
|
||||
}
|
||||
if ex := excluded["Pop Punk"]; len(ex) != 0 {
|
||||
t.Fatalf("first mix excludes %v, want nothing", ex)
|
||||
}
|
||||
}
|
||||
|
||||
// A genre only partly covered by narrower ones is a channel of its own, and
|
||||
// keeps its rank.
|
||||
func TestSelectGenreMixes_AGenreMostlyItsOwnKeepsItsRank(t *testing.T) {
|
||||
cands := []genreCandidate{
|
||||
genreOf("Rock", 1, songs(1, 2000)),
|
||||
genreOf("Pop Punk", 1, songs(1, 300)),
|
||||
genreOf("Jazz", 1, songs(3000, 3100)),
|
||||
}
|
||||
got := labels(selectGenreMixes(cands, buildN(30)))
|
||||
if len(got) != 2 || got[0] != "Rock mix" || got[1] != "Jazz mix" {
|
||||
t.Fatalf("mixes = %v, want [Rock mix Jazz mix]", got)
|
||||
}
|
||||
}
|
||||
|
||||
// buildN returns a builder that gives every genre n tracks, except those
|
||||
// listed in short, which get too few to make a mix.
|
||||
func buildN(n int, short ...string) func(genreCandidate, []string) []rankedCandidate {
|
||||
thin := map[string]bool{}
|
||||
for _, s := range short {
|
||||
thin[s] = true
|
||||
}
|
||||
return func(c genreCandidate, _ []string) []rankedCandidate {
|
||||
if thin[c.Label] {
|
||||
return make([]rankedCandidate, genreMixMinLen-1)
|
||||
}
|
||||
return make([]rankedCandidate, n)
|
||||
}
|
||||
}
|
||||
|
||||
func labels(built []builtPlaylist) []string {
|
||||
out := make([]string, len(built))
|
||||
for i, b := range built {
|
||||
out[i] = b.Name
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// distinctGenre is a genre of 100 songs of its own, numbered from n*100.
|
||||
func distinctGenre(label string, tier int32, n int) genreCandidate {
|
||||
return genreOf(label, tier, songs(n*100, n*100+99))
|
||||
}
|
||||
|
||||
func TestSelectGenreMixes_TakesTheUsersGenresAndStopsAtTheCount(t *testing.T) {
|
||||
var cands []genreCandidate
|
||||
for i := 0; i < 14; i++ {
|
||||
cands = append(cands, distinctGenre(string(rune('A'+i)), 1, i+1))
|
||||
}
|
||||
got := selectGenreMixes(cands, buildN(30))
|
||||
if len(got) != genreMixCount {
|
||||
t.Fatalf("built %d mixes, want %d", len(got), genreMixCount)
|
||||
}
|
||||
if got[0].Name != "A mix" || got[0].Variant != "genre_mix" {
|
||||
t.Fatalf("first mix = %q (%s), want \"A mix\" (genre_mix)", got[0].Name, got[0].Variant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectGenreMixes_ThinGenreGivesWayToTheNext(t *testing.T) {
|
||||
cands := []genreCandidate{
|
||||
distinctGenre("Jazz", 1, 1),
|
||||
distinctGenre("Dub", 1, 2),
|
||||
distinctGenre("Soul", 2, 3),
|
||||
}
|
||||
got := labels(selectGenreMixes(cands, buildN(30, "Dub")))
|
||||
if len(got) != 2 || got[0] != "Jazz mix" || got[1] != "Soul mix" {
|
||||
t.Fatalf("mixes = %v, want [Jazz mix Soul mix]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectGenreMixes_LibraryGenresOnlyFillAShortShelf(t *testing.T) {
|
||||
// Two listened genres: tier 3 tops the shelf up to genreMixMinShelf.
|
||||
cands := []genreCandidate{
|
||||
distinctGenre("Jazz", 1, 1),
|
||||
distinctGenre("Soul", 2, 2),
|
||||
distinctGenre("Polka", 3, 3),
|
||||
distinctGenre("Opera", 3, 4),
|
||||
distinctGenre("Ska", 3, 5),
|
||||
}
|
||||
got := labels(selectGenreMixes(cands, buildN(30)))
|
||||
if len(got) != genreMixMinShelf {
|
||||
t.Fatalf("mixes = %v, want %d", got, genreMixMinShelf)
|
||||
}
|
||||
|
||||
// Enough listened genres: no tier-3 genre at all.
|
||||
var full []genreCandidate
|
||||
for i := 0; i < genreMixMinShelf; i++ {
|
||||
full = append(full, distinctGenre(string(rune('A'+i)), 1, i+1))
|
||||
}
|
||||
full = append(full, distinctGenre("Polka", 3, 99))
|
||||
for _, name := range labels(selectGenreMixes(full, buildN(30))) {
|
||||
if name == "Polka mix" {
|
||||
t.Fatal("a library-only genre was added to a shelf the user's own genres already fill")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -531,8 +531,8 @@ type systemPlaylistKind struct {
|
||||
// Discover, the discovery mixes) and so can be addressed by kind:
|
||||
// the generic /system/{kind}/{refresh,shuffle} endpoints + the
|
||||
// per-tile refresh affordance only apply to these. Non-singleton
|
||||
// kinds (songs_like_artist → up to 3 per user) are played by
|
||||
// playlist id and have no by-kind endpoint.
|
||||
// kinds (songs_like_artist, genre_mix — several per user) are played
|
||||
// by playlist id and have no by-kind endpoint.
|
||||
Singleton bool
|
||||
Produce systemPlaylistProducer
|
||||
}
|
||||
@@ -563,6 +563,7 @@ var systemPlaylistRegistry = func() []systemPlaylistKind {
|
||||
{Key: "for_you", Singleton: true, Produce: produceForYou},
|
||||
{Key: "songs_like_artist", Singleton: false, Produce: produceSeedMixes},
|
||||
{Key: "discover", Singleton: true, Produce: produceDiscover},
|
||||
{Key: "genre_mix", Singleton: false, Produce: produceGenreMixes},
|
||||
}
|
||||
// The five discovery mixes share one Produce closure factory keyed
|
||||
// by a per-mix spec; spec list + factory live in system_mixes.go.
|
||||
|
||||
@@ -114,7 +114,7 @@ func TestBuildSystemPlaylists_SufficientActivity(t *testing.T) {
|
||||
if !r.SeedArtistID.Valid {
|
||||
t.Errorf("songs_like_artist row should have non-NULL seed_artist_id")
|
||||
}
|
||||
case "discover", "deep_cuts", "rediscover", "new_for_you", "on_this_day", "first_listens":
|
||||
case "discover", "deep_cuts", "rediscover", "new_for_you", "on_this_day", "first_listens", "genre_mix":
|
||||
// Discover and the discovery mixes are all seedless —
|
||||
// no seed_artist_id required.
|
||||
default:
|
||||
|
||||
@@ -35,6 +35,7 @@ var (
|
||||
SubsonicPassword = Column{"users", "subsonic_password", "id", false}
|
||||
CoverArtAPIKey = Column{"cover_art_provider_settings", "api_key", "provider_id", false}
|
||||
TagProviderAPIKey = Column{"tag_provider_settings", "api_key", "provider_id", false}
|
||||
BackupSecretKey = Column{"backup_settings", "secret_key", "id", true}
|
||||
)
|
||||
|
||||
// Columns is every sealed column, for SealStored and for anything that has to
|
||||
@@ -42,6 +43,7 @@ var (
|
||||
var Columns = []Column{
|
||||
AcquisitionAPIKey, AcquisitionPassword, AcoustIDAPIKey, SMTPPassword,
|
||||
ListenBrainzToken, SubsonicPassword, CoverArtAPIKey, TagProviderAPIKey,
|
||||
BackupSecretKey,
|
||||
}
|
||||
|
||||
// Owner is the owner string for a row whose key is key. A single-row table's
|
||||
|
||||
@@ -84,6 +84,7 @@ func TestSealStoredSealsWhatIsInTheClear_Integration(t *testing.T) {
|
||||
`INSERT INTO tag_provider_settings (provider_id, enabled, display_order, api_key)
|
||||
VALUES ('seal-test', false, 99, 'tag-key')
|
||||
ON CONFLICT (provider_id) DO UPDATE SET api_key = EXCLUDED.api_key`,
|
||||
`UPDATE backup_settings SET secret_key = 'bucket-secret' WHERE id = true`,
|
||||
}
|
||||
for _, s := range seed {
|
||||
if _, err := pool.Exec(ctx, s); err != nil {
|
||||
@@ -97,6 +98,7 @@ func TestSealStoredSealsWhatIsInTheClear_Integration(t *testing.T) {
|
||||
`DELETE FROM acquisition_services WHERE code = 'seal-test'`,
|
||||
`UPDATE acoustid_settings SET api_key = NULL WHERE id = true`,
|
||||
`UPDATE smtp_config SET password = '' WHERE id = true`,
|
||||
`UPDATE backup_settings SET secret_key = '' WHERE id = true`,
|
||||
} {
|
||||
_, _ = pool.Exec(context.Background(), s)
|
||||
}
|
||||
@@ -115,6 +117,7 @@ func TestSealStoredSealsWhatIsInTheClear_Integration(t *testing.T) {
|
||||
{secret.SubsonicPassword, u.ID.String(), "sub-pw"},
|
||||
{secret.CoverArtAPIKey, "seal-test", "cover-key"},
|
||||
{secret.TagProviderAPIKey, "seal-test", "tag-key"},
|
||||
{secret.BackupSecretKey, "true", "bucket-secret"},
|
||||
}
|
||||
if len(want) != len(secret.Columns) {
|
||||
t.Fatalf("the test covers %d columns, secret.Columns lists %d", len(want), len(secret.Columns))
|
||||
|
||||
@@ -18,6 +18,23 @@ const maxRequestBody = 4 << 20
|
||||
// bodyReadTimeout bounds how long a client may take to send a request body.
|
||||
const bodyReadTimeout = 30 * time.Second
|
||||
|
||||
// restorePath takes a whole backup file, so it gets room for one: the file is
|
||||
// the database alone (the music travels separately), which is tens of MB for
|
||||
// a large library.
|
||||
const (
|
||||
restorePath = "/api/auth/restore"
|
||||
maxRestoreBody = 1 << 30
|
||||
restoreReadTimeout = 15 * time.Minute
|
||||
)
|
||||
|
||||
// bodyLimits is the size and time a request may take to send its body.
|
||||
func bodyLimits(r *http.Request) (int64, time.Duration) {
|
||||
if r.Method == http.MethodPost && r.URL.Path == restorePath {
|
||||
return maxRestoreBody, restoreReadTimeout
|
||||
}
|
||||
return maxRequestBody, bodyReadTimeout
|
||||
}
|
||||
|
||||
// limitRequestBody caps the body size and the time allowed to deliver it.
|
||||
//
|
||||
// Why not http.Server.ReadTimeout: that deadline stays armed for the whole
|
||||
@@ -35,8 +52,9 @@ func limitRequestBody(next http.Handler) http.Handler {
|
||||
rc := http.NewResponseController(w)
|
||||
// Best-effort: a ResponseWriter that can't set deadlines (a test
|
||||
// recorder) still gets the size limit.
|
||||
armed := rc.SetReadDeadline(time.Now().Add(bodyReadTimeout)) == nil
|
||||
body := http.MaxBytesReader(w, r.Body, maxRequestBody)
|
||||
size, wait := bodyLimits(r)
|
||||
armed := rc.SetReadDeadline(time.Now().Add(wait)) == nil
|
||||
body := http.MaxBytesReader(w, r.Body, size)
|
||||
if armed {
|
||||
body = &deadlineClearingBody{ReadCloser: body, rc: rc}
|
||||
}
|
||||
@@ -83,6 +101,10 @@ func (b *deadlineClearingBody) Close() error {
|
||||
// attaches on its own, so they are not checked. The Android app does send
|
||||
// the cookie, but every body it sends is JSON (Retrofit's kotlinx converter)
|
||||
// and its bodiless writes carry no Content-Type, so it passes unchanged.
|
||||
//
|
||||
// The restore upload is a backup file, not JSON. It is safe without the check:
|
||||
// it acts only with the setup token in a request header, which a form cannot
|
||||
// send and a cross-site fetch cannot send without a preflight.
|
||||
func requireJSONForCookieWrites(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
@@ -90,7 +112,7 @@ func requireJSONForCookieWrites(next http.Handler) http.Handler {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(r.URL.Path, "/api/") {
|
||||
if !strings.HasPrefix(r.URL.Path, "/api/") || r.URL.Path == restorePath {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -40,6 +40,21 @@ func TestLimitRequestBody_RejectsOversizedBody(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A backup file is far larger than any other body, and only the restore
|
||||
// upload may send one.
|
||||
func TestLimitRequestBody_TheRestoreUploadTakesABackupFile(t *testing.T) {
|
||||
h := limitRequestBody(okHandler())
|
||||
file := strings.NewReader(strings.Repeat("x", maxRequestBody*2))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, restorePath, file))
|
||||
if w.Code != http.StatusNoContent {
|
||||
t.Errorf("a backup-sized restore upload: status = %d, want 204", w.Code)
|
||||
}
|
||||
if size, wait := bodyLimits(httptest.NewRequest(http.MethodPost, "/api/playlists", nil)); size != maxRequestBody || wait != bodyReadTimeout {
|
||||
t.Errorf("another path's limits = %d, %v; want the ordinary ones", size, wait)
|
||||
}
|
||||
}
|
||||
|
||||
// The cases a cross-site form or no-preflight fetch could produce must be
|
||||
// refused when the session cookie rides along; everything a real client
|
||||
// sends must pass.
|
||||
@@ -55,6 +70,7 @@ func TestRequireJSONForCookieWrites(t *testing.T) {
|
||||
want int
|
||||
}{
|
||||
{"form post with cookie", http.MethodPost, "/api/me/password", "application/x-www-form-urlencoded", "a=b", true, http.StatusUnsupportedMediaType},
|
||||
{"restore upload with a stale cookie", http.MethodPost, "/api/auth/restore", "application/x-tar", "tar", true, http.StatusNoContent},
|
||||
{"text/plain post with cookie", http.MethodPost, "/api/me/password", "text/plain", `{"a":1}`, true, http.StatusUnsupportedMediaType},
|
||||
{"multipart with cookie", http.MethodPut, "/api/me/profile", "multipart/form-data; boundary=x", "--x--", true, http.StatusUnsupportedMediaType},
|
||||
{"body without content type", http.MethodPost, "/api/me/profile", "", `{"a":1}`, true, http.StatusUnsupportedMediaType},
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/api"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/buildinfo"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/config"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
|
||||
@@ -94,12 +95,18 @@ type Server struct {
|
||||
// the admin card reaches it without a restart. Router() constructs a
|
||||
// fallback when nil (tests).
|
||||
ReacqSettings *reacquisition.SettingsService
|
||||
// Backups is the server's own backup, the instance whose loop runs the
|
||||
// nightly schedule. Nil builds one that runs only when an admin asks.
|
||||
Backups *backup.Service
|
||||
// StreamSecret is the HMAC key used by /api/cast/stream-token to
|
||||
// mint signed UPnP / Sonos stream URLs and by /api/tracks/{id}/stream
|
||||
// to verify them. Sourced from config.Config.StreamSecret. Tests that
|
||||
// leave it nil leave the cookie path intact and reject all signed
|
||||
// tokens (HMAC of empty key matches nothing a client could mint).
|
||||
StreamSecret []byte
|
||||
// Restart asks the process to stop and start again in place. A web
|
||||
// restore calls it so the restored settings are read at boot.
|
||||
Restart func()
|
||||
}
|
||||
|
||||
func New(logger *slog.Logger, pool *pgxpool.Pool, scanner ScanTrigger, subCfg subsonic.Config, eventsCfg config.EventsConfig, recCfg config.RecommendationConfig, dataDir string, brandingCfg config.BrandingConfig, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, libraryScanner *library.Scanner, scanCfg library.RunScanConfig) *Server {
|
||||
@@ -200,7 +207,11 @@ func (s *Server) Router() http.Handler {
|
||||
}
|
||||
acoustIDLookup = library.NewAcoustIDLookupWorker(s.Pool, s.Logger, acoustIDSettings, nil)
|
||||
}
|
||||
api.Mount(r, s.Pool, s.Logger, writer, s.RecommendationCfg, recSettings, requestSvc, quarantineSvc, tracksSvc, playlistsSvc, s.CoverEnricher, s.CoverSettings, s.TagSettings, s.LibraryScanner, s.ScanCfg, s.DataDir, smtpSender, bus, s.PlaylistScheduler, s.StreamSecret, netSettings, reacqSettings, fpSettings, loudSettings, acoustIDLookup)
|
||||
backups := s.Backups
|
||||
if backups == nil {
|
||||
backups = backup.NewService(s.Pool, s.Logger, buildinfo.Version)
|
||||
}
|
||||
api.Mount(r, s.Pool, s.Logger, writer, s.RecommendationCfg, recSettings, requestSvc, quarantineSvc, tracksSvc, playlistsSvc, s.CoverEnricher, s.CoverSettings, s.TagSettings, s.LibraryScanner, s.ScanCfg, s.DataDir, smtpSender, bus, s.PlaylistScheduler, s.StreamSecret, netSettings, reacqSettings, fpSettings, loudSettings, acoustIDLookup, backups, s.Restart)
|
||||
// /api/admin/scan is the only admin route owned by the server package
|
||||
// (it needs the Scanner). Register it as a single inline-middleware
|
||||
// route — using r.Route("/api/admin", ...) here would create a second
|
||||
|
||||
+39
-24
@@ -1,17 +1,22 @@
|
||||
// Periodic compactor for the library_changes log. Every mutation in the
|
||||
// library writes one row, so the table grows unbounded without a retention
|
||||
// policy. This worker trims rows older than CompactorRetention on a daily
|
||||
// tick.
|
||||
// library writes one row, so the log grows without bound unless trimmed. This
|
||||
// worker trims it daily, keeping each entity's latest row:
|
||||
//
|
||||
// Why 30 days: clients keep a cursor and pull deltas from the last seen id.
|
||||
// A client that checks in at least once per retention window never sees its
|
||||
// cursor go stale; one offline longer than that gets the /api/library/sync
|
||||
// 410 and does a full resync.
|
||||
// - A row a newer row for the same entity has superseded goes at once. An
|
||||
// app reading from any cursor still meets the newer row.
|
||||
// - A delete goes once it is CompactorRetention old, and raises the trimmed
|
||||
// mark (library_changes_trimmed) past its id. An app whose cursor is below
|
||||
// the mark may have missed it, so /api/library/sync answers 410 and the
|
||||
// app syncs again from 0.
|
||||
// - The latest upsert of an entity that still exists stays, however old. An
|
||||
// app syncing from 0 (a fresh install, or after a 410) builds its whole
|
||||
// library from the log, so it must hold every artist, album and track.
|
||||
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
@@ -21,9 +26,9 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
)
|
||||
|
||||
// CompactorRetention is the age past which library_changes rows are
|
||||
// dropped. Mutable for tests; production runs should not change it
|
||||
// without considering the corresponding client cursor-stale fallback.
|
||||
// CompactorRetention is how long a delete stays in the log. An app that
|
||||
// checks in at least once in that window never misses one; one away longer
|
||||
// is sent back to 0. Mutable for tests.
|
||||
var CompactorRetention = 30 * 24 * time.Hour
|
||||
|
||||
// CompactorInterval is how often the worker checks. Daily is enough —
|
||||
@@ -31,8 +36,8 @@ var CompactorRetention = 30 * 24 * time.Hour
|
||||
// active library, well below a size where intra-day trimming matters.
|
||||
var CompactorInterval = 24 * time.Hour
|
||||
|
||||
// Compactor is a background worker that periodically deletes
|
||||
// library_changes rows older than CompactorRetention.
|
||||
// Compactor is a background worker that trims the library_changes log down
|
||||
// to each entity's latest change.
|
||||
type Compactor struct {
|
||||
pool *pgxpool.Pool
|
||||
logger *slog.Logger
|
||||
@@ -65,18 +70,28 @@ func (c *Compactor) Run(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// tickOnce deletes rows older than CompactorRetention and logs the
|
||||
// row count. Errors are logged at WARN; we never abort the loop on
|
||||
// transient DB failures.
|
||||
// tickOnce trims the log once and logs what went. Errors are logged at WARN;
|
||||
// the loop never stops on a transient DB failure.
|
||||
func (c *Compactor) tickOnce(ctx context.Context) {
|
||||
cutoff := pgtype.Timestamptz{Time: time.Now().Add(-CompactorRetention), Valid: true}
|
||||
rows, err := dbq.New(c.pool).DeleteOldLibraryChanges(ctx, cutoff)
|
||||
if err != nil {
|
||||
c.logger.Warn("library_changes compactor: delete failed", "err", err)
|
||||
return
|
||||
}
|
||||
if rows > 0 {
|
||||
c.logger.Info("library_changes compactor: trimmed rows",
|
||||
"count", rows, "older_than", CompactorRetention)
|
||||
if err := Compact(ctx, c.pool, time.Now().Add(-CompactorRetention), c.logger); err != nil {
|
||||
c.logger.Warn("library_changes compactor: trim failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Compact drops every superseded row, then every delete older than cutoff.
|
||||
func Compact(ctx context.Context, db dbq.DBTX, cutoff time.Time, logger *slog.Logger) error {
|
||||
q := dbq.New(db)
|
||||
superseded, err := q.DeleteSupersededLibraryChanges(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("drop superseded changes: %w", err)
|
||||
}
|
||||
deletes, err := q.DeleteOldLibraryDeletes(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true})
|
||||
if err != nil {
|
||||
return fmt.Errorf("drop old deletes: %w", err)
|
||||
}
|
||||
if superseded > 0 || deletes > 0 {
|
||||
logger.Info("library_changes compactor: trimmed rows",
|
||||
"superseded", superseded, "old_deletes", deletes, "deletes_older_than", CompactorRetention)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package sync_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/sync"
|
||||
)
|
||||
|
||||
// The compactor keeps each entity's latest change however old, so an app
|
||||
// syncing from 0 still gets an artist nobody has touched in months. It drops
|
||||
// what a newer row supersedes, and deletes past their retention, raising the
|
||||
// trimmed mark past the newest delete it dropped.
|
||||
func TestCompact_KeepsEachEntitysLatestChange_Integration(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
old := time.Now().Add(-90 * 24 * time.Hour)
|
||||
recent := time.Now().Add(-time.Hour)
|
||||
|
||||
add := func(entity, op string, at time.Time) int64 {
|
||||
t.Helper()
|
||||
var id int64
|
||||
require.NoError(t, pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op, changed_at)
|
||||
VALUES ('artist', $1, $2, $3) RETURNING id`, entity, op, at).Scan(&id))
|
||||
return id
|
||||
}
|
||||
untouched := add("untouched", "upsert", old) // kept: latest, however old
|
||||
add("edited", "upsert", old) // dropped: superseded
|
||||
edited := add("edited", "upsert", recent) // kept
|
||||
add("removed-long-ago", "upsert", old) // dropped: superseded
|
||||
removedLongAgo := add("removed-long-ago", "delete", old) // dropped: past retention
|
||||
add("removed-lately", "upsert", old) // dropped: superseded
|
||||
removedLately := add("removed-lately", "delete", recent) // kept: within retention
|
||||
|
||||
require.NoError(t, sync.Compact(ctx, pool, time.Now().Add(-sync.CompactorRetention),
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil))))
|
||||
|
||||
rows, err := pool.Query(ctx, `SELECT id FROM library_changes ORDER BY id`)
|
||||
require.NoError(t, err)
|
||||
left, err := pgx.CollectRows(rows, pgx.RowTo[int64])
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []int64{untouched, edited, removedLately}, left)
|
||||
|
||||
var through int64
|
||||
require.NoError(t, pool.QueryRow(ctx, `SELECT through FROM library_changes_trimmed`).Scan(&through))
|
||||
require.Equal(t, removedLongAgo, through, "the trimmed mark stands at the newest dropped delete")
|
||||
|
||||
// A second pass finds nothing more, and leaves the mark where it is.
|
||||
require.NoError(t, sync.Compact(ctx, pool, time.Now().Add(-sync.CompactorRetention),
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil))))
|
||||
require.NoError(t, pool.QueryRow(ctx, `SELECT through FROM library_changes_trimmed`).Scan(&through))
|
||||
require.Equal(t, removedLongAgo, through)
|
||||
}
|
||||
@@ -1086,3 +1086,93 @@ export async function clearAcquisitionSecret(
|
||||
): Promise<void> {
|
||||
await api.del(`${acquisitionBase}/${encodeURIComponent(code)}/${which}`);
|
||||
}
|
||||
|
||||
// The server's own backup ---------------------------------------------------
|
||||
|
||||
export type BackupDestination = 'folder' | 's3';
|
||||
|
||||
export type BackupConfig = {
|
||||
enabled: boolean;
|
||||
destination: BackupDestination;
|
||||
folder: string;
|
||||
endpoint: string;
|
||||
bucket: string;
|
||||
access_key: string;
|
||||
/** The secret key itself never comes back. */
|
||||
has_secret_key: boolean;
|
||||
/** The hour of day the nightly backup starts, in UTC. */
|
||||
run_hour: number;
|
||||
/** KiB a second an upload to the bucket may use; 0 is no limit. */
|
||||
upload_limit_kib: number;
|
||||
/** Also send the library's music files, each once, to music/. */
|
||||
include_music: boolean;
|
||||
};
|
||||
|
||||
export type BackupState = 'off' | 'incomplete' | 'failing' | 'waiting' | 'overdue' | 'ok';
|
||||
|
||||
export type BackupRun = {
|
||||
id: string;
|
||||
trigger: 'schedule' | 'manual';
|
||||
status: 'running' | 'succeeded' | 'failed';
|
||||
started_at: string;
|
||||
finished_at: string | null;
|
||||
/** The file's path in the folder, or its key in the bucket. */
|
||||
location: string;
|
||||
bytes: number;
|
||||
error: string;
|
||||
/** What the run sent of the music, and why it stopped early, if it did. */
|
||||
music_files: number;
|
||||
music_bytes: number;
|
||||
music_note: string;
|
||||
};
|
||||
|
||||
/** A library file the destination wouldn't take; tried again once it changes. */
|
||||
export type BackupRefusedFile = { path: string; error: string; at: string };
|
||||
|
||||
/** How much of the library the backup holds as it is now. */
|
||||
export type BackupMusicProgress = {
|
||||
total_files: number;
|
||||
total_bytes: number;
|
||||
sent_files: number;
|
||||
sent_bytes: number;
|
||||
refused_files: number;
|
||||
refused: BackupRefusedFile[];
|
||||
};
|
||||
|
||||
export type BackupStatus = {
|
||||
config: BackupConfig;
|
||||
/** Worked out by the server; the panel only words it. */
|
||||
state: BackupState;
|
||||
problems: string[];
|
||||
running_since: string | null;
|
||||
last_success: string | null;
|
||||
failing_since: string | null;
|
||||
next_run: string | null;
|
||||
/** The longest a run may take; the panel stops polling a little after. */
|
||||
run_limit_seconds: number;
|
||||
runs: BackupRun[];
|
||||
/** Present when the backup includes the music. */
|
||||
music: BackupMusicProgress | null;
|
||||
};
|
||||
|
||||
/** A save or test. An empty secret_key keeps the stored one. */
|
||||
export type BackupUpdate = Omit<BackupConfig, 'has_secret_key'> & { secret_key: string };
|
||||
|
||||
export type BackupTestResult = { backups: number; latest: string | null };
|
||||
|
||||
export async function getBackups(): Promise<BackupStatus> {
|
||||
return api.get<BackupStatus>('/api/admin/backups');
|
||||
}
|
||||
|
||||
export async function saveBackups(u: BackupUpdate): Promise<BackupStatus> {
|
||||
return api.put<BackupStatus>('/api/admin/backups', u);
|
||||
}
|
||||
|
||||
/** Checks the destination in u without saving it; nothing is written there. */
|
||||
export async function testBackups(u: BackupUpdate): Promise<BackupTestResult> {
|
||||
return api.post<BackupTestResult>('/api/admin/backups/test', u);
|
||||
}
|
||||
|
||||
export async function runBackup(): Promise<void> {
|
||||
await api.post<null>('/api/admin/backups/run', {});
|
||||
}
|
||||
|
||||
@@ -23,7 +23,11 @@ const DETAIL_CODES: ReadonlySet<string> = new Set([
|
||||
// The server says what shape of address it wants.
|
||||
'invalid_public_url',
|
||||
// The server names the folder and what is wrong with it.
|
||||
'bad_folders'
|
||||
'bad_folders',
|
||||
// The server says which backup setting and why.
|
||||
'invalid_backup_setting',
|
||||
// The destination's own answer: the bucket's refusal, or the missing folder.
|
||||
'destination_failed'
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { api, type User, type LoginResponse } from '#lib/api/client.js';
|
||||
import { api, apiFetch, type User, type LoginResponse } from '#lib/api/client.js';
|
||||
import { putMyTimezone } from '#lib/api/me.js';
|
||||
import { queryClient } from '#lib/query/client.js';
|
||||
import { signalSessionEnd } from './sessionEnd.svelte';
|
||||
@@ -75,6 +75,20 @@ export async function register(opts: {
|
||||
void sendTimezoneIfStale();
|
||||
}
|
||||
|
||||
export type RestoreResult = { accounts: number; restarting: boolean };
|
||||
|
||||
/**
|
||||
* Loads a backup file into a new server, which has no accounts yet. The file
|
||||
* is the request body; the setup token from the server log goes in a header.
|
||||
*/
|
||||
export async function restoreBackup(file: Blob, setupToken: string): Promise<RestoreResult> {
|
||||
return (await apiFetch('/api/auth/restore', {
|
||||
method: 'POST',
|
||||
body: file,
|
||||
headers: { 'Content-Type': 'application/x-tar', 'X-Setup-Token': setupToken }
|
||||
})) as RestoreResult;
|
||||
}
|
||||
|
||||
export async function forgotPassword(email: string): Promise<void> {
|
||||
await api.post('/api/auth/forgot-password', { email });
|
||||
}
|
||||
|
||||
@@ -67,13 +67,12 @@
|
||||
try {
|
||||
const variant = playlist.system_variant;
|
||||
// systemShuffle keys by variant alone, which only works for
|
||||
// SINGLE-instance variants (for_you, discover, deep_cuts, …).
|
||||
// songs_like_artist has one playlist PER seed artist; using
|
||||
// the variant alone would return the wrong one (or fail).
|
||||
// For those, fall through to getPlaylist so the play handler
|
||||
// hits the exact playlist the user clicked on. Detect via
|
||||
// seed_artist_id which is non-null only for per-artist mixes.
|
||||
const isPerArtist = playlist.seed_artist_id != null;
|
||||
// SINGLE-instance variants (for_you, discover, deep_cuts, …) — the
|
||||
// ones the server reports as refreshable. A kind with several
|
||||
// playlists per user (songs_like_artist, genre_mix) would get the
|
||||
// wrong one, so those fall through to getPlaylist and play the exact
|
||||
// playlist the user clicked on.
|
||||
const isPerArtist = !playlist.refreshable;
|
||||
if (variant != null && !isPerArtist) {
|
||||
// Server returns the rotation-aware order (unplayed
|
||||
// this rotation first). Play it as-is — no client shuffle —
|
||||
|
||||
@@ -266,6 +266,7 @@ describe('PlaylistCard', () => {
|
||||
...base,
|
||||
kind: 'system',
|
||||
system_variant: 'for_you',
|
||||
refreshable: true,
|
||||
name: 'For You'
|
||||
};
|
||||
render(PlaylistCard, { props: { playlist: forYouPlaylist } });
|
||||
@@ -280,6 +281,34 @@ describe('PlaylistCard', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('genre-mix play fetches that playlist by id, tagged genre_mix', async () => {
|
||||
// Several genre mixes per user: the variant alone can't address one,
|
||||
// so the card plays the exact playlist it shows.
|
||||
const { getPlaylist, systemShuffle } = await import('#lib/api/playlists.js');
|
||||
const { playQueue } = await import('#lib/player/store.svelte.js');
|
||||
vi.mocked(getPlaylist).mockClear();
|
||||
vi.mocked(systemShuffle).mockClear();
|
||||
const jazzMix: Playlist = {
|
||||
...base,
|
||||
id: 'gm-jazz',
|
||||
kind: 'system',
|
||||
system_variant: 'genre_mix',
|
||||
refreshable: false,
|
||||
name: 'Jazz mix'
|
||||
};
|
||||
render(PlaylistCard, { props: { playlist: jazzMix } });
|
||||
await fireEvent.click(screen.getByLabelText(/Play Jazz mix/i));
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
|
||||
expect(getPlaylist).toHaveBeenCalledWith('gm-jazz');
|
||||
expect(systemShuffle).not.toHaveBeenCalled();
|
||||
expect(playQueue).toHaveBeenCalledWith(
|
||||
expect.any(Array),
|
||||
0,
|
||||
expect.objectContaining({ source: 'genre_mix' })
|
||||
);
|
||||
});
|
||||
|
||||
test('user-playlist play uses getPlaylist + plain playQueue (no source)', async () => {
|
||||
const { getPlaylist, systemShuffle } = await import('#lib/api/playlists.js');
|
||||
const { playQueue } = await import('#lib/player/store.svelte.js');
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
<script lang="ts">
|
||||
import { onDestroy, onMount } from 'svelte';
|
||||
import { getBackups, runBackup, type BackupStatus } from '#lib/api/admin.js';
|
||||
import { errMessage } from '#lib/api/errors.js';
|
||||
import { pushToast } from '#lib/stores/toast.svelte.js';
|
||||
import { formatBytes } from '#lib/utils/bytes.js';
|
||||
import { plural } from '#lib/utils/plural.js';
|
||||
import SettingsColumns from '#lib/components/settings/SettingsColumns.svelte';
|
||||
import SettingsGroup from '#lib/components/settings/SettingsGroup.svelte';
|
||||
import BackupSettings from './BackupSettings.svelte';
|
||||
import { when } from './backupTime.js';
|
||||
|
||||
// The server's own backup: a verdict line first, then what to do about it,
|
||||
// then the latest runs, with the settings beside them. The server works out
|
||||
// the state; this only words it.
|
||||
|
||||
let status = $state<BackupStatus | null>(null);
|
||||
let loadError = $state(false);
|
||||
let starting = $state(false);
|
||||
|
||||
async function load() {
|
||||
try {
|
||||
status = await getBackups();
|
||||
loadError = false;
|
||||
} catch {
|
||||
loadError = true;
|
||||
}
|
||||
}
|
||||
|
||||
onMount(load);
|
||||
|
||||
// Polled only while a backup runs, and never past the server's own limit
|
||||
// on a run plus a margin, so a stuck run can't keep a tab asking forever.
|
||||
const POLL_MS = 3000;
|
||||
const POLL_MARGIN_MS = 5 * 60 * 1000;
|
||||
let poll: ReturnType<typeof setTimeout> | null = null;
|
||||
let pollStarted = 0;
|
||||
|
||||
function stopPolling() {
|
||||
if (poll !== null) clearTimeout(poll);
|
||||
poll = null;
|
||||
}
|
||||
|
||||
function pollWhileRunning() {
|
||||
stopPolling();
|
||||
const limit = (status?.run_limit_seconds ?? 1800) * 1000 + POLL_MARGIN_MS;
|
||||
if (!status?.running_since || Date.now() - pollStarted > limit) return;
|
||||
poll = setTimeout(async () => {
|
||||
try {
|
||||
await load();
|
||||
} finally {
|
||||
pollWhileRunning();
|
||||
}
|
||||
}, POLL_MS);
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
if (status?.running_since && poll === null) {
|
||||
pollStarted = Date.now();
|
||||
pollWhileRunning();
|
||||
}
|
||||
});
|
||||
onDestroy(stopPolling);
|
||||
|
||||
async function backUpNow() {
|
||||
starting = true;
|
||||
try {
|
||||
await runBackup();
|
||||
pushToast('Backup started.');
|
||||
await load();
|
||||
} catch (e) {
|
||||
pushToast(errMessage(e), 'error');
|
||||
await load();
|
||||
} finally {
|
||||
starting = false;
|
||||
}
|
||||
}
|
||||
|
||||
type Tone = 'good' | 'warn' | 'bad' | 'plain';
|
||||
const toneClass: Record<Tone, string> = {
|
||||
good: 'text-success-fg',
|
||||
warn: 'text-warning-fg',
|
||||
bad: 'text-error-fg',
|
||||
plain: 'text-text-primary'
|
||||
};
|
||||
|
||||
const verdict = $derived.by((): { tone: Tone; text: string } | null => {
|
||||
const s = status;
|
||||
if (!s) return null;
|
||||
if (s.running_since) return { tone: 'plain', text: `Backing up, since ${when(s.running_since)}.` };
|
||||
switch (s.state) {
|
||||
case 'off':
|
||||
return { tone: 'plain', text: 'Off. Nothing is backed up on a schedule.' };
|
||||
case 'incomplete':
|
||||
return { tone: 'warn', text: 'Switched on, but not set up yet.' };
|
||||
case 'waiting':
|
||||
return { tone: 'plain', text: 'Set up. No backup has been made yet.' };
|
||||
case 'ok':
|
||||
return { tone: 'good', text: `Backed up ${when(s.last_success!)}.` };
|
||||
case 'overdue':
|
||||
return { tone: 'warn', text: `No backup since ${when(s.last_success!)}.` };
|
||||
case 'failing':
|
||||
return { tone: 'bad', text: `Backups failing since ${when(s.failing_since!)}.` };
|
||||
}
|
||||
});
|
||||
|
||||
const latest = $derived(status?.runs[0]);
|
||||
const ready = $derived(!!status && status.problems.length === 0);
|
||||
|
||||
const button =
|
||||
'rounded-md border border-border px-3 py-1.5 text-sm text-text-secondary hover:bg-surface-hover hover:text-text-primary disabled:opacity-50';
|
||||
</script>
|
||||
|
||||
{#if loadError && !status}
|
||||
<p class="text-sm text-error-fg">
|
||||
Couldn't load the backup status.
|
||||
<button type="button" class="underline hover:no-underline" onclick={load}>Try again</button>
|
||||
</p>
|
||||
{:else if !status}
|
||||
<p class="text-sm text-text-secondary" aria-busy="true">Loading the backup status…</p>
|
||||
{:else}
|
||||
<SettingsColumns>
|
||||
{#snippet main()}
|
||||
{#if status}
|
||||
<SettingsGroup title="Backup" id="backup" hint="Your accounts, likes, playlists, history and settings, in one file a night.">
|
||||
<div class="space-y-1 px-4 py-3" role="status" data-testid="backup-verdict">
|
||||
<p class="text-sm {toneClass[verdict?.tone ?? 'plain']}">{verdict?.text}</p>
|
||||
{#if status.state === 'failing' && latest?.error && !status.running_since}
|
||||
<p class="text-xs text-text-secondary">{latest.error}</p>
|
||||
{/if}
|
||||
{#if status.config.enabled}
|
||||
{#each status.problems as problem (problem)}
|
||||
<p class="text-xs text-text-secondary">{problem}</p>
|
||||
{/each}
|
||||
{/if}
|
||||
{#if status.next_run && !status.running_since}
|
||||
<p class="text-xs text-text-secondary">Next backup {when(status.next_run)}.</p>
|
||||
{/if}
|
||||
</div>
|
||||
<div class="flex flex-wrap items-center gap-2 px-4 py-3">
|
||||
<button
|
||||
type="button"
|
||||
class={button}
|
||||
disabled={starting || !!status.running_since || !ready}
|
||||
title={ready ? undefined : 'Set up where backups go first'}
|
||||
onclick={backUpNow}
|
||||
>
|
||||
{status.running_since ? 'Backing up…' : 'Back up now'}
|
||||
</button>
|
||||
</div>
|
||||
</SettingsGroup>
|
||||
|
||||
{#if status.music}
|
||||
{@const m = status.music}
|
||||
<SettingsGroup title="Music" id="backup-music-progress">
|
||||
<div class="space-y-1 px-4 py-3" data-testid="backup-music">
|
||||
<p class="text-sm text-text-primary">
|
||||
{#if m.total_files === 0}
|
||||
No music in the library yet.
|
||||
{:else if m.sent_files === m.total_files}
|
||||
All {plural(m.total_files, 'file', 'files')} backed up · {formatBytes(m.sent_bytes)}
|
||||
{:else}
|
||||
{m.sent_files} of {plural(m.total_files, 'file', 'files')} backed up · {formatBytes(m.sent_bytes)} of
|
||||
{formatBytes(m.total_bytes)}
|
||||
{/if}
|
||||
</p>
|
||||
{#if m.sent_files + m.refused_files < m.total_files && m.total_files > 0}
|
||||
<p class="text-xs text-text-secondary">The rest goes with the next backups.</p>
|
||||
{/if}
|
||||
</div>
|
||||
{#if m.refused_files > 0}
|
||||
<div class="px-4 pb-3">
|
||||
<p class="text-sm text-warning-fg">
|
||||
The destination refused {plural(m.refused_files, 'file', 'files')}. Each goes again once it changes, or
|
||||
once you save the settings.
|
||||
</p>
|
||||
<ul class="mt-1 space-y-1 text-xs" aria-label="Refused files">
|
||||
{#each m.refused as f (f.path)}
|
||||
<li>
|
||||
<span class="block break-all font-mono text-text-primary">{f.path}</span>
|
||||
<span class="block text-text-secondary">{f.error} · {when(f.at)}</span>
|
||||
</li>
|
||||
{/each}
|
||||
</ul>
|
||||
</div>
|
||||
{/if}
|
||||
</SettingsGroup>
|
||||
{/if}
|
||||
|
||||
{#if status.runs.length > 0}
|
||||
<SettingsGroup title="Latest backups" id="backup-runs">
|
||||
<ul class="divide-y divide-border text-sm" aria-label="Latest backups">
|
||||
{#each status.runs as run (run.id)}
|
||||
<li class="flex items-start justify-between gap-4 px-4 py-2">
|
||||
<span class="min-w-0">
|
||||
<span class="block text-text-primary">{when(run.started_at)}</span>
|
||||
<span class="block break-all text-xs text-text-secondary">
|
||||
{run.trigger === 'manual' ? 'Started by hand' : 'Scheduled'}
|
||||
{#if run.status === 'succeeded'}· {run.location} · {formatBytes(run.bytes)}{/if}
|
||||
{#if run.status === 'failed'}· {run.error}{/if}
|
||||
</span>
|
||||
{#if run.music_files > 0}
|
||||
<span class="block text-xs text-text-secondary">
|
||||
Music: {plural(run.music_files, 'file', 'files')} · {formatBytes(run.music_bytes)}
|
||||
</span>
|
||||
{/if}
|
||||
{#if run.music_note}
|
||||
<span class="block text-xs text-text-secondary">{run.music_note}</span>
|
||||
{/if}
|
||||
</span>
|
||||
<span
|
||||
class="shrink-0 text-xs {run.status === 'succeeded'
|
||||
? 'text-success-fg'
|
||||
: run.status === 'failed'
|
||||
? 'text-error-fg'
|
||||
: 'text-text-secondary'}"
|
||||
>
|
||||
{run.status === 'succeeded' ? 'Saved' : run.status === 'failed' ? 'Failed' : 'Running'}
|
||||
</span>
|
||||
</li>
|
||||
{/each}
|
||||
</ul>
|
||||
</SettingsGroup>
|
||||
{/if}
|
||||
{/if}
|
||||
{/snippet}
|
||||
{#snippet side()}
|
||||
{#if status}
|
||||
<BackupSettings config={status.config} onsaved={(s) => (status = s)} />
|
||||
{/if}
|
||||
{/snippet}
|
||||
</SettingsColumns>
|
||||
{/if}
|
||||
@@ -0,0 +1,248 @@
|
||||
import { afterEach, describe, expect, test, vi } from 'vitest';
|
||||
import { render, screen, fireEvent, waitFor } from '@testing-library/svelte';
|
||||
import type { BackupStatus } from '#lib/api/admin.js';
|
||||
|
||||
vi.mock('#lib/api/admin.js', () => ({
|
||||
getBackups: vi.fn(),
|
||||
saveBackups: vi.fn(),
|
||||
testBackups: vi.fn(),
|
||||
runBackup: vi.fn()
|
||||
}));
|
||||
|
||||
vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() }));
|
||||
|
||||
import BackupPanel from './BackupPanel.svelte';
|
||||
import { getBackups, runBackup, saveBackups, testBackups } from '#lib/api/admin.js';
|
||||
import { pushToast } from '#lib/stores/toast.svelte.js';
|
||||
|
||||
const base: BackupStatus = {
|
||||
config: {
|
||||
enabled: true,
|
||||
destination: 's3',
|
||||
folder: '',
|
||||
endpoint: 'https://s3.example.com',
|
||||
bucket: 'minstrel',
|
||||
access_key: 'ak',
|
||||
has_secret_key: true,
|
||||
run_hour: 3,
|
||||
upload_limit_kib: 4096,
|
||||
include_music: false
|
||||
},
|
||||
state: 'ok',
|
||||
problems: [],
|
||||
running_since: null,
|
||||
last_success: '2026-11-02T03:00:00Z',
|
||||
failing_since: null,
|
||||
next_run: '2026-11-03T03:00:00Z',
|
||||
run_limit_seconds: 1800,
|
||||
runs: [
|
||||
{
|
||||
id: 'r1',
|
||||
trigger: 'schedule',
|
||||
status: 'succeeded',
|
||||
started_at: '2026-11-02T03:00:00Z',
|
||||
finished_at: '2026-11-02T03:00:09Z',
|
||||
location: 'database/minstrel-20261102T030000Z.tar',
|
||||
bytes: 12_000_000,
|
||||
error: '',
|
||||
music_files: 0,
|
||||
music_bytes: 0,
|
||||
music_note: ''
|
||||
}
|
||||
],
|
||||
music: null
|
||||
};
|
||||
|
||||
function status(over: Partial<BackupStatus> = {}): BackupStatus {
|
||||
return { ...base, ...over, config: { ...base.config, ...over.config } };
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
async function renderPanel(s: BackupStatus = base) {
|
||||
vi.mocked(getBackups).mockResolvedValue(s);
|
||||
render(BackupPanel);
|
||||
await screen.findByTestId('backup-verdict');
|
||||
}
|
||||
|
||||
const verdict = () => screen.getByTestId('backup-verdict').textContent?.replace(/\s+/g, ' ') ?? '';
|
||||
|
||||
describe('BackupPanel', () => {
|
||||
test('leads with the verdict and lists the latest runs', async () => {
|
||||
await renderPanel();
|
||||
expect(verdict()).toMatch(/^ ?Backed up /);
|
||||
expect(verdict()).toMatch(/Next backup /);
|
||||
const runs = screen.getByRole('list', { name: /latest backups/i });
|
||||
expect(runs.textContent).toMatch(/Scheduled · database\/minstrel-20261102T030000Z\.tar · 12\.0 MB/);
|
||||
expect(runs.textContent).toMatch(/Saved/);
|
||||
});
|
||||
|
||||
test('a failing backup says since when and why', async () => {
|
||||
await renderPanel(
|
||||
status({
|
||||
state: 'failing',
|
||||
failing_since: '2026-11-02T03:00:00Z',
|
||||
runs: [{ ...base.runs[0], status: 'failed', error: 'HTTP 403 SignatureDoesNotMatch: the secret key is wrong' }]
|
||||
})
|
||||
);
|
||||
expect(verdict()).toMatch(/Backups failing since /);
|
||||
expect(verdict()).toMatch(/the secret key is wrong/);
|
||||
});
|
||||
|
||||
test('switched on but not set up: names what is missing and holds Back up now', async () => {
|
||||
await renderPanel(status({ state: 'incomplete', problems: ['Enter the secret key.'], next_run: null }));
|
||||
expect(verdict()).toMatch(/not set up yet/);
|
||||
expect(verdict()).toMatch(/Enter the secret key\./);
|
||||
expect(screen.getByRole('button', { name: /back up now/i })).toHaveProperty('disabled', true);
|
||||
});
|
||||
|
||||
test('switched off reads as off, without nagging about the destination', async () => {
|
||||
await renderPanel(status({ state: 'off', config: { ...base.config, enabled: false }, problems: ['Enter the secret key.'] }));
|
||||
expect(verdict()).toMatch(/^ ?Off\./);
|
||||
expect(verdict()).not.toMatch(/secret key/);
|
||||
});
|
||||
|
||||
test('Back up now starts a run and polls only while it runs', async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.mocked(runBackup).mockResolvedValue();
|
||||
const running = status({ running_since: '2026-11-02T10:00:00Z' });
|
||||
vi.mocked(getBackups).mockResolvedValueOnce(base).mockResolvedValueOnce(running).mockResolvedValueOnce(running).mockResolvedValue(base);
|
||||
render(BackupPanel);
|
||||
await vi.waitFor(() => screen.getByTestId('backup-verdict'));
|
||||
|
||||
await fireEvent.click(screen.getByRole('button', { name: /back up now/i }));
|
||||
await vi.waitFor(() => expect(runBackup).toHaveBeenCalled());
|
||||
await vi.waitFor(() => expect(verdict()).toMatch(/Backing up, since /));
|
||||
expect(pushToast).toHaveBeenCalledWith('Backup started.');
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3000);
|
||||
await vi.advanceTimersByTimeAsync(3000);
|
||||
await vi.waitFor(() => expect(verdict()).toMatch(/Backed up /));
|
||||
const calls = vi.mocked(getBackups).mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(vi.mocked(getBackups).mock.calls.length).toBe(calls);
|
||||
});
|
||||
|
||||
test('a stuck run stops being polled after the run limit and a margin', async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.mocked(getBackups).mockResolvedValue(status({ running_since: '2026-11-02T10:00:00Z', run_limit_seconds: 60 }));
|
||||
render(BackupPanel);
|
||||
await vi.waitFor(() => screen.getByTestId('backup-verdict'));
|
||||
// 60 s limit + 5 min margin, and a little more.
|
||||
await vi.advanceTimersByTimeAsync(7 * 60 * 1000);
|
||||
const calls = vi.mocked(getBackups).mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(vi.mocked(getBackups).mock.calls.length).toBe(calls);
|
||||
});
|
||||
|
||||
test('never shows the stored secret key, and sends one only when typed', async () => {
|
||||
vi.mocked(saveBackups).mockResolvedValue(base);
|
||||
await renderPanel();
|
||||
const key = screen.getByLabelText(/secret key/i) as HTMLInputElement;
|
||||
expect(key.value).toBe('');
|
||||
expect(key.placeholder).toBe('Key saved');
|
||||
const save = screen.getByRole('button', { name: /^save$/i });
|
||||
expect(save).toHaveProperty('disabled', true);
|
||||
|
||||
await fireEvent.input(screen.getByLabelText(/^bucket$/i), { target: { value: 'music-backup' } });
|
||||
await fireEvent.click(save);
|
||||
await waitFor(() =>
|
||||
expect(saveBackups).toHaveBeenCalledWith({
|
||||
enabled: true,
|
||||
destination: 's3',
|
||||
folder: '',
|
||||
endpoint: 'https://s3.example.com',
|
||||
bucket: 'music-backup',
|
||||
access_key: 'ak',
|
||||
secret_key: '',
|
||||
run_hour: 3,
|
||||
upload_limit_kib: 4096,
|
||||
include_music: false
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
test('Test destination checks the form as typed and says what it found', async () => {
|
||||
vi.mocked(testBackups).mockResolvedValue({ backups: 3, latest: '2026-11-02T03:00:05Z' });
|
||||
await renderPanel();
|
||||
await fireEvent.click(screen.getByRole('button', { name: /test destination/i }));
|
||||
const out = await screen.findByTestId('backup-test');
|
||||
expect(out.textContent).toMatch(/Reached it\.\s+3 backups there, the newest from /);
|
||||
});
|
||||
|
||||
test('a refused test shows the destination’s answer', async () => {
|
||||
vi.mocked(testBackups).mockRejectedValue({
|
||||
code: 'destination_failed',
|
||||
message: 'HTTP 404 NoSuchBucket: there is no bucket by that name'
|
||||
});
|
||||
await renderPanel();
|
||||
await fireEvent.click(screen.getByRole('button', { name: /test destination/i }));
|
||||
expect((await screen.findByTestId('backup-test')).textContent).toMatch(
|
||||
/Couldn't reach the backup destination\. HTTP 404 NoSuchBucket/
|
||||
);
|
||||
});
|
||||
|
||||
test('with the music included, says how much of the library the backup holds', async () => {
|
||||
await renderPanel(
|
||||
status({
|
||||
config: { ...base.config, include_music: true },
|
||||
music: { total_files: 10, total_bytes: 300_000_000, sent_files: 4, sent_bytes: 120_000_000, refused_files: 0, refused: [] },
|
||||
runs: [
|
||||
{
|
||||
...base.runs[0],
|
||||
music_files: 4,
|
||||
music_bytes: 120_000_000,
|
||||
music_note: 'Music stopped for tonight after 6 hours; it carries on with the next backup.'
|
||||
}
|
||||
]
|
||||
})
|
||||
);
|
||||
const music = screen.getByTestId('backup-music').textContent?.replace(/\s+/g, ' ') ?? '';
|
||||
expect(music).toMatch(/4 of 10 files backed up · 120 MB of 300 MB/);
|
||||
expect(music).toMatch(/The rest goes with the next backups\./);
|
||||
const runs = screen.getByRole('list', { name: /latest backups/i }).textContent?.replace(/\s+/g, ' ') ?? '';
|
||||
expect(runs).toMatch(/Music: 4 files · 120 MB/);
|
||||
expect(runs).toMatch(/carries on with the next backup/);
|
||||
});
|
||||
|
||||
test('a file the destination refused is named with its answer', async () => {
|
||||
await renderPanel(
|
||||
status({
|
||||
config: { ...base.config, include_music: true },
|
||||
music: {
|
||||
total_files: 2,
|
||||
total_bytes: 20,
|
||||
sent_files: 1,
|
||||
sent_bytes: 10,
|
||||
refused_files: 1,
|
||||
refused: [{ path: '/music/A/one.flac', error: 'HTTP 403 AccessDenied: object is locked', at: '2026-11-02T03:00:09Z' }]
|
||||
}
|
||||
})
|
||||
);
|
||||
const refused = screen.getByRole('list', { name: /refused files/i });
|
||||
expect(refused.textContent).toMatch(/\/music\/A\/one\.flac/);
|
||||
expect(refused.textContent).toMatch(/object is locked/);
|
||||
expect(screen.getByTestId('backup-music').textContent).not.toMatch(/The rest goes/);
|
||||
});
|
||||
|
||||
test('the music switch is off by default and, switched on, says how a changed file is kept', async () => {
|
||||
vi.mocked(saveBackups).mockResolvedValue(base);
|
||||
await renderPanel();
|
||||
const music = screen.getByRole('switch', { name: /include the music/i }) as HTMLInputElement;
|
||||
expect(music.checked).toBe(false);
|
||||
expect(screen.queryByTestId('backup-music-retention')).toBeNull();
|
||||
await fireEvent.click(music);
|
||||
expect(screen.getByTestId('backup-music-retention').textContent).toMatch(/retention/);
|
||||
await fireEvent.click(screen.getByRole('button', { name: /^save$/i }));
|
||||
await waitFor(() => expect(saveBackups).toHaveBeenCalledWith(expect.objectContaining({ include_music: true })));
|
||||
});
|
||||
|
||||
test('the folder destination asks for a path, not a bucket', async () => {
|
||||
await renderPanel(status({ config: { ...base.config, destination: 'folder', folder: '/backups' } }));
|
||||
expect(screen.getByRole('textbox', { name: /^folder$/i })).toHaveProperty('value', '/backups');
|
||||
expect(screen.queryByLabelText(/^bucket$/i)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,206 @@
|
||||
<script lang="ts">
|
||||
import {
|
||||
saveBackups,
|
||||
testBackups,
|
||||
type BackupConfig,
|
||||
type BackupDestination,
|
||||
type BackupStatus,
|
||||
type BackupTestResult,
|
||||
type BackupUpdate
|
||||
} from '#lib/api/admin.js';
|
||||
import { errMessage } from '#lib/api/errors.js';
|
||||
import { pushToast } from '#lib/stores/toast.svelte.js';
|
||||
import SettingsGroup from '#lib/components/settings/SettingsGroup.svelte';
|
||||
import SettingRow from '#lib/components/settings/SettingRow.svelte';
|
||||
import Segmented from '#lib/components/settings/Segmented.svelte';
|
||||
import { plural } from '#lib/utils/plural.js';
|
||||
import { hourLabel, when } from './backupTime.js';
|
||||
|
||||
// Where the nightly backup goes and when. The form keeps its own drafts,
|
||||
// filled from the server on load and after each save, so a status poll
|
||||
// never overwrites what is being typed.
|
||||
|
||||
let { config, onsaved }: { config: BackupConfig; onsaved: (s: BackupStatus) => void } = $props();
|
||||
|
||||
// svelte-ignore state_referenced_locally
|
||||
let draft = $state(fromConfig(config));
|
||||
// Empty means "keep the stored key". The key itself never comes back.
|
||||
let secretKey = $state('');
|
||||
let saving = $state(false);
|
||||
let testing = $state(false);
|
||||
let check = $state<BackupTestResult | null>(null);
|
||||
let testError = $state('');
|
||||
|
||||
function fromConfig(c: BackupConfig) {
|
||||
return {
|
||||
enabled: c.enabled,
|
||||
destination: c.destination,
|
||||
folder: c.folder,
|
||||
endpoint: c.endpoint,
|
||||
bucket: c.bucket,
|
||||
access_key: c.access_key,
|
||||
run_hour: c.run_hour,
|
||||
upload_limit_kib: c.upload_limit_kib,
|
||||
include_music: c.include_music
|
||||
};
|
||||
}
|
||||
|
||||
const update = (): BackupUpdate => ({ ...draft, secret_key: secretKey.trim() });
|
||||
|
||||
const dirty = $derived(
|
||||
secretKey.trim() !== '' || JSON.stringify(draft) !== JSON.stringify(fromConfig(config))
|
||||
);
|
||||
|
||||
async function save() {
|
||||
saving = true;
|
||||
try {
|
||||
const s = await saveBackups(update());
|
||||
secretKey = '';
|
||||
draft = fromConfig(s.config);
|
||||
check = null;
|
||||
onsaved(s);
|
||||
pushToast('Backup settings saved.');
|
||||
} catch (e) {
|
||||
pushToast(errMessage(e), 'error');
|
||||
} finally {
|
||||
saving = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function test() {
|
||||
testing = true;
|
||||
check = null;
|
||||
testError = '';
|
||||
try {
|
||||
check = await testBackups(update());
|
||||
} catch (e) {
|
||||
testError = errMessage(e);
|
||||
} finally {
|
||||
testing = false;
|
||||
}
|
||||
}
|
||||
|
||||
const destinations: { value: BackupDestination; label: string }[] = [
|
||||
{ value: 's3', label: 'S3 bucket' },
|
||||
{ value: 'folder', label: 'Folder' }
|
||||
];
|
||||
const hours = Array.from({ length: 24 }, (_, h) => h);
|
||||
|
||||
const button =
|
||||
'rounded-md border border-border px-3 py-1.5 text-sm text-text-secondary hover:bg-surface-hover hover:text-text-primary disabled:opacity-50';
|
||||
const field = 'rounded-md border border-border bg-background px-3 py-1.5 text-sm';
|
||||
</script>
|
||||
|
||||
<SettingsGroup
|
||||
title="Backup settings"
|
||||
id="backup-settings"
|
||||
hint="Minstrel only ever adds backups. Thin out old ones with the destination's own retention."
|
||||
>
|
||||
<SettingRow label="Back up every night" for="backup-enabled" hint="Off until you switch it on.">
|
||||
<input
|
||||
id="backup-enabled"
|
||||
type="checkbox"
|
||||
role="switch"
|
||||
aria-checked={draft.enabled}
|
||||
bind:checked={draft.enabled}
|
||||
class="h-4 w-4 cursor-pointer accent-accent"
|
||||
/>
|
||||
</SettingRow>
|
||||
<SettingRow label="Destination">
|
||||
<Segmented
|
||||
name="backup-destination"
|
||||
label="Destination"
|
||||
options={destinations}
|
||||
value={draft.destination}
|
||||
onchange={(v) => (draft.destination = v)}
|
||||
/>
|
||||
</SettingRow>
|
||||
|
||||
{#if draft.destination === 'folder'}
|
||||
<SettingRow label="Folder" for="backup-folder" hint="A full path on the server, such as a mounted volume.">
|
||||
<input id="backup-folder" bind:value={draft.folder} placeholder="/backups" autocomplete="off" spellcheck="false" class="{field} w-56 font-mono" />
|
||||
</SettingRow>
|
||||
{:else}
|
||||
<SettingRow label="Endpoint" for="backup-endpoint" hint="The bucket's server, path-style.">
|
||||
<input
|
||||
id="backup-endpoint"
|
||||
bind:value={draft.endpoint}
|
||||
placeholder="https://s3.example.com"
|
||||
autocomplete="off"
|
||||
spellcheck="false"
|
||||
class="{field} w-56"
|
||||
/>
|
||||
</SettingRow>
|
||||
<SettingRow label="Bucket" for="backup-bucket">
|
||||
<input id="backup-bucket" bind:value={draft.bucket} placeholder="minstrel" autocomplete="off" spellcheck="false" class="{field} w-56" />
|
||||
</SettingRow>
|
||||
<SettingRow label="Access key" for="backup-access-key">
|
||||
<input id="backup-access-key" bind:value={draft.access_key} autocomplete="off" spellcheck="false" class="{field} w-56 font-mono" />
|
||||
</SettingRow>
|
||||
<SettingRow label="Secret key" for="backup-secret-key" hint="Kept encrypted on the server.">
|
||||
<input
|
||||
id="backup-secret-key"
|
||||
type="password"
|
||||
autocomplete="new-password"
|
||||
bind:value={secretKey}
|
||||
placeholder={config.has_secret_key ? 'Key saved' : 'Paste key'}
|
||||
class="{field} w-56 font-mono"
|
||||
/>
|
||||
</SettingRow>
|
||||
<SettingRow label="Upload limit" for="backup-limit" hint="KiB a second, so an upload leaves room on the connection. 0 is no limit.">
|
||||
<input id="backup-limit" type="number" min="0" step="256" bind:value={draft.upload_limit_kib} class="{field} w-28" />
|
||||
</SettingRow>
|
||||
{/if}
|
||||
|
||||
<SettingRow
|
||||
label="Include the music"
|
||||
for="backup-music"
|
||||
hint="Each file goes once, and again only when it changes. A large library takes several nights the first time."
|
||||
>
|
||||
<input
|
||||
id="backup-music"
|
||||
type="checkbox"
|
||||
role="switch"
|
||||
aria-checked={draft.include_music}
|
||||
bind:checked={draft.include_music}
|
||||
class="h-4 w-4 cursor-pointer accent-accent"
|
||||
/>
|
||||
</SettingRow>
|
||||
{#if draft.include_music}
|
||||
<p class="px-4 pb-2 text-xs text-text-secondary" data-testid="backup-music-retention">
|
||||
A changed file goes to the same name. On a bucket that keeps versions or locks objects, set its retention to
|
||||
keep only what you want.
|
||||
</p>
|
||||
{/if}
|
||||
|
||||
<SettingRow label="Time of day" for="backup-hour" hint="On your clock.">
|
||||
<select id="backup-hour" bind:value={draft.run_hour} class={field}>
|
||||
{#each hours as h (h)}
|
||||
<option value={h}>{hourLabel(h)}</option>
|
||||
{/each}
|
||||
</select>
|
||||
</SettingRow>
|
||||
|
||||
<div class="space-y-2 px-4 py-3">
|
||||
<div class="flex flex-wrap items-center justify-end gap-2">
|
||||
<button type="button" class={button} disabled={testing} onclick={test}>
|
||||
{testing ? 'Testing…' : 'Test destination'}
|
||||
</button>
|
||||
<button type="button" class={button} disabled={!dirty || saving} onclick={save}>
|
||||
{saving ? 'Saving…' : 'Save'}
|
||||
</button>
|
||||
</div>
|
||||
{#if testError}
|
||||
<p class="text-sm text-error-fg" role="alert" data-testid="backup-test">{testError}</p>
|
||||
{:else if check}
|
||||
<p class="text-sm text-success-fg" role="status" data-testid="backup-test">
|
||||
Reached it.
|
||||
{#if check.backups === 0}
|
||||
No backups there yet.
|
||||
{:else}
|
||||
{plural(check.backups, 'backup', 'backups')} there, the newest from {when(check.latest!)}.
|
||||
{/if}
|
||||
</p>
|
||||
{/if}
|
||||
</div>
|
||||
</SettingsGroup>
|
||||
@@ -0,0 +1,14 @@
|
||||
/** An exact moment in the viewer's own zone, with its date. */
|
||||
export function when(iso: string): string {
|
||||
return new Date(iso).toLocaleString(undefined, { dateStyle: 'medium', timeStyle: 'short' });
|
||||
}
|
||||
|
||||
/**
|
||||
* The server keeps the run hour in UTC; this is that hour on the viewer's
|
||||
* clock today. Today's offset is used, so across daylight saving the local
|
||||
* label moves while the run stays at the same UTC hour.
|
||||
*/
|
||||
export function hourLabel(utcHour: number, now: Date = new Date()): string {
|
||||
const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), utcHour));
|
||||
return d.toLocaleTimeString(undefined, { hour: 'numeric', minute: '2-digit' });
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
<script lang="ts">
|
||||
import { restoreBackup, type RestoreResult } from '#lib/auth/store.svelte.js';
|
||||
import { errCode } from '#lib/api/errors.js';
|
||||
import { plural } from '#lib/utils/plural.js';
|
||||
|
||||
// A new server can start from a backup instead of a new admin account. It
|
||||
// asks for the same setup token as the first account, since the backup's
|
||||
// admins become this server's.
|
||||
|
||||
let file = $state<File | null>(null);
|
||||
let setupToken = $state('');
|
||||
let restoring = $state(false);
|
||||
let error = $state<string | null>(null);
|
||||
let done = $state<RestoreResult | null>(null);
|
||||
|
||||
function messageFor(code: string): string {
|
||||
switch (code) {
|
||||
case 'setup_token_invalid':
|
||||
return "That setup token doesn't match. Copy it from the server log; it changes each time the server restarts.";
|
||||
case 'not_a_backup':
|
||||
return "That file isn't a Minstrel backup. Choose a minstrel-….tar file from the backup's database folder.";
|
||||
case 'newer_backup':
|
||||
return 'That backup was made by a newer Minstrel. Update this server first, then restore it.';
|
||||
case 'not_empty':
|
||||
return 'This server already has accounts, so it can’t take a backup.';
|
||||
case 'too_large':
|
||||
return 'That file is larger than a backup can be.';
|
||||
default:
|
||||
return 'The restore failed. Please try again.';
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSubmit(e: SubmitEvent) {
|
||||
e.preventDefault();
|
||||
if (restoring || !file) return;
|
||||
error = null;
|
||||
restoring = true;
|
||||
try {
|
||||
done = await restoreBackup(file, setupToken.trim());
|
||||
} catch (err: unknown) {
|
||||
error = messageFor(errCode(err));
|
||||
} finally {
|
||||
restoring = false;
|
||||
}
|
||||
}
|
||||
|
||||
const field = 'w-full rounded border border-border bg-background px-3 py-2 outline-hidden focus:border-accent';
|
||||
</script>
|
||||
|
||||
{#if done}
|
||||
<div class="space-y-3" role="status" data-testid="restore-done">
|
||||
<p class="text-sm text-text-primary">
|
||||
Restored {plural(done.accounts, 'account', 'accounts')}.
|
||||
{done.restarting
|
||||
? 'Minstrel is restarting to load them; in a few seconds, sign in with the password you had.'
|
||||
: 'Sign in with the password you had.'}
|
||||
</p>
|
||||
<p class="text-sm text-text-secondary">
|
||||
The library scan finds each music file and fetches the artwork and tags again.
|
||||
</p>
|
||||
<a href="/login" class="block w-full rounded bg-accent px-3 py-2 text-center font-medium text-background">Sign in</a>
|
||||
</div>
|
||||
{:else}
|
||||
<form class="space-y-4" onsubmit={handleSubmit}>
|
||||
<label class="block">
|
||||
<span class="mb-1 block text-sm text-text-secondary">Backup file</span>
|
||||
<input
|
||||
type="file"
|
||||
accept=".tar,application/x-tar"
|
||||
class="{field} text-sm"
|
||||
onchange={(e) => (file = (e.currentTarget as HTMLInputElement).files?.[0] ?? null)}
|
||||
/>
|
||||
<span class="mt-1 block text-xs text-text-secondary">
|
||||
A minstrel-….tar file from the backup's database folder. Copy the music back into the library folder first.
|
||||
</span>
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="mb-1 block text-sm text-text-secondary">Setup token</span>
|
||||
<input type="text" autocomplete="off" spellcheck="false" required class="{field} font-mono" bind:value={setupToken} />
|
||||
<span class="mt-1 block text-xs text-text-secondary">Printed in the server log (look for "setup_token").</span>
|
||||
</label>
|
||||
<button
|
||||
type="submit"
|
||||
class="w-full rounded bg-accent px-3 py-2 font-medium text-background disabled:opacity-60"
|
||||
aria-busy={restoring ? 'true' : undefined}
|
||||
disabled={restoring || !file}
|
||||
>
|
||||
{restoring ? 'Restoring…' : 'Restore'}
|
||||
</button>
|
||||
{#if error}
|
||||
<p role="alert" class="text-sm text-danger">{error}</p>
|
||||
{/if}
|
||||
</form>
|
||||
{/if}
|
||||
@@ -8,8 +8,8 @@ import { playlistTrackToRef } from './playlistTrackToRef';
|
||||
// current snapshot and resume.
|
||||
//
|
||||
// Single-instance variants (for_you, discover, deep_cuts, …) re-pull by
|
||||
// variant via the rotation-aware shuffle endpoint. Per-artist mixes
|
||||
// (songs_like_artist — one playlist per seed artist) can't be addressed by
|
||||
// variant via the rotation-aware shuffle endpoint. Kinds with several
|
||||
// playlists per user (songs_like_artist, genre_mix) can't be addressed by
|
||||
// variant alone, so they re-pull by playlist id. Mirrors the play routing
|
||||
// in PlaylistCard.
|
||||
|
||||
|
||||
@@ -72,5 +72,10 @@
|
||||
"block_not_found": "That release isn't set aside any more.",
|
||||
"not_running": "That download isn't running any more. Refresh the list.",
|
||||
"bad_lifecycle": "Seed ratio is 0 to 100, seed days 0 to 365, and stall hours 0 to 720.",
|
||||
"bad_upgrade_until": "Upgrade until has to be one of the qualities listed."
|
||||
"bad_upgrade_until": "Upgrade until has to be one of the qualities listed.",
|
||||
"invalid_backup_setting": "That backup setting can't be saved.",
|
||||
"destination_failed": "Couldn't reach the backup destination.",
|
||||
"backup_running": "A backup is already running.",
|
||||
"backup_not_ready": "Set up where backups go first.",
|
||||
"shutting_down": "The server is shutting down. Try again in a moment."
|
||||
}
|
||||
|
||||
@@ -50,6 +50,15 @@
|
||||
.filter((p) => p.system_variant === 'songs_like_artist')
|
||||
);
|
||||
|
||||
// Genre mixes: one "{Genre} mix" per genre the user listens to, on their
|
||||
// own row in name order so the shelf reads like a stable channel list.
|
||||
// Mirrors Android's buildGenreMixRow (HomeScreen.kt).
|
||||
const genreMixPlaylists = $derived(
|
||||
(systemPlaylistsQ.data?.owned ?? [])
|
||||
.filter((p) => p.system_variant === 'genre_mix')
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
);
|
||||
|
||||
// Secondary system kinds the server generates that don't get pinned
|
||||
// to fixed Home slots: surfaced after the Songs-like slots when they
|
||||
// exist, in server-registry order (internal/playlists/system.go).
|
||||
@@ -189,6 +198,25 @@
|
||||
</HorizontalScrollRow>
|
||||
</section>
|
||||
|
||||
<!-- Your genres: one mix per genre the user listens to. Hidden until the
|
||||
server has built at least one — a genre mix needs listening history
|
||||
or a library with genre tags, so there is no placeholder state. -->
|
||||
{#if genreMixPlaylists.length > 0}
|
||||
<section class="space-y-3">
|
||||
<HorizontalScrollRow
|
||||
rows={[genreMixPlaylists]}
|
||||
title="Your genres"
|
||||
ariaLabel="Your genres"
|
||||
>
|
||||
{#snippet item(playlist: Playlist)}
|
||||
<div class="w-56">
|
||||
<PlaylistCard {playlist} />
|
||||
</div>
|
||||
{/snippet}
|
||||
</HorizontalScrollRow>
|
||||
</section>
|
||||
{/if}
|
||||
|
||||
{#if query.isError}
|
||||
<ApiErrorBanner error={query.error} onRetry={query.refetch} />
|
||||
{:else if showSkeleton.value && !data}
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
{ href: '/admin/metadata', label: 'Metadata sources' },
|
||||
{ href: '/admin/email', label: 'Email & network' },
|
||||
{ href: '/admin/users', label: 'Users' },
|
||||
{ href: '/admin/backups', label: 'Backups' },
|
||||
{ href: '/admin/diagnostics', label: 'Diagnostics' },
|
||||
{ href: '/admin/tuning', label: 'Tuning' }
|
||||
]);
|
||||
|
||||
@@ -291,6 +291,7 @@ describe('admin layout', () => {
|
||||
['/admin/metadata', 'Metadata sources'],
|
||||
['/admin/email', 'Email & network'],
|
||||
['/admin/users', 'Users'],
|
||||
['/admin/backups', 'Backups'],
|
||||
['/admin/diagnostics', 'Diagnostics'],
|
||||
['/admin/tuning', 'Tuning']
|
||||
]);
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
<script lang="ts">
|
||||
import { pageTitle } from '#lib/branding.js';
|
||||
import BackupPanel from '#lib/components/admin/BackupPanel.svelte';
|
||||
|
||||
// Admin › Backups: Minstrel's own nightly backup, where it goes, and how
|
||||
// the latest runs went.
|
||||
</script>
|
||||
|
||||
<svelte:head><title>{pageTitle('Admin · Backups')}</title></svelte:head>
|
||||
|
||||
<h2 class="sr-only">Backups</h2>
|
||||
|
||||
<BackupPanel />
|
||||
@@ -154,6 +154,52 @@ describe('home Playlists section', () => {
|
||||
expect(within(songsSection).queryByTestId('playlist-placeholder-card')).toBeNull();
|
||||
});
|
||||
|
||||
test('Your genres row lists every genre mix in name order, out of Playlists', () => {
|
||||
const makeGenreMix = (id: string, name: string): Playlist => ({
|
||||
id,
|
||||
user_id: 'u1',
|
||||
owner_username: 'u1',
|
||||
name,
|
||||
description: '',
|
||||
is_public: false,
|
||||
kind: 'system',
|
||||
system_variant: 'genre_mix',
|
||||
refreshable: false,
|
||||
seed_artist_id: null,
|
||||
cover_url: '',
|
||||
track_count: 50,
|
||||
duration_sec: 3000,
|
||||
created_at: '2026-11-05T00:00:00Z',
|
||||
updated_at: '2026-11-05T00:00:00Z'
|
||||
});
|
||||
const owned = [
|
||||
makeGenreMix('g1', 'Shoegaze mix'),
|
||||
makeGenreMix('g2', 'Jazz mix'),
|
||||
makeGenreMix('g3', 'Dub mix')
|
||||
];
|
||||
(createPlaylistsQuery as unknown as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
(kind?: string) =>
|
||||
kind === 'system'
|
||||
? readable({ data: { owned, public: [] }, isPending: false, isError: false })
|
||||
: readable({ data: emptyPlaylistsResponse, isPending: false, isError: false })
|
||||
);
|
||||
const { container } = render(Page);
|
||||
|
||||
const genres = container.querySelector('[aria-label="Your genres"]') as HTMLElement;
|
||||
expect(genres).not.toBeNull();
|
||||
const names = within(genres)
|
||||
.getAllByText(/ mix$/)
|
||||
.map((el) => el.textContent?.trim());
|
||||
expect(names).toEqual(['Dub mix', 'Jazz mix', 'Shoegaze mix']);
|
||||
const playlistsSection = container.querySelector('[aria-label="Playlists"]') as HTMLElement;
|
||||
expect(within(playlistsSection).queryByText('Jazz mix')).toBeNull();
|
||||
});
|
||||
|
||||
test('Your genres row is absent until a genre mix exists', () => {
|
||||
const { container } = render(Page);
|
||||
expect(container.querySelector('[aria-label="Your genres"]')).toBeNull();
|
||||
});
|
||||
|
||||
test('renders secondary system kinds (deep_cuts / new_for_you) in the Playlists row', () => {
|
||||
// Web Home surfaces the 5 secondary
|
||||
// system kinds when generated. No placeholders for them — they
|
||||
|
||||
@@ -20,7 +20,8 @@
|
||||
removePlaylistTrack,
|
||||
reorderPlaylist,
|
||||
refreshSystem,
|
||||
systemShuffle
|
||||
systemShuffle,
|
||||
listPlaylists
|
||||
} from '#lib/api/playlists.js';
|
||||
import { qk } from '#lib/api/queries.js';
|
||||
import { user } from '#lib/auth/store.svelte.js';
|
||||
@@ -108,7 +109,7 @@
|
||||
refetch: systemPlaylistRefetch({
|
||||
variant,
|
||||
playlistId: id,
|
||||
perArtist: data.seed_artist_id != null
|
||||
perArtist: !data.refreshable
|
||||
})
|
||||
});
|
||||
} else {
|
||||
@@ -216,14 +217,24 @@
|
||||
|
||||
let reloadingStale = $state(false);
|
||||
async function onReloadStale() {
|
||||
const variant = playlistQuery?.data?.system_variant;
|
||||
if (!variant) return;
|
||||
const current = playlistQuery?.data;
|
||||
const variant = current?.system_variant;
|
||||
if (!current || !variant) return;
|
||||
reloadingStale = true;
|
||||
try {
|
||||
// The rebuild minted a new playlist id for this variant; resolve it and
|
||||
// navigate there so the page loads the fresh mix.
|
||||
// The rebuild minted a new playlist id; resolve it and navigate there
|
||||
// so the page loads the fresh mix. A kind with one playlist per user
|
||||
// resolves by kind. A kind with several (Songs like…, genre mixes)
|
||||
// resolves by kind and name — "Jazz mix" stays "Jazz mix" — and goes
|
||||
// Home when today's build no longer has that mix.
|
||||
if (current.refreshable) {
|
||||
const fresh = await systemShuffle(variant);
|
||||
await goto(`/playlists/${fresh.id}`);
|
||||
return;
|
||||
}
|
||||
const { owned } = await listPlaylists('system');
|
||||
const fresh = owned.find((p) => p.system_variant === variant && p.name === current.name);
|
||||
await goto(fresh ? `/playlists/${fresh.id}` : '/');
|
||||
} catch (e: unknown) {
|
||||
pushToast(`Couldn’t load the refreshed mix: ${errCode(e)}`, 'error');
|
||||
} finally {
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
import { getSetupStatus, register } from '#lib/auth/store.svelte.js';
|
||||
import { errCode } from '#lib/api/errors.js';
|
||||
import { rateLimitMessage } from '#lib/api/client.js';
|
||||
import RestoreForm from '#lib/components/auth/RestoreForm.svelte';
|
||||
|
||||
let username = $state('');
|
||||
let password = $state('');
|
||||
@@ -14,6 +15,8 @@
|
||||
// True on a brand-new server: the first account becomes admin and must
|
||||
// carry the setup token from the server log.
|
||||
let setupRequired = $state(false);
|
||||
// A new server can instead start from a backup of another Minstrel.
|
||||
let restoring = $state(false);
|
||||
|
||||
onMount(async () => {
|
||||
try {
|
||||
@@ -76,7 +79,24 @@
|
||||
|
||||
<main class="flex min-h-screen items-center justify-center bg-background text-text-primary">
|
||||
<div class="w-full max-w-sm rounded-lg border border-border bg-surface p-6 shadow-sm">
|
||||
<h1 class="mb-6 text-center text-2xl font-semibold">Create account</h1>
|
||||
<h1 class="mb-6 text-center text-2xl font-semibold">{restoring ? 'Restore a backup' : 'Create account'}</h1>
|
||||
{#if setupRequired}
|
||||
<p class="mb-4 text-center text-sm text-text-secondary">
|
||||
{#if restoring}
|
||||
<button type="button" class="text-accent-fg hover:underline" onclick={() => (restoring = false)}>
|
||||
Create a new admin account instead
|
||||
</button>
|
||||
{:else}
|
||||
Moving from another Minstrel?
|
||||
<button type="button" class="text-accent-fg hover:underline" onclick={() => (restoring = true)}>
|
||||
Restore a backup
|
||||
</button>
|
||||
{/if}
|
||||
</p>
|
||||
{/if}
|
||||
{#if restoring}
|
||||
<RestoreForm />
|
||||
{:else}
|
||||
<form class="space-y-4" onsubmit={handleSubmit}>
|
||||
<label class="block">
|
||||
<span class="mb-1 block text-sm text-text-secondary">Username</span>
|
||||
@@ -167,6 +187,7 @@
|
||||
<p role="alert" class="text-sm text-danger">{error}</p>
|
||||
{/if}
|
||||
</form>
|
||||
{/if}
|
||||
|
||||
<p class="mt-4 text-center text-sm text-text-secondary">
|
||||
Already have an account?
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { afterEach, describe, expect, test, vi } from 'vitest';
|
||||
import { render, screen, fireEvent, waitFor } from '@testing-library/svelte';
|
||||
import { render, screen, fireEvent, waitFor, within } from '@testing-library/svelte';
|
||||
|
||||
vi.mock('#lib/branding.js', () => ({
|
||||
pageTitle: (s: string) => `${s} · Minstrel`
|
||||
@@ -11,12 +11,13 @@ vi.mock('$app/navigation', () => ({
|
||||
|
||||
vi.mock('#lib/auth/store.svelte.js', () => ({
|
||||
register: vi.fn(),
|
||||
restoreBackup: vi.fn(),
|
||||
getSetupStatus: vi.fn().mockResolvedValue({ setup_required: false }),
|
||||
user: { value: null }
|
||||
}));
|
||||
|
||||
import RegisterPage from './+page.svelte';
|
||||
import { getSetupStatus, register } from '#lib/auth/store.svelte.js';
|
||||
import { getSetupStatus, register, restoreBackup } from '#lib/auth/store.svelte.js';
|
||||
import { goto } from '$app/navigation';
|
||||
|
||||
afterEach(() => {
|
||||
@@ -156,4 +157,42 @@ describe('/register page', () => {
|
||||
const link = screen.getByRole('link', { name: /sign in/i });
|
||||
expect(link).toHaveAttribute('href', '/login');
|
||||
});
|
||||
|
||||
test('only a new server offers to restore a backup', async () => {
|
||||
render(RegisterPage);
|
||||
await waitFor(() => expect(getSetupStatus).toHaveBeenCalled());
|
||||
expect(screen.queryByRole('button', { name: /restore a backup/i })).toBeNull();
|
||||
});
|
||||
|
||||
test('a new server restores a backup with the setup token, then sends you to sign in', async () => {
|
||||
(getSetupStatus as ReturnType<typeof vi.fn>).mockResolvedValue({ setup_required: true });
|
||||
(restoreBackup as ReturnType<typeof vi.fn>).mockResolvedValue({ accounts: 2, restarting: true });
|
||||
render(RegisterPage);
|
||||
await fireEvent.click(await screen.findByRole('button', { name: /restore a backup/i }));
|
||||
expect(screen.getByRole('heading', { name: /restore a backup/i })).toBeInTheDocument();
|
||||
expect(screen.queryByLabelText(/username/i)).toBeNull();
|
||||
|
||||
const file = new File(['tar'], 'minstrel-20261102T030000Z.tar', { type: 'application/x-tar' });
|
||||
await fireEvent.change(screen.getByLabelText(/backup file/i), { target: { files: [file] } });
|
||||
await fireEvent.input(screen.getByLabelText(/setup token/i), { target: { value: ' 0123abcd ' } });
|
||||
await fireEvent.click(screen.getByRole('button', { name: /^restore$/i }));
|
||||
|
||||
await waitFor(() => expect(restoreBackup).toHaveBeenCalledWith(file, '0123abcd'));
|
||||
const done = await screen.findByTestId('restore-done');
|
||||
expect(done.textContent).toMatch(/Restored 2 accounts\.\s+Minstrel is restarting to load them; in a few seconds, sign in with the password you had\./);
|
||||
expect(done.textContent).toMatch(/library scan finds each music file/);
|
||||
expect(within(done).getByRole('link', { name: /^sign in$/i })).toHaveAttribute('href', '/login');
|
||||
});
|
||||
|
||||
test('a file that is not a backup says so', async () => {
|
||||
(getSetupStatus as ReturnType<typeof vi.fn>).mockResolvedValue({ setup_required: true });
|
||||
(restoreBackup as ReturnType<typeof vi.fn>).mockRejectedValue({ code: 'not_a_backup', status: 400 });
|
||||
render(RegisterPage);
|
||||
await fireEvent.click(await screen.findByRole('button', { name: /restore a backup/i }));
|
||||
const file = new File(['x'], 'notes.txt');
|
||||
await fireEvent.change(screen.getByLabelText(/backup file/i), { target: { files: [file] } });
|
||||
await fireEvent.input(screen.getByLabelText(/setup token/i), { target: { value: 'tok' } });
|
||||
await fireEvent.click(screen.getByRole('button', { name: /^restore$/i }));
|
||||
expect((await screen.findByRole('alert')).textContent).toMatch(/isn't a Minstrel backup/);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user