Compare commits

..
Author SHA1 Message Date
bvandeusen 7529266f5b Merge pull request 'DLNA renderers, app updates told like Librarian, acquisition without Lidarr, Collection and admin rework (M502–M505, M542)' (#156) from dev into main
release / web (push) Successful in 1m29s
release / go (push) Successful in 1m49s
release / govulncheck (push) Successful in 26s
release / android (push) Successful in 6m19s
release / integration (push) Successful in 6m47s
release / Build signed APK (releases and dev) (push) Successful in 6m17s
release / Attach APK to the Release (tag releases only) (push) Successful in 3s
release / Build + push container image (push) Successful in 23s
release / Verify release artifacts (tag releases only) (push) Successful in 2s
2026-10-26 12:32:32 -04:00
bvandeusen 165c87047e Merge pull request 'Duplicates and video rips resolve themselves (M498)' (#155) from dev into main
release / Build signed APK (releases and dev) (push) Skipped
release / govulncheck (push) Successful in 17s
release / web (push) Successful in 1m20s
release / go (push) Successful in 1m36s
release / integration (push) Successful in 4m15s
release / android (push) Successful in 5m18s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 28s
release / Verify release artifacts (tag releases only) (push) Skipped
2026-10-08 22:30:10 -04:00
bvandeusen 05bc85f2b8 Merge dev: scroll-loaded lists, Liked tabs, missing files kept out of the queue (#5416, #5417, #5418)
release / Build signed APK (releases and dev) (push) Skipped
release / govulncheck (push) Successful in 14s
release / web (push) Successful in 1m4s
release / go (push) Successful in 1m21s
release / integration (push) Successful in 3m56s
release / android (push) Successful in 4m14s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 15s
release / Verify release artifacts (tag releases only) (push) Skipped
2026-10-08 20:08:04 -04:00
bvandeusen d1fe3758d5 Merge dev: Suspect sources report (#5410)
release / Build signed APK (releases and dev) (push) Skipped
release / web (push) Successful in 1m13s
release / go (push) Successful in 1m28s
release / govulncheck (push) Successful in 14s
release / integration (push) Successful in 4m46s
release / android (push) Successful in 5m29s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 30s
release / Verify release artifacts (tag releases only) (push) Skipped
2026-10-08 16:03:53 -04:00
bvandeusen 664b129947 Merge pull request 'One track-list shape across the web app, drag reorder that snaps (M494)' (#152) from dev into main
release / web (push) Successful in 1m17s
release / go (push) Successful in 1m36s
release / Build signed APK (releases and dev) (push) Skipped
release / govulncheck (push) Successful in 27s
release / integration (push) Successful in 4m20s
release / android (push) Successful in 4m45s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 1m14s
release / Verify release artifacts (tag releases only) (push) Skipped
2026-10-08 15:08:09 -04:00
bvandeusen c9fa18ecb7 Merge pull request 'Notifications inbox (M489), Discover taste-arm fix, web tooling majors' (#151) from dev into main
release / govulncheck (push) Successful in 16s
release / web (push) Successful in 1m8s
release / Build signed APK (releases and dev) (push) Skipped
release / go (push) Successful in 1m27s
release / integration (push) Successful in 5m1s
release / android (push) Successful in 5m35s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 1m34s
release / Verify release artifacts (tag releases only) (push) Skipped
2026-10-08 11:11:08 -04:00
bvandeusen 3077ed75fe Merge pull request 'Readable accent text app-wide; Android queue drag math under test' (#144) from dev into main
release / govulncheck (push) Successful in 18s
release / Build signed APK (releases and dev) (push) Skipped
release / web (push) Successful in 1m16s
release / go (push) Successful in 1m30s
release / integration (push) Successful in 4m2s
release / android (push) Successful in 4m20s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 14s
release / Verify release artifacts (tag releases only) (push) Skipped
2026-10-08 06:29:48 -04:00
bvandeusen 97e84a79eb Merge pull request 'Multi-value genres for FLAC/Ogg/MP4, readable tinted and error text, no missing-track seeds' (#143) from dev into main
release / Build signed APK (releases and dev) (push) Skipped
release / govulncheck (push) Successful in 31s
release / web (push) Successful in 1m19s
release / go (push) Successful in 1m41s
release / android (push) Successful in 4m44s
release / integration (push) Successful in 17m13s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 1m20s
release / Verify release artifacts (tag releases only) (push) Skipped
2026-10-07 23:53:09 -04:00
bvandeusen a29ceaaeab Merge pull request 'Similarity cache and round-robin artist arms (#5296, #5297)' (#142) from dev into main
release / android (push) Successful in 5m1s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 2m25s
release / Verify release artifacts (tag releases only) (push) Skipped
release / go (push) Successful in 2m28s
release / web (push) Successful in 57s
release / govulncheck (push) Successful in 19s
release / Build signed APK (releases and dev) (push) Skipped
release / integration (push) Successful in 5m9s
2026-10-07 20:02:59 -04:00
bvandeusen 54b2720d76 Merge pull request 'Fold stale missing tracks into their on-disk replacement (M485)' (#141) from dev into main
release / Build signed APK (releases and dev) (push) Skipped
release / govulncheck (push) Successful in 15s
release / web (push) Successful in 1m2s
release / go (push) Successful in 1m20s
release / Build + push container image (push) Successful in 14s
release / Verify release artifacts (tag releases only) (push) Skipped
release / integration (push) Successful in 3m51s
release / android (push) Successful in 4m19s
release / Attach APK to the Release (tag releases only) (push) Skipped
2026-10-07 18:17:51 -04:00
bvandeusen fe142e3451 Merge dev: request completion fix, x/text bump, full-logo tab icon (#140)
release / govulncheck (push) Successful in 22s
release / Build signed APK (releases and dev) (push) Skipped
release / web (push) Successful in 1m54s
release / go (push) Successful in 2m22s
release / integration (push) Successful in 5m7s
release / android (push) Successful in 5m47s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 17s
release / Verify release artifacts (tag releases only) (push) Skipped
2026-10-07 15:35:01 -04:00
bvandeusen 3bdfa94f39 Merge dev: M483 release-group ids for Lidarr (#139)
release / integration (push) Successful in 3m46s
release / android (push) Successful in 4m24s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build signed APK (releases and dev) (push) Skipped
release / govulncheck (push) Successful in 17s
release / web (push) Successful in 1m1s
release / go (push) Successful in 1m21s
release / Build + push container image (push) Successful in 16s
release / Verify release artifacts (tag releases only) (push) Skipped
2026-10-07 12:29:12 -04:00
bvandeusen 22ea27efff Merge pull request 'Lidarr: album adds that Lidarr accepts, artist monitoring that sticks' (#138) from dev into main
release / govulncheck (push) Successful in 27s
release / Build signed APK (releases and dev) (push) Skipped
release / web (push) Successful in 1m21s
release / go (push) Successful in 1m48s
release / integration (push) Successful in 4m24s
release / android (push) Successful in 5m16s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 25s
2026-10-07 11:08:18 -04:00
bvandeusen 09bd71629a Merge pull request 'M401: AcoustID recording-id lookup' (#137) from dev into main
release / Build signed APK (releases and dev) (push) Skipped
release / govulncheck (push) Successful in 34s
release / web (push) Successful in 1m6s
release / go (push) Successful in 1m24s
release / integration (push) Successful in 4m3s
release / android (push) Successful in 5m17s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 1m26s
release / Verify release artifacts (tag releases only) (push) Skipped
2026-10-07 09:50:11 -04:00
bvandeusen c83670d216 Merge pull request 'M464 loudness leveling (steps 4–8), APK distribution and security baseline adoptions' (#136) from dev into main
release / web (push) Successful in 1m41s
release / go (push) Successful in 2m0s
release / Build signed APK (releases and dev) (push) Skipped
release / govulncheck (push) Successful in 21s
release / integration (push) Successful in 6m6s
release / android (push) Successful in 7m23s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 15s
release / Verify release artifacts (tag releases only) (push) Skipped
2026-10-06 23:11:35 -04:00
bvandeusen 1e9aed214b Merge pull request 'M462 security hardening and M464 loudness analysis (steps 1–3)' (#135) from dev into main
release / go (push) Successful in 2m19s
release / web (push) Successful in 1m43s
release / govulncheck (push) Successful in 35s
release / Build signed APK (releases and dev) (push) Skipped
release / android (push) Successful in 6m10s
release / integration (push) Successful in 19m35s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 2m24s
release / Verify release artifacts (tag releases only) (push) Skipped
2026-10-06 14:19:58 -04:00
bvandeusen 3638d1d822 Merge pull request 'M400: acoustic duplicate detection, history-preserving merge, and fingerprinting settings' (#134) from dev into main
release / Build signed APK (releases and dev) (push) Skipped
release / Build + push container image (push) Successful in 1m32s
release / Verify release artifacts (tag releases only) (push) Skipped
test-go / test (push) Successful in 1m46s
test-go / integration (push) Successful in 4m19s
test-web / test (push) Successful in 33s
2026-09-11 20:56:55 -04:00
bvandeusen 18618bd135 Backfill fingerprints for the existing library (M400 #3908) (#133)
release / Build signed APK (releases and dev) (push) Skipped
release / Build + push container image (push) Successful in 16s
release / Verify release artifacts (tag releases only) (push) Skipped
test-web / test (push) Successful in 49s
test-go / test (push) Successful in 1m11s
test-go / integration (push) Successful in 3m16s
2026-09-11 15:34:19 -04:00
bvandeusen d2985f3841 Acoustic fingerprints at ingest, a delete that can't lose history, and a reproducible candidate draw (#132)
release / Build signed APK (releases and dev) (push) Skipped
release / Build + push container image (push) Successful in 20s
release / Verify release artifacts (tag releases only) (push) Skipped
test-web / test (push) Successful in 1m0s
test-go / test (push) Successful in 1m20s
test-go / integration (push) Successful in 3m35s
android / Build + lint + test (push) Successful in 4m48s
2026-09-11 14:43:00 -04:00
bvandeusen f70df9f827 Recommendation relevance, the rollback unit, and a version that names what shipped (#131)
test-web / test (push) Successful in 1m5s
test-go / test (push) Successful in 1m33s
test-go / integration (push) Successful in 4m25s
android / Build + lint + test (push) Successful in 5m33s
release / Build signed APK (releases and dev) (push) Successful in 4m38s
release / Build + push container image (push) Successful in 16s
release / Verify release artifacts (tag releases only) (push) Successful in 2s
2026-09-10 23:37:57 -04:00
bvandeusen 439c8625d5 Fix the rebundle step: a non-matching grep must not kill it (#130)
test-go / test (push) Successful in 59s
test-go / integration (push) Successful in 3m1s
release / Build signed APK (releases and dev) (push) Successful in 4m41s
release / Build + push container image (push) Successful in 14s
release / Verify release artifacts (tag releases only) (push) Successful in 2s
2026-09-10 10:49:40 -04:00
bvandeusen 1d67c160b2 Versioning rework, a dev channel, and the miniplayer gap (#129)
release / Build signed APK (releases and dev) (push) Skipped
android / Build + lint + test (push) Failing after 4s
release / Build + push container image (push) Failing after 5s
release / Verify release artifacts (tag releases only) (push) Skipped
test-go / test (push) Successful in 1m24s
test-go / integration (push) Successful in 6m25s
2026-09-10 08:33:26 -04:00
bvandeusen 237380b122 New brand mark, and both clients stop fetching their fonts at runtime (#128)
test-web / test (push) Successful in 54s
android / Build + lint + test (push) Successful in 5m36s
release / Build signed APK (tag releases only) (push) Successful in 4m36s
release / Build + push container image (push) Successful in 24s
release / Verify release artifacts (tag releases only) (push) Successful in 2s
2026-09-09 14:36:15 -04:00
bvandeusen 4f077736b6 Merge PR #127: Sonos queue verification, cast double-download fix, stutter instrumentation
android / Build + lint + test (push) Successful in 4m16s
release / Build signed APK (tag releases only) (push) Successful in 5m25s
release / Build + push container image (push) Successful in 1m16s
release / Verify release artifacts (tag releases only) (push) Successful in 1s
2026-08-18 10:50:10 -04:00
bvandeusen 727f68950e Merge PR #126: missing-file lifecycle, UPnP stall recovery, Android browse parity, Flutter client removed
release / Build signed APK (tag releases only) (push) Skipped
test-web / test (push) Successful in 1m5s
test-go / test (push) Successful in 1m26s
release / Build + push container image (push) Successful in 1m38s
release / Verify release artifacts (tag releases only) (push) Skipped
android / Build + lint + test (push) Successful in 4m54s
test-go / integration (push) Successful in 5m19s
2026-08-17 16:28:13 -04:00
bvandeusen aa9f534f3c Merge pull request 'Genre index: sort A–Z, and repair casing damage at scan time' (#124) from dev into main
test-web / test (push) Successful in 49s
test-go / test (push) Successful in 1m9s
test-go / integration (push) Successful in 5m1s
release / Build signed APK (tag releases only) (push) Successful in 4m2s
release / Build + push container image (push) Successful in 14s
release / Verify release artifacts (tag releases only) (push) Successful in 2s
2026-08-07 21:40:33 -04:00
bvandeusen 011b4d9a9c Merge pull request 'ci(release): verify a tag release actually shipped its artifacts' (#123) from dev into main
release / Build signed APK (tag releases only) (push) Successful in 3m38s
release / Build + push container image (push) Successful in 1m31s
release / Verify release artifacts (tag releases only) (push) Successful in 2s
2026-08-07 08:32:48 -04:00
bvandeusen d5aa081157 Merge pull request 'Recommendation metrics: publish the margin of error on every delta' (#122) from dev into main
test-web / test (push) Successful in 52s
test-go / test (push) Successful in 1m10s
test-go / integration (push) Successful in 4m53s
release / Build signed APK (tag releases only) (push) Successful in 3m57s
release / Build + push container image (push) Successful in 1m37s
2026-08-06 21:50:41 -04:00
bvandeusen a99f855e98 Merge pull request 'Missing files: detect them, stop offering them, and follow them when they move' (#121) from dev into main
test-go / test (push) Successful in 56s
test-go / integration (push) Successful in 5m0s
release / Build signed APK (tag releases only) (push) Successful in 4m14s
release / Build + push container image (push) Successful in 15s
2026-08-06 20:40:39 -04:00
bvandeusen 7e4727fc49 Merge pull request 'Genre tags: read multi-value frames correctly, and repair existing rows' (#120) from dev into main
test-go / test (push) Successful in 57s
test-go / integration (push) Successful in 4m57s
release / Build signed APK (tag releases only) (push) Successful in 4m23s
release / Build + push container image (push) Successful in 1m39s
2026-08-05 22:10:41 -04:00
bvandeusen 1b7fa635d8 Merge pull request 'Silent self-update, active sessions with real client IPs, genre/year browsing, handoff fix' (#119) from dev into main
test-web / test (push) Successful in 1m3s
test-go / test (push) Successful in 1m13s
test-go / integration (push) Successful in 5m29s
android / Build + lint + test (push) Successful in 5m34s
release / Build signed APK (tag releases only) (push) Successful in 5m5s
release / Build + push container image (push) Successful in 16s
2026-08-05 15:14:48 -04:00
bvandeusen 57d2299180 Merge pull request 'Queue row gestures: album art as grab surface + swipe-to-remove' (#118) from dev into main
test-web / test (push) Successful in 48s
android / Build + lint + test (push) Successful in 5m15s
release / Build signed APK (tag releases only) (push) Successful in 4m38s
release / Build + push container image (push) Successful in 1m48s
2026-08-04 11:37:30 -04:00
bvandeusen fa7ea41ccf Merge pull request 'Minstrel gets a mark — favicon, header lockup, Android adaptive icon' (#117) from dev into main
test-web / test (push) Successful in 47s
android / Build + lint + test (push) Successful in 4m37s
release / Build signed APK (tag releases only) (push) Successful in 8m36s
release / Build + push container image (push) Successful in 1m37s
2026-08-03 20:52:27 -04:00
bvandeusen 324059b2bd Merge pull request 'Discover request surface — taste-aware, rotating, snoozable, tag-targeted (milestone #268)' (#116) from dev into main
test-web / test (push) Successful in 1m5s
test-go / test (push) Successful in 1m30s
android / Build + lint + test (push) Successful in 5m1s
test-go / integration (push) Successful in 5m29s
release / Build signed APK (tag releases only) (push) Successful in 4m21s
release / Build + push container image (push) Successful in 17s
2026-08-03 08:38:24 -04:00
bvandeusen 1138d75a45 Merge pull request 'Playlist-track atomic replace + ci-requirements true-up' (#115) from dev into main
release / Build signed APK (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 1m33s
android / Build + lint + test (push) Successful in 4m30s
2026-08-01 12:23:37 -04:00
29 changed files with 207 additions and 1610 deletions
+102 -39
View File
@@ -2,9 +2,9 @@ name: release
# Builds and pushes the minstrel container image to the Gitea registry.
#
# push to dev → :dev (freshly-built dev APK bundled)
# push to main → :latest + :<sha> (freshly-built stable APK bundled)
# push tag vYYYY.MM.DD.HHMM → :latest (fresh APK bundled, and attached to the Release)
# push to dev → :dev (freshly-built dev APK bundled)
# push to main → :latest + :<sha> (latest-release APK bundled)
# push tag vYYYY.MM.DD.HHMM → :latest (fresh APK bundled)
# workflow_dispatch → manual trigger (same rules based on the ref)
#
# That is the whole tag map.
@@ -41,12 +41,6 @@ name: release
# image in the same run from the same commit, and the channel cannot go
# stale against its own artifact.
#
# Every branch build carries the app built from its own commit, so a merge to
# main is all it takes to put a new app in front of stable phones: `:latest`
# serves it from /api/client/apk and the in-app update offers it. A release
# is an optional bookmark — a named point in history with the APK attached
# for download — never a step the app has to wait for.
#
# Release model: the tag IS the artifact's version name with a `v` in front.
# `v2026.09.10.1432` and `2026.09.10.1432` are the same string, derived from
# the tagged commit's UTC timestamp — so there is no mismatch to reconcile
@@ -62,14 +56,20 @@ name: release
# :latest is updated by every main push AND every tag push, so it always
# reflects the newest blessed image.
#
# APK pipeline: on every push — dev, main and tags alike — the
# android-release job builds + signs the Android APK and uploads it as a
# workflow artifact. The image-release job declares `needs: android-release`
# and requires its success, so the docker image cannot start building until
# the APK is guaranteed-ready — no polling, no race, no silent-failure mode,
# and no image that moves a channel without an app inside it. Attaching the
# APK to the gitea Release is its own job (release-assets), tags only, behind
# the test gate below.
# APK pipeline: on tag pushes the android-release job builds + signs the
# Android APK and uploads it as a workflow artifact. The image-release
# job declares `needs: android-release`, so the docker image cannot
# start building until the APK is guaranteed-ready — no polling, no
# race, no silent-failure mode. Attaching the APK to the gitea Release is
# its own job (release-assets), behind the test gate below.
#
# :latest always carries an APK. Because every main push also moves
# :latest (not just tags), a main build with no APK would silently strip
# the in-app update channel off :latest until the next release. So on
# non-tag builds image-release pulls the MOST RECENT release's signed APK
# AND the version sidecar published beside it — the recorded values, not
# recomputed ones — so no rebuild is needed, just a rebundle. Tag builds
# keep bundling their own freshly-built APK.
#
# THE GATE. Every verifying lane lives in this file —
# Go (vet, lint, short tests), the Postgres integration suite, the web app
@@ -400,14 +400,11 @@ jobs:
# ------------------------------------------------------------ artifacts --
android-release:
name: Build signed APK
# Builds on every ref, so each channel's image carries the app from its
# own commit: `dev` for the test channel, `main` for stable phones, and a
# tag for the copy attached to its Release.
if: >-
startsWith(github.ref, 'refs/tags/v')
|| github.ref == 'refs/heads/dev'
|| github.ref == 'refs/heads/main'
name: Build signed APK (releases and dev)
# Also builds on `dev`, which is what makes a test channel possible at
# all: without it the only way to get a build onto a phone is to cut a
# release, which quietly turns `main` into the staging area.
if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev'
runs-on: flutter-ci
container:
image: git.fabledsword.com/bvandeusen/ci-android:36
@@ -604,9 +601,9 @@ jobs:
path: release-apk/
- name: Attach APK to gitea Release
# Tag releases only. A branch build has no Release to hang assets on
# and does not need one — its image bundles the APK, and the server
# serves it from /api/client/apk.
# Tag releases only. A dev build has no Release to hang assets on and
# does not need one — the :dev image bundles the APK, and the server
# serves it from /api/client/apk like any other.
shell: bash
env:
CI_TOKEN: ${{ secrets.CI_TOKEN }}
@@ -670,9 +667,10 @@ jobs:
image-release:
name: Build + push container image
# Every lane must have SUCCEEDED, each named here, and so must the APK
# build: every ref bundles the APK built from its own commit, so an
# image without one would move a channel with no app for phones to take.
# Every lane must have SUCCEEDED, each named here. Then the
# APK: tag and dev pushes build one and it must have succeeded; main
# pushes skip android-release and bundle the latest release's APK
# instead, so for main alone a skipped android-release is expected.
needs: [go, integration, web, android, govulncheck, android-release]
if: >-
${{
@@ -682,7 +680,8 @@ jobs:
&& needs.web.result == 'success'
&& needs.android.result == 'success'
&& needs.govulncheck.result == 'success'
&& needs.android-release.result == 'success' }}
&& (needs.android-release.result == 'success'
|| (needs.android-release.result == 'skipped' && github.ref == 'refs/heads/main')) }}
runs-on: go-ci
container:
image: git.fabledsword.com/bvandeusen/ci-go:1.26
@@ -748,10 +747,9 @@ jobs:
# publish the moving channel tag and never a commit-addressable
# one.
#
# :latest moves here so the server hands out the same APK build
# that is attached to the Release: a phone that installed from the
# Release page and one that updated in-app hold one build, with one
# ordering key. Cut a release on main's tip, as :latest follows it.
# :latest must move here rather than waiting for the next main
# push, or the channel would carry the PREVIOUS release's APK
# indefinitely — a channel that cannot refresh itself.
CHANNEL=stable
echo "args=-t ${IMAGE}:latest" >> "$GITHUB_OUTPUT"
echo "::notice::Release build ${VERSION}: refreshing :latest around the new APK"
@@ -784,8 +782,12 @@ jobs:
| docker login git.fabledsword.com -u "${{ github.actor }}" --password-stdin
- name: Download signed APK artifact
# android-release just produced this, on every ref.
if: steps.guard.outputs.ready == 'true'
# Tag and dev pushes — android-release just produced this. Only `main`
# takes the "Bundle latest release APK" path below, because it is the
# one ref that moves a channel without building an APK of its own.
if: >-
steps.guard.outputs.ready == 'true' &&
(startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev')
# Consuming half of the pair: stock download-artifact, which works here for
# the same reason as the upload (gitea/runner 3.x edits the GHES refusal
# out of the bundle). v8 runs on node24, which every CI-runner image
@@ -796,7 +798,9 @@ jobs:
path: client/
- name: Stage bundled APK + version sidecar
if: steps.guard.outputs.ready == 'true'
if: >-
steps.guard.outputs.ready == 'true' &&
(startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev')
shell: bash
env:
# All three pulled from android-release's outputs so the sidecar the
@@ -817,6 +821,65 @@ jobs:
cat client/minstrel.apk.version
ls -lh client/
- name: Bundle latest release APK (non-tag :latest builds)
# Main pushes don't build an APK, but they DO move :latest — so
# without this the in-app update channel would vanish from :latest
# until the next tag. Pull the most-recent release's signed APK and
# the sidecar published beside it, so what the server reports is what
# that build actually recorded rather than something re-derived here.
# Degrades to an empty client/ (404 update channel) — never a wrong
# version — if no release or APK asset can be resolved. That
# degradation only actually works because the greps below carry
# `|| true`; under the runner's default pipefail a non-matching grep
# kills the step instead of falling through to the empty-case branch.
if: steps.guard.outputs.ready == 'true' && github.ref == 'refs/heads/main'
shell: bash
env:
CI_TOKEN: ${{ secrets.CI_TOKEN }}
run: |
set -eu
REPO="${GITHUB_REPOSITORY}"
REL_JSON="$(curl -fsSL -H "Authorization: token ${CI_TOKEN}" \
"https://git.fabledsword.com/api/v1/repos/${REPO}/releases/latest" || true)"
if [ -z "${REL_JSON}" ]; then
echo "::notice::no published release — image ships without bundled APK"; exit 0
fi
# `|| true` on every one of these, and it is load-bearing rather
# than defensive habit. The runner already invokes this shell as
# `bash -e -o pipefail`, so a pipeline whose grep matches NOTHING
# exits non-zero even though `head` succeeded — and the step dies at
# the assignment, before ever reaching the `if` written to handle the
# empty case. Every "degrades gracefully" branch below is unreachable
# without this.
TAG="$(printf '%s' "${REL_JSON}" | grep -oP '"tag_name":\s*"\K[^"]+' | head -1)" || true
APK_URL="$(printf '%s' "${REL_JSON}" | grep -oP '"browser_download_url":\s*"\K[^"]+' | grep -E '\.apk$' | head -1)" || true
if [ -z "${TAG}" ] || [ -z "${APK_URL}" ]; then
echo "::notice::latest release '${TAG:-?}' has no APK asset — image ships without bundled APK"; exit 0
fi
curl -fsSL -H "Authorization: token ${CI_TOKEN}" -o client/minstrel.apk "${APK_URL}"
# Take the version the release RECORDED rather than recomputing it.
# Re-deriving it from the tagged commit would keep the formula in two
# files that must stay in step, and could only ever recover the name —
# the ordering key is build-time minutes and exists nowhere after that
# build ends.
SIDECAR_URL="$(printf '%s' "${REL_JSON}" | grep -oP '"browser_download_url":\s*"\K[^"]+' | grep -E '\.apk\.version$' | head -1)" || true
if [ -n "${SIDECAR_URL}" ]; then
curl -fsSL -H "Authorization: token ${CI_TOKEN}" -o client/minstrel.apk.version "${SIDECAR_URL}"
cat client/minstrel.apk.version
else
# Releases published before sidecars were attached. Their name is
# still recoverable from the tag, but their ordering key genuinely
# is not — so it is reported ABSENT rather than guessed. A wrong
# key is an install the platform refuses; an absent one just tells
# the client to fall back to comparing names, which is exactly
# what those builds already do.
echo "::notice::release ${TAG} predates the version sidecar — bundling with name only, no ordering key"
printf '{"name":"%s","code":null,"channel":"stable"}\n' "${TAG#v}" > client/minstrel.apk.version
fi
echo "::notice::bundled release APK from ${TAG}"
ls -lh client/
- name: Build and push
if: steps.guard.outputs.ready == 'true'
# --pull: the Dockerfile's base images are floating tags (golang:1.26,
-13
View File
@@ -35,7 +35,6 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/requests"
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble"
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble/listenbrainz"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
"git.fabledsword.com/bvandeusen/minstrel/internal/server"
"git.fabledsword.com/bvandeusen/minstrel/internal/similarity"
"git.fabledsword.com/bvandeusen/minstrel/internal/subsonic"
@@ -129,18 +128,6 @@ func run() error {
}
defer pool.Close()
// Stored credentials are sealed under a key derived from the stream
// secret. Install it before anything reads one, and seal any value still
// stored in the clear.
secret.Install(cfg.StreamSecret)
sealed, err := secret.SealStored(ctx, pool)
if err != nil {
return fmt.Errorf("seal stored credentials: %w", err)
}
if sealed > 0 {
logger.Info("sealed stored credentials", "count", sealed)
}
// Fingerprinting settings: one instance, shared by the scanner,
// the fingerprint backfill, the duplicate sweep and the admin API, so a save
// reaches all of them without a restart. A load failure is logged, not fatal:
+6 -26
View File
@@ -49,42 +49,22 @@ cross-origin preflight. Requests authenticated with a bearer token or the
Subsonic query parameters carry nothing a browser attaches on its own, so
they aren't checked.
## Credentials Minstrel has to read back
Some credentials can't be hashed, because Minstrel has to send or compare the
value itself:
- the API keys and passwords for indexers, Prowlarr and download clients;
- the AcoustID key, and the cover-art and tag provider keys;
- the SMTP password;
- each user's ListenBrainz token and Subsonic password.
These are encrypted in the database (AES-256-GCM). Each column has its own key,
derived from the stream secret (`MINSTREL_STREAM_SECRET`, or the
`stream_secret` file in the data directory), and each value is bound to its
row. The stream secret is never stored in the database, so a copy of the
database alone doesn't reveal any of them. None of them is ever sent back to a
browser: Settings only shows whether one is set.
**Keep the stream secret with the data directory, and don't change it
casually.** If it changes, every stored credential reads as not set, and each
one has to be entered again in Settings (Subsonic passwords generated again).
Minstrel logs a warning for each one that no longer opens.
## Subsonic sign-in
## Subsonic sign-in, and the one password stored in plain text
Classic Subsonic clients sign in with `t` and `s`: the MD5 of the password
followed by a random salt. To check that, the server has to know the password
itself, so supporting this sign-in method means storing a password Minstrel
can read: the Subsonic password, encrypted as described above.
can read. That is the Subsonic password, and it is the only credential
Minstrel keeps unhashed.
- **The recommended way in is the API key.** Clients that support the
OpenSubsonic `apiKey` should use it. The key is stored hashed and can be
replaced at any time in Settings.
- **The Subsonic password is never your login password.** Minstrel generates
it (**Settings → Subsonic password**), shows it once, and lets you replace
or turn it off. Because it is random, even someone holding both the database
and the stream secret gains access to this server's Subsonic API and nothing
else: it can't be a password you also use somewhere else.
or turn it off. Because it is random, a copy of the database exposes access
to this server's Subsonic API and nothing else: it can't be a password you
also use somewhere else.
- **`t`/`s` and `p=` sign-in are off for an account until it has a Subsonic
password.** Plain `p=` sign-in is additionally off server-wide unless
`subsonic.allow_plaintext_password` is enabled.
+1 -1
View File
@@ -24,7 +24,7 @@ func Build(ctx context.Context, pool *pgxpool.Pool) (*Service, dbq.AcquisitionSe
if err != nil {
return nil, settings, fmt.Errorf("acquire: read search sources: %w", err)
}
clients, err := usableServices(ctx, q, JobDownload)
clients, err := q.ListUsableAcquisitionServices(ctx, JobDownload)
if err != nil {
return nil, settings, fmt.Errorf("acquire: read download clients: %w", err)
}
+2 -32
View File
@@ -23,7 +23,6 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/exchange"
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
"git.fabledsword.com/bvandeusen/minstrel/internal/prowlarr"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
// Kinds: which list a row appears in.
@@ -160,7 +159,7 @@ func DownloadClient(row dbq.AcquisitionService) (downloader.Driver, error) {
// DownloadClientFor builds the download client a grab went to, by the code
// of its acquisition_services row.
func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (downloader.Driver, error) {
row, err := getService(ctx, dbq.New(pool), code)
row, err := dbq.New(pool).GetAcquisitionService(ctx, code)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, fmt.Errorf("the download client %q is no longer set up", code)
@@ -172,7 +171,7 @@ func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (do
// Sources builds every enabled search source.
func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) {
rows, err := usableServices(ctx, dbq.New(pool), JobSearch)
rows, err := dbq.New(pool).ListUsableAcquisitionServices(ctx, JobSearch)
if err != nil {
return nil, err
}
@@ -183,33 +182,4 @@ func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) {
return out, nil
}
// getService and usableServices are the only reads of a full
// acquisition_services row. They hand back the credentials opened, so the
// clients built from a row are given the key and password themselves.
func getService(ctx context.Context, q *dbq.Queries, code string) (dbq.AcquisitionService, error) {
row, err := q.GetAcquisitionService(ctx, code)
if err != nil {
return row, err
}
return revealed(row), nil
}
func usableServices(ctx context.Context, q *dbq.Queries, job string) ([]dbq.AcquisitionService, error) {
rows, err := q.ListUsableAcquisitionServices(ctx, job)
if err != nil {
return nil, err
}
for i := range rows {
rows[i] = revealed(rows[i])
}
return rows, nil
}
// revealed is row with its sealed credentials opened.
func revealed(row dbq.AcquisitionService) dbq.AcquisitionService {
row.ApiKey = secret.Reveal(secret.AcquisitionAPIKey.Purpose(), row.Code, row.ApiKey)
row.Password = secret.Reveal(secret.AcquisitionPassword.Purpose(), row.Code, row.Password)
return row
}
func msDuration(ms int32) time.Duration { return time.Duration(ms) * time.Millisecond }
+6 -15
View File
@@ -11,7 +11,6 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
// ErrNotFound is a code with no row.
@@ -50,7 +49,7 @@ func (s *Store) List(ctx context.Context) ([]dbq.ListAcquisitionServicesRow, err
// Get returns one row, secrets included: for building a client only.
func (s *Store) Get(ctx context.Context, code string) (dbq.AcquisitionService, error) {
row, err := getService(ctx, dbq.New(s.pool), code)
row, err := dbq.New(s.pool).GetAcquisitionService(ctx, code)
if errors.Is(err, pgx.ErrNoRows) {
return row, ErrNotFound
}
@@ -164,27 +163,19 @@ func (s *Store) Delete(ctx context.Context, code string) error {
return nil
}
// SetAPIKey stores a key, trimmed and sealed; empty clears it.
// SetAPIKey stores a key, trimmed; empty clears it.
func (s *Store) SetAPIKey(ctx context.Context, code, key string) error {
sealed, err := secret.Seal(secret.AcquisitionAPIKey.Purpose(), code, strings.TrimSpace(key))
if err != nil {
return err
}
n, err := dbq.New(s.pool).SetAcquisitionServiceAPIKey(ctx, dbq.SetAcquisitionServiceAPIKeyParams{
Code: code, ApiKey: sealed,
Code: code, ApiKey: strings.TrimSpace(key),
})
return affected(n, err)
}
// SetPassword stores a password as typed (spaces are part of a password),
// sealed; empty clears it.
// SetPassword stores a password as typed (spaces are part of a password);
// empty clears it.
func (s *Store) SetPassword(ctx context.Context, code, password string) error {
sealed, err := secret.Seal(secret.AcquisitionPassword.Purpose(), code, password)
if err != nil {
return err
}
n, err := dbq.New(s.pool).SetAcquisitionServicePassword(ctx, dbq.SetAcquisitionServicePasswordParams{
Code: code, Password: sealed,
Code: code, Password: password,
})
return affected(n, err)
}
-10
View File
@@ -12,7 +12,6 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
func newPool(t *testing.T) *pgxpool.Pool {
@@ -84,15 +83,6 @@ func TestStore_AddsConfiguresAndRemoves_Integration(t *testing.T) {
if err != nil || full.ApiKey != "key" {
t.Errorf("stored key = %q (err %v), want it trimmed", full.ApiKey, err)
}
// The column holds the key sealed, never the key.
var raw string
if err := s.pool.QueryRow(ctx,
"SELECT api_key FROM acquisition_services WHERE code = $1", first.Code).Scan(&raw); err != nil {
t.Fatalf("read api_key: %v", err)
}
if raw == "key" || !secret.IsSealed(raw) {
t.Errorf("api_key column = %q, want it sealed", raw)
}
rows, err := s.List(ctx)
if err != nil || len(rows) != 2 {
t.Fatalf("list: %d rows, err %v", len(rows), err)
+5 -10
View File
@@ -10,7 +10,6 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
type smtpConfigResp struct {
@@ -77,15 +76,11 @@ func (h *handlers) handleUpdateSMTPConfig(w http.ResponseWriter, r *http.Request
writeErrWithLog(w, h.logger, "admin smtp: get failed", apierror.Internal(err))
return
}
// Preserve the stored (sealed) password when the request sends "", so
// the admin doesn't have to re-enter it on every save.
password := current.Password
if req.Password != "" {
password, err = secret.Seal(secret.SMTPPassword.Purpose(), "", req.Password)
if err != nil {
writeErrWithLog(w, h.logger, "admin smtp: seal password", apierror.Internal(err))
return
}
// Preserve existing password when the request sends "" (so the
// admin doesn't have to re-enter it on every save).
password := req.Password
if password == "" {
password = current.Password
}
if err := q.UpdateSMTPConfig(r.Context(), dbq.UpdateSMTPConfigParams{
Enabled: req.Enabled,
+4 -6
View File
@@ -11,7 +11,6 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
// LBStatus is the GET response body and the PUT response body. Token is
@@ -54,11 +53,10 @@ func (h *handlers) handlePutListenBrainz(w http.ResponseWriter, r *http.Request)
q := dbq.New(h.pool)
if body.Token != nil {
tokenPtr, err := secret.SealPtr(secret.ListenBrainzToken.Purpose(), user.ID.String(), body.Token)
if err != nil {
h.logger.Error("api: seal listenbrainz token", "err", err)
writeErr(w, apierror.InternalMsg("update failed", err))
return
t := *body.Token
var tokenPtr *string
if t != "" {
tokenPtr = &t
}
if err := q.SetListenBrainzToken(r.Context(), dbq.SetListenBrainzTokenParams{
ID: user.ID, ListenbrainzToken: tokenPtr,
+1 -8
View File
@@ -8,7 +8,6 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
// The Subsonic password is for clients that sign in with Subsonic's t/s
@@ -52,15 +51,9 @@ func (h *handlers) handleGenerateMySubsonicPassword(w http.ResponseWriter, r *ht
return
}
pw := base64.RawURLEncoding.EncodeToString(b)
sealed, err := secret.Seal(secret.SubsonicPassword.Purpose(), user.ID.String(), pw)
if err != nil {
h.logger.Error("generate subsonic password: seal failed", "err", err)
writeErr(w, apierror.Internal(err))
return
}
if err := dbq.New(h.pool).SetSubsonicPassword(r.Context(), dbq.SetSubsonicPasswordParams{
ID: user.ID,
SubsonicPassword: &sealed,
SubsonicPassword: &pw,
}); err != nil {
h.logger.Error("generate subsonic password: update failed", "err", err)
writeErr(w, apierror.Internal(err))
+1 -14
View File
@@ -11,7 +11,6 @@ import (
"github.com/go-chi/chi/v5"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
func newMeSubsonicPasswordRouter(h *handlers) chi.Router {
@@ -22,8 +21,6 @@ func newMeSubsonicPasswordRouter(h *handlers) chi.Router {
return r
}
// readSubsonicPassword returns the stored Subsonic password opened. It fails
// the test if the column holds the password in the clear.
func readSubsonicPassword(t *testing.T, h *handlers, user dbq.User) *string {
t.Helper()
var pw *string
@@ -31,17 +28,7 @@ func readSubsonicPassword(t *testing.T, h *handlers, user dbq.User) *string {
"SELECT subsonic_password FROM users WHERE id = $1", user.ID).Scan(&pw); err != nil {
t.Fatalf("read subsonic_password: %v", err)
}
if pw == nil {
return nil
}
if !secret.IsSealed(*pw) {
t.Fatalf("subsonic_password is stored in the clear")
}
plain, err := secret.Open(secret.SubsonicPassword.Purpose(), user.ID.String(), *pw)
if err != nil {
t.Fatalf("open subsonic_password: %v", err)
}
return &plain
return pw
}
func TestSubsonicPassword_GenerateIsRandomAndNotTheLoginPassword(t *testing.T) {
-238
View File
@@ -1,238 +0,0 @@
package backup_test
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"log/slog"
"os"
"slices"
"testing"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
)
func newPool(t *testing.T) *pgxpool.Pool {
t.Helper()
if testing.Short() {
t.Skip("skipping integration test in -short mode")
}
dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
}
if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil {
t.Fatalf("migrate: %v", err)
}
pool, err := pgxpool.New(context.Background(), dsn)
if err != nil {
t.Fatalf("pool: %v", err)
}
t.Cleanup(pool.Close)
dbtest.ResetDB(t, pool)
return pool
}
func index(name string) int {
return slices.IndexFunc(backup.Tables, func(t backup.Table) bool { return t.Name == name })
}
// Every table in the schema is backed up or excluded with a reason, so a new
// table fails here until someone decides which.
func TestEveryTableIsAccountedFor_Integration(t *testing.T) {
pool := newPool(t)
rows, err := pool.Query(context.Background(), "SELECT tablename FROM pg_tables WHERE schemaname = 'public'")
if err != nil {
t.Fatal(err)
}
schema, err := pgx.CollectRows(rows, pgx.RowTo[string])
if err != nil {
t.Fatal(err)
}
for _, p := range backup.Coverage(schema) {
t.Error(p)
}
}
// A restore loads each table after the ones it references, and never needs a
// table the backup leaves out.
func TestTablesComeAfterWhatTheyReference_Integration(t *testing.T) {
pool := newPool(t)
rows, err := pool.Query(context.Background(), `
SELECT cl.relname, pc.relname
FROM pg_constraint co
JOIN pg_class cl ON cl.oid = co.conrelid
JOIN pg_class pc ON pc.oid = co.confrelid
JOIN pg_namespace n ON n.oid = cl.relnamespace
WHERE co.contype = 'f' AND n.nspname = 'public'`)
if err != nil {
t.Fatal(err)
}
type fk struct{ child, parent string }
fks, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (fk, error) {
var f fk
err := r.Scan(&f.child, &f.parent)
return f, err
})
if err != nil {
t.Fatal(err)
}
if len(fks) == 0 {
t.Fatal("found no foreign keys; the query is wrong")
}
for _, f := range fks {
child, parent := index(f.child), index(f.parent)
if child < 0 {
continue
}
switch {
case parent < 0:
t.Errorf("%s is backed up but references %s, which is excluded", f.child, f.parent)
case parent >= child && f.parent != f.child:
t.Errorf("%s references %s, so it must come after it in Tables", f.child, f.parent)
}
}
}
// Every omitted column exists, so a renamed column fails here rather than a
// sealed credential slipping into the file under its new name.
func TestOmittedColumnsExist_Integration(t *testing.T) {
pool := newPool(t)
for _, tbl := range backup.Tables {
for _, col := range tbl.Omit {
var n int
if err := pool.QueryRow(context.Background(), `
SELECT count(*) FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2`,
tbl.Name, col).Scan(&n); err != nil || n != 1 {
t.Errorf("%s.%s: found %d columns, err %v", tbl.Name, col, n, err)
}
}
}
}
func mustExec(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) {
t.Helper()
if _, err := pool.Exec(context.Background(), sql, args...); err != nil {
t.Fatalf("%s: %v", sql, err)
}
}
func mustID(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) string {
t.Helper()
var id string
if err := pool.QueryRow(context.Background(), sql, args...).Scan(&id); err != nil {
t.Fatalf("%s: %v", sql, err)
}
return id
}
// A backup restored into an emptied instance gives back the same rows, byte
// for byte, without the credentials and markers it leaves out.
func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) {
pool := newPool(t)
ctx := context.Background()
user := mustID(t, pool, `INSERT INTO users (username, password_hash, api_token_hash, listenbrainz_token)
VALUES ($1, 'hash', 'backup-token', 'lb-token') RETURNING id::text`, dbtest.TestUserPrefix+"backup")
artist := mustID(t, pool, `INSERT INTO artists (name, sort_name, artist_art_source) VALUES ('Artist', 'artist', 'none') RETURNING id::text`)
album := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id, cover_art_source, cover_art_path)
VALUES ('Album', 'album', $1, 'mbcaa', '/data/covers/a.jpg') RETURNING id::text`, artist)
track := mustID(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format, tag_source)
VALUES ('Track', $1, $2, 180000, '/music/a.flac', 1000, 'flac', 'musicbrainz') RETURNING id::text`, album, artist)
session := mustID(t, pool, `INSERT INTO play_sessions (user_id, started_at, last_event_at) VALUES ($1, now(), now()) RETURNING id::text`, user)
mustExec(t, pool, `INSERT INTO play_events (user_id, track_id, session_id, started_at) VALUES ($1, $2, $3, now())`, user, track, session)
mustExec(t, pool, `INSERT INTO general_likes (user_id, track_id) VALUES ($1, $2)`, user, track)
playlist := mustID(t, pool, `INSERT INTO playlists (user_id, name) VALUES ($1, 'Mine') RETURNING id::text`, user)
mustExec(t, pool, `INSERT INTO playlist_tracks (playlist_id, position, track_id, title, artist_name, album_title, duration_sec)
VALUES ($1, 0, $2, 'Track', 'Artist', 'Album', 180)`, playlist, track)
mustExec(t, pool, `INSERT INTO follows (artist_mbid, artist_name, profile_id, followed_by)
SELECT 'mbid-1', 'Artist', id, $1 FROM release_type_profiles WHERE is_default`, user)
mustExec(t, pool, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert')`, track)
first, err := backup.Snapshot(ctx, pool)
if err != nil {
t.Fatalf("snapshot: %v", err)
}
for _, tbl := range backup.Tables {
for _, line := range first.Tables[tbl.Name] {
var row map[string]json.RawMessage
if err := json.Unmarshal(line, &row); err != nil {
t.Fatalf("%s: %v", tbl.Name, err)
}
for _, col := range tbl.Omit {
if _, ok := row[col]; ok {
t.Errorf("%s.%s is in the backup", tbl.Name, col)
}
}
}
}
for _, tbl := range []string{"users", "tracks", "play_events", "general_likes", "playlist_tracks", "follows", "library_changes"} {
if len(first.Tables[tbl]) == 0 {
t.Errorf("the backup holds no %s", tbl)
}
}
if err := backup.Restore(ctx, pool, first); !errors.Is(err, backup.ErrNotEmpty) {
t.Fatalf("restore over data: err = %v, want ErrNotEmpty", err)
}
newer := first
newer.Schema++
if err := backup.Restore(ctx, pool, newer); !errors.Is(err, backup.ErrNewerSchema) {
t.Fatalf("restore from a newer schema: err = %v, want ErrNewerSchema", err)
}
mustExec(t, pool, "TRUNCATE users, artists, library_changes CASCADE")
if err := backup.Restore(ctx, pool, first); err != nil {
t.Fatalf("restore: %v", err)
}
second, err := backup.Snapshot(ctx, pool)
if err != nil {
t.Fatalf("snapshot after restore: %v", err)
}
for _, tbl := range backup.Tables {
a, b := first.Tables[tbl.Name], second.Tables[tbl.Name]
if len(a) != len(b) {
t.Errorf("%s: %d rows restored, the backup held %d", tbl.Name, len(b), len(a))
continue
}
for i := range a {
if !bytes.Equal(a[i], b[i]) {
t.Errorf("%s row %d:\nbacked up %s\nrestored %s", tbl.Name, i+1, a[i], b[i])
}
}
}
// What the backup leaves out comes back unset, so the server asks for the
// credential again and redoes the art and tags.
var lbToken, coverSource, tagSource *string
if err := pool.QueryRow(ctx, `SELECT u.listenbrainz_token, al.cover_art_source, tr.tag_source
FROM users u, albums al, tracks tr WHERE u.id = $1 AND al.id = $2 AND tr.id = $3`,
user, album, track).Scan(&lbToken, &coverSource, &tagSource); err != nil {
t.Fatal(err)
}
if lbToken != nil || coverSource != nil || tagSource != nil {
t.Errorf("token %v, cover source %v, tag source %v; want all unset", lbToken, coverSource, tagSource)
}
// The change feed carries on after the restored ids.
var restoredMax, next int64
if err := pool.QueryRow(ctx, "SELECT max(id) FROM library_changes").Scan(&restoredMax); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op)
VALUES ('track', $1, 'upsert') RETURNING id`, track).Scan(&next); err != nil {
t.Fatal(err)
}
if next <= restoredMax {
t.Errorf("next change id %d, want it after the restored %d", next, restoredMax)
}
}
-196
View File
@@ -1,196 +0,0 @@
package backup
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
var (
// ErrNotEmpty is a restore into an instance that already has accounts or
// a library.
ErrNotEmpty = errors.New("backup: this Minstrel already has accounts or a library; a backup restores into an empty one")
// ErrNewerSchema is a backup written by a newer Minstrel than this one.
ErrNewerSchema = errors.New("backup: the backup was written by a newer version of Minstrel; upgrade this one first")
// ErrNotABackup is contents that aren't rows a backup writes.
ErrNotABackup = errors.New("backup: not a Minstrel backup")
)
// batchRows is how many rows one INSERT carries.
const batchRows = 500
// Restore loads a backup into an empty instance, in one transaction: either
// all of it lands or none of it does.
//
// The instance must have no accounts and no tracks. Its other rows in the
// tables the backup holds are replaced, which is what a fresh instance's
// seeded settings need. A table the backup doesn't hold, because it was
// written before that table existed, keeps what the instance has.
//
// A row's columns are those the backup and the table both have: an older
// backup's rows take the defaults of columns added since, and a column dropped
// since is left out.
func Restore(ctx context.Context, pool *pgxpool.Pool, c Contents) error {
tx, err := pool.Begin(ctx)
if err != nil {
return fmt.Errorf("backup: begin: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
schema, err := schemaVersion(ctx, tx)
if err != nil {
return err
}
if c.Schema > schema {
return ErrNewerSchema
}
// Taken before the emptiness check, so a first account or a scan starting
// at the same moment waits for the restore, then finds the data.
if _, err := tx.Exec(ctx, "LOCK TABLE users, tracks IN EXCLUSIVE MODE"); err != nil {
return fmt.Errorf("backup: lock: %w", err)
}
var occupied bool
if err := tx.QueryRow(ctx,
"SELECT EXISTS (SELECT 1 FROM users) OR EXISTS (SELECT 1 FROM tracks)").Scan(&occupied); err != nil {
return fmt.Errorf("backup: check the instance is empty: %w", err)
}
if occupied {
return ErrNotEmpty
}
// Children first, so no row is left pointing at one already gone.
for _, t := range slices.Backward(Tables) {
if _, ok := c.Tables[t.Name]; !ok {
continue
}
if _, err := tx.Exec(ctx, "DELETE FROM "+pgx.Identifier{t.Name}.Sanitize()); err != nil {
return fmt.Errorf("backup: clear %s: %w", t.Name, err)
}
}
for _, t := range Tables {
rows, ok := c.Tables[t.Name]
if !ok {
continue
}
if err := restoreTable(ctx, tx, t.Name, rows); err != nil {
return err
}
}
if err := tx.Commit(ctx); err != nil {
return fmt.Errorf("backup: commit: %w", err)
}
return nil
}
type column struct {
name string
identity bool
sequence bool
}
func restoreTable(ctx context.Context, tx pgx.Tx, table string, rows [][]byte) error {
cols, err := writableColumns(ctx, tx, table)
if err != nil {
return err
}
ident := pgx.Identifier{table}.Sanitize()
overriding := ""
if slices.ContainsFunc(cols, func(c column) bool { return c.identity }) {
overriding = " OVERRIDING SYSTEM VALUE"
}
// Rows go in batches that share one key set; a backup's rows for a table
// all do, since one read wrote them.
var (
batch []json.RawMessage
list string
)
flush := func() error {
if len(batch) == 0 {
return nil
}
payload, err := json.Marshal(batch)
if err != nil {
return err
}
sql := fmt.Sprintf("INSERT INTO %s (%s)%s SELECT %s FROM jsonb_populate_recordset(NULL::%s, $1::jsonb)",
ident, list, overriding, list, ident)
if _, err := tx.Exec(ctx, sql, string(payload)); err != nil {
return fmt.Errorf("backup: restore %s: %w", table, err)
}
batch = batch[:0]
return nil
}
for i, raw := range rows {
var keys map[string]json.RawMessage
if err := json.Unmarshal(raw, &keys); err != nil {
return fmt.Errorf("%w: row %d of %s is not an object", ErrNotABackup, i+1, table)
}
var names []string
for _, c := range cols {
if _, ok := keys[c.name]; ok {
names = append(names, pgx.Identifier{c.name}.Sanitize())
}
}
if len(names) == 0 {
continue
}
slices.Sort(names)
rowList := strings.Join(names, ", ")
if rowList != list || len(batch) == batchRows {
if err := flush(); err != nil {
return err
}
list = rowList
}
batch = append(batch, raw)
}
if err := flush(); err != nil {
return err
}
// A sequence carries on after the highest restored value, so the next row
// doesn't collide with one from the backup.
for _, c := range cols {
if !c.sequence {
continue
}
col := pgx.Identifier{c.name}.Sanitize()
if _, err := tx.Exec(ctx, fmt.Sprintf(
"SELECT setval(pg_get_serial_sequence($1, $2), COALESCE((SELECT max(%s) FROM %s), 0) + 1, false)", col, ident),
table, c.name); err != nil {
return fmt.Errorf("backup: carry on %s's sequence: %w", table, err)
}
}
return nil
}
// writableColumns returns the columns of a table an INSERT can set: all but
// generated ones, which Postgres computes.
func writableColumns(ctx context.Context, tx pgx.Tx, table string) ([]column, error) {
rows, err := tx.Query(ctx, `
SELECT column_name, is_identity = 'YES', pg_get_serial_sequence($1::text, column_name) IS NOT NULL
FROM information_schema.columns
WHERE table_schema = current_schema() AND table_name = $1::text AND is_generated = 'NEVER'
ORDER BY ordinal_position`, table)
if err != nil {
return nil, fmt.Errorf("backup: read %s's columns: %w", table, err)
}
cols, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (column, error) {
var c column
err := row.Scan(&c.name, &c.identity, &c.sequence)
return c, err
})
if err != nil {
return nil, fmt.Errorf("backup: read %s's columns: %w", table, err)
}
return cols, nil
}
-114
View File
@@ -1,114 +0,0 @@
package backup
import (
"bytes"
"context"
"encoding/json"
"fmt"
"strings"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
// Contents is a backup's data: the schema version that wrote it, and each
// table's rows as JSON objects with their keys sorted, in primary-key order.
// An unchanged row is the same bytes in every backup.
type Contents struct {
Schema int64
Tables map[string][][]byte
}
// Snapshot reads every table in Tables as of one moment.
func Snapshot(ctx context.Context, pool *pgxpool.Pool) (Contents, error) {
c := Contents{Tables: make(map[string][][]byte, len(Tables))}
tx, err := pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly})
if err != nil {
return c, fmt.Errorf("backup: begin: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
if c.Schema, err = schemaVersion(ctx, tx); err != nil {
return c, err
}
for _, t := range Tables {
lines, err := readTable(ctx, tx, t)
if err != nil {
return c, err
}
c.Tables[t.Name] = lines
}
return c, nil
}
func readTable(ctx context.Context, tx pgx.Tx, t Table) ([][]byte, error) {
pk, err := primaryKey(ctx, tx, t.Name)
if err != nil {
return nil, err
}
order := make([]string, len(pk))
for i, col := range pk {
order[i] = "t." + pgx.Identifier{col}.Sanitize()
}
omit := t.Omit
if omit == nil {
omit = []string{}
}
rows, err := tx.Query(ctx, fmt.Sprintf("SELECT to_jsonb(t) - $1::text[] FROM %s t ORDER BY %s",
pgx.Identifier{t.Name}.Sanitize(), strings.Join(order, ", ")), omit)
if err != nil {
return nil, fmt.Errorf("backup: read %s: %w", t.Name, err)
}
lines, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) ([]byte, error) {
var raw []byte
if err := row.Scan(&raw); err != nil {
return nil, err
}
return canonical(raw)
})
if err != nil {
return nil, fmt.Errorf("backup: read %s: %w", t.Name, err)
}
return lines, nil
}
// canonical re-encodes a row with its keys sorted. Numbers pass through as
// written, so a bigint or a float stays exact.
func canonical(raw []byte) ([]byte, error) {
dec := json.NewDecoder(bytes.NewReader(raw))
dec.UseNumber()
var row map[string]any
if err := dec.Decode(&row); err != nil {
return nil, err
}
return json.Marshal(row)
}
// primaryKey returns a table's primary-key columns in key order.
func primaryKey(ctx context.Context, tx pgx.Tx, table string) ([]string, error) {
rows, err := tx.Query(ctx, `
SELECT a.attname
FROM pg_index i
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY (i.indkey)
WHERE i.indrelid = to_regclass($1) AND i.indisprimary
ORDER BY array_position(i.indkey, a.attnum)`, table)
if err != nil {
return nil, fmt.Errorf("backup: read %s's primary key: %w", table, err)
}
cols, err := pgx.CollectRows(rows, pgx.RowTo[string])
if err != nil {
return nil, fmt.Errorf("backup: read %s's primary key: %w", table, err)
}
if len(cols) == 0 {
return nil, fmt.Errorf("backup: %s has no primary key to order its rows by", table)
}
return cols, nil
}
func schemaVersion(ctx context.Context, tx pgx.Tx) (int64, error) {
var v int64
if err := tx.QueryRow(ctx, "SELECT version FROM schema_migrations").Scan(&v); err != nil {
return 0, fmt.Errorf("backup: read the schema version: %w", err)
}
return v, nil
}
-193
View File
@@ -1,193 +0,0 @@
// Package backup reads Minstrel's irreplaceable data out of Postgres as JSON
// Lines, one table at a time, and loads it back into an empty instance.
//
// What goes in is everything a person made or decided: accounts, likes, play
// and skip history, playlists, follows, wants and requests, download history,
// admin decisions, and settings. What stays out is everything the restored
// server rebuilds by itself: what it computes from the music files, what it
// fetches again from MusicBrainz, ListenBrainz and the other providers, and
// logs of past runs. Excluded says why for each, and the coverage test fails
// on a table that is in neither list.
package backup
// Table is one table a backup holds.
//
// Omit names columns left out of the file. They restore to their defaults:
// - a credential Minstrel stores sealed, which only the server holding the
// stream secret could open, so the restored server asks for it again;
// - a marker saying work on an excluded table is done (a tag source, an art
// source), so the restored server does that work again rather than
// believing it has the result.
type Table struct {
Name string
Omit []string
}
// Tables is every table a backup holds, in the order a restore loads them:
// each after every table it references. There are no self-referencing
// tables, so no table needs its rows loaded in depth order.
var Tables = []Table{
{Name: "users", Omit: []string{"subsonic_password", "listenbrainz_token"}},
// The library's identity. Every like, play and playlist entry hangs off a
// track id, so the rows travel even though a scan could rebuild most of
// their columns; the scan after a restore finds each file at its path and
// keeps its id.
{Name: "artists", Omit: []string{"artist_thumb_path", "artist_fanart_path", "artist_art_source", "artist_art_sources_version"}},
{Name: "albums", Omit: []string{"cover_art_path", "cover_art_source", "cover_art_sources_version"}},
{Name: "tracks", Omit: []string{"tag_source", "tag_sources_version"}},
// Settings.
{Name: "library_settings"},
{Name: "network_settings"},
{Name: "registration_settings"},
{Name: "smtp_config", Omit: []string{"password"}},
{Name: "notification_email_settings"},
{Name: "acoustid_settings", Omit: []string{"api_key"}},
{Name: "fingerprint_settings"},
{Name: "loudness_settings"},
{Name: "cover_art_provider_settings", Omit: []string{"api_key"}},
{Name: "cover_art_sources_meta"},
{Name: "tag_provider_settings", Omit: []string{"api_key"}},
{Name: "tag_sources_meta"},
{Name: "acquisition_services", Omit: []string{"api_key", "password"}},
{Name: "acquisition_settings"},
{Name: "reacquisition_settings"},
{Name: "discover_tuning"},
{Name: "taste_tuning"},
{Name: "recommendation_weight_profiles"},
{Name: "recommendation_tuning_audit"},
// Follows, wants, requests and downloads.
{Name: "release_type_profiles"},
{Name: "follows"},
{Name: "follow_seen_groups"},
{Name: "wants"},
{Name: "grabs"},
{Name: "download_interventions"},
{Name: "release_blocks"},
{Name: "requests"},
// Listening history and likes.
{Name: "play_sessions"},
{Name: "play_events"},
{Name: "skip_events"},
{Name: "scrobble_queue"},
{Name: "contextual_likes"},
{Name: "general_likes"},
{Name: "general_likes_albums"},
{Name: "general_likes_artists"},
// Playlists and what is built from history. The taste profile and the
// "you might like" rows are rebuilt nightly from the history above; they
// travel so a restored Home is full before that first nightly pass.
{Name: "playlists"},
{Name: "playlist_tracks"},
{Name: "system_playlist_runs"},
{Name: "system_playlist_rotation_state"},
{Name: "taste_profile_artists"},
{Name: "taste_profile_tags"},
{Name: "taste_profile_eras"},
{Name: "taste_profile_moods"},
{Name: "you_might_like_albums"},
{Name: "you_might_like_artists"},
// Per-user preferences and notifications.
{Name: "user_normalization_prefs"},
{Name: "user_notification_prefs"},
{Name: "user_notification_email_state"},
{Name: "user_notifications"},
{Name: "suggestion_snoozes"},
{Name: "quarantine"},
{Name: "quarantine_actions"},
// Admin decisions: an album taken off the wanted listing stays off, and a
// dismissed duplicate group stays dismissed when the sweep finds it again.
{Name: "library_album_matches"},
{Name: "duplicate_sweeps"},
{Name: "duplicate_groups"},
{Name: "duplicate_group_members"},
// The library change feed is how the apps sync, including a first sync,
// so a restored server carries it on from where the backup left it.
{Name: "library_changes"},
{Name: "audit_log"},
{Name: "playback_errors"},
}
// Excluded is every table a backup leaves out, with the reason.
var Excluded = map[string]string{
"schema_migrations": "the restored server's own migrations write it; the backup's schema version is in its manifest",
"sessions": "sign-ins; everyone signs in again on the restored server",
"password_resets": "one-time reset links, which expire within hours",
"user_invites": "open sign-up links; an admin sends an invite again from the restored server",
"track_fingerprints": "computed from the music files",
"track_loudness": "computed from the music files",
"album_loudness": "computed from the music files",
"track_acoustid_lookups": "looked up again from each track's fingerprint",
"track_tags": "fetched again from the tag providers",
"candidate_artist_tags": "fetched again from the tag providers",
"candidate_artist_tag_state": "fetched again from the tag providers",
"mb_artists": "MusicBrainz cache, fetched again",
"mb_release_groups": "MusicBrainz cache, fetched again",
"mb_releases": "MusicBrainz cache, fetched again",
"mb_release_tracks": "MusicBrainz cache, fetched again",
"mb_artist_release_groups": "MusicBrainz cache, fetched again",
"listenbrainz_similar_recordings": "similarity, fetched again from ListenBrainz",
"track_similarity": "similarity, fetched again from the similarity providers",
"track_similarity_fetches": "similarity, fetched again from the similarity providers",
"artist_similarity": "similarity, fetched again from the similarity providers",
"artist_similarity_fetches": "similarity, fetched again from the similarity providers",
"artist_similarity_unmatched": "similarity, fetched again from the similarity providers",
"want_candidates": "the last search's results; the next search finds them again",
"scan_runs": "a log of past library scans",
"diagnostic_events": "debug-mode diagnostics",
}
// carried reports whether a table is in Tables.
func carried(name string) bool {
for _, t := range Tables {
if t.Name == name {
return true
}
}
return false
}
// Coverage checks a schema's tables against Tables and Excluded. It returns
// one line per problem: a table in neither list, in both, or listed twice,
// and a listed table the schema doesn't have.
func Coverage(schema []string) []string {
var problems []string
inSchema := map[string]bool{}
for _, name := range schema {
inSchema[name] = true
_, excluded := Excluded[name]
switch {
case carried(name) && excluded:
problems = append(problems, name+": both backed up and excluded")
case !carried(name) && !excluded:
problems = append(problems, name+": neither backed up nor excluded; add it to Tables, or to Excluded with the reason")
}
}
seen := map[string]bool{}
for _, t := range Tables {
if seen[t.Name] {
problems = append(problems, t.Name+": listed twice in Tables")
}
seen[t.Name] = true
if !inSchema[t.Name] {
problems = append(problems, t.Name+": in Tables but not in the schema")
}
}
for name := range Excluded {
if !inSchema[name] {
problems = append(problems, name+": in Excluded but not in the schema")
}
}
return problems
}
-50
View File
@@ -1,50 +0,0 @@
package backup
import (
"slices"
"strings"
"testing"
)
func everyListedTable() []string {
var names []string
for _, t := range Tables {
names = append(names, t.Name)
}
for name := range Excluded {
names = append(names, name)
}
return names
}
func TestCoverageAcceptsASchemaThatMatchesTheLists(t *testing.T) {
if problems := Coverage(everyListedTable()); len(problems) != 0 {
t.Fatalf("problems = %v, want none", problems)
}
}
func TestCoverageNamesATableInNeitherList(t *testing.T) {
problems := Coverage(append(everyListedTable(), "new_table"))
if len(problems) != 1 || !strings.HasPrefix(problems[0], "new_table: neither") {
t.Fatalf("problems = %v, want one naming new_table", problems)
}
}
func TestCoverageNamesAListedTableTheSchemaLacks(t *testing.T) {
schema := slices.DeleteFunc(everyListedTable(), func(n string) bool { return n == "playlists" || n == "scan_runs" })
problems := Coverage(schema)
if len(problems) != 2 {
t.Fatalf("problems = %v, want playlists and scan_runs named", problems)
}
}
func TestEveryExclusionHasAReason(t *testing.T) {
for name, why := range Excluded {
if strings.TrimSpace(why) == "" {
t.Errorf("%s is excluded without a reason", name)
}
if carried(name) {
t.Errorf("%s is both backed up and excluded", name)
}
}
}
+3 -8
View File
@@ -13,7 +13,6 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
// SettingsService reconciles the registry of compiled-in providers
@@ -113,7 +112,7 @@ func (s *SettingsService) reconcile(ctx context.Context) error {
}
key := ""
if row.ApiKey != nil {
key = secret.Reveal(secret.CoverArtAPIKey.Purpose(), row.ProviderID, *row.ApiKey)
key = *row.ApiKey
}
s.enabledIDs[p.ID()] = row.Enabled
s.apiKeys[p.ID()] = key
@@ -205,15 +204,11 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
// (apiKeyChanged flag) and $4 (api_key value). The actual generated
// field for $4 is ApiKey *string.
apiKeyChanged := patch.APIKey != nil
sealedKey, err := secret.SealPtr(secret.CoverArtAPIKey.Purpose(), providerID, patch.APIKey)
if err != nil {
return false, fmt.Errorf("seal provider api key: %w", err)
}
if err := q.UpdateProviderSettings(ctx, dbq.UpdateProviderSettingsParams{
ProviderID: providerID,
Enabled: enabledVal,
Column3: apiKeyChanged,
ApiKey: sealedKey,
ApiKey: patch.APIKey,
}); err != nil {
return false, fmt.Errorf("update provider settings: %w", err)
}
@@ -230,7 +225,7 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
for _, r := range rows {
key := ""
if r.ApiKey != nil {
key = secret.Reveal(secret.CoverArtAPIKey.Purpose(), r.ProviderID, *r.ApiKey)
key = *r.ApiKey
}
s.enabledIDs[r.ProviderID] = r.Enabled
s.apiKeys[r.ProviderID] = key
-10
View File
@@ -1,10 +0,0 @@
package dbtest
import "git.fabledsword.com/bvandeusen/minstrel/internal/secret"
// Integration tests run with a master key installed, as the server runs
// after startup, so code that seals or opens a stored credential behaves as
// it does in production.
func init() {
secret.Install([]byte("minstrel integration tests: not a deployment key"))
}
+2 -7
View File
@@ -10,7 +10,6 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
// AcoustID lookup settings: a database row, changed without a restart, shared
@@ -98,12 +97,8 @@ func (s *AcoustIDSettingsService) Set(ctx context.Context, in AcoustIDSettingsUp
}
params := dbq.UpdateAcoustIDSettingsParams{Enabled: in.Enabled, MinScore: in.MinScore}
if in.APIKey != nil {
sealed, err := secret.Seal(secret.AcoustIDAPIKey.Purpose(), "", *in.APIKey)
if err != nil {
return AcoustIDSettings{}, fmt.Errorf("acoustid settings: seal key: %w", err)
}
params.SetApiKey = true
params.ApiKey = sealed
params.ApiKey = *in.APIKey
}
row, err := dbq.New(s.pool).UpdateAcoustIDSettings(ctx, params)
if err != nil {
@@ -131,7 +126,7 @@ func (s *AcoustIDSettingsService) Changed() <-chan struct{} {
func acoustIDSettingsFromRow(row dbq.AcoustidSetting) AcoustIDSettings {
out := AcoustIDSettings{Enabled: row.Enabled, MinScore: row.MinScore, UpdatedAt: row.UpdatedAt.Time}
if row.ApiKey != nil {
out.APIKey = secret.Reveal(secret.AcoustIDAPIKey.Purpose(), "", *row.ApiKey)
out.APIKey = *row.ApiKey
}
return out
}
+1 -3
View File
@@ -23,7 +23,6 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
// ErrNotConfigured is returned when smtp_config.enabled is false or
@@ -69,8 +68,7 @@ func (s *SMTPSender) Send(ctx context.Context, to, subject, textBody, htmlBody s
var auth smtp.Auth
if cfg.Username != "" {
password := secret.Reveal(secret.SMTPPassword.Purpose(), "", cfg.Password)
auth = smtp.PlainAuth("", cfg.Username, password, cfg.Host)
auth = smtp.PlainAuth("", cfg.Username, cfg.Password, cfg.Host)
}
// We respect the use_tls flag: if true, we require STARTTLS via
+1 -7
View File
@@ -13,7 +13,6 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
func testPool(t *testing.T) (*pgxpool.Pool, *dbq.Queries) {
@@ -64,13 +63,8 @@ func seed(t *testing.T, opts seedOpts) setup {
t.Fatalf("user: %v", err)
}
if opts.lbToken != "" {
// Stored sealed, as the settings route stores it.
sealed, err := secret.Seal(secret.ListenBrainzToken.Purpose(), u.ID.String(), opts.lbToken)
if err != nil {
t.Fatalf("seal token: %v", err)
}
if err := q.SetListenBrainzToken(ctx, dbq.SetListenBrainzTokenParams{
ID: u.ID, ListenbrainzToken: &sealed,
ID: u.ID, ListenbrainzToken: &opts.lbToken,
}); err != nil {
t.Fatalf("token: %v", err)
}
+1 -6
View File
@@ -11,7 +11,6 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble/listenbrainz"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
// backoffSchedule maps the failure count (1-indexed) to the delay before
@@ -99,11 +98,7 @@ func (w *Worker) tickOnce(ctx context.Context) error {
key := r.UserID.String()
ub := batches[key]
if ub == nil {
token := secret.Reveal(secret.ListenBrainzToken.Purpose(), key, *r.LbToken)
if token == "" {
continue
}
ub = &userBatch{token: token, userID: r.UserID}
ub = &userBatch{token: *r.LbToken, userID: r.UserID}
batches[key] = ub
}
ub.queueIDs = append(ub.queueIDs, r.QueueID)
-179
View File
@@ -1,179 +0,0 @@
// Package secret seals the credentials Minstrel keeps for outside systems
// (indexer and download-client keys, the SMTP password, provider API keys,
// ListenBrainz tokens, Subsonic passwords) so the database holds ciphertext,
// not the credential.
//
// Each column has its own key, derived from the install's master key with
// HKDF-SHA256 and the column as the info string, so a value sealed for one
// column cannot be opened as another, and the master key never touches a
// cipher. Each value is also bound to the row it belongs to (its owner), so a
// sealed value copied onto another row does not open there.
//
// The master key is the stream secret (MINSTREL_STREAM_SECRET, or the
// stream_secret file in the data directory). It lives outside the database,
// so a copy of the database alone opens nothing. Changing it makes every
// sealed value unreadable: each one reads as unset and has to be entered
// again in Settings.
package secret
import (
"crypto/aes"
"crypto/cipher"
"crypto/hkdf"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"errors"
"fmt"
"log/slog"
"strings"
"sync"
)
// ErrNoKey is a seal or open before Install has given the package a key.
var ErrNoKey = errors.New("secret: no master key installed")
// ErrCorrupt is a stored value that does not open: the master key changed,
// the value was edited by hand, or it belongs to a different row.
var ErrCorrupt = errors.New("secret: the stored value does not open")
// sealedPrefix versions the format, so a future change of cipher can still
// read what this one wrote. Nothing a person types starts with it by chance
// often enough to matter, and Open rejects anything without it.
const sealedPrefix = "v1."
var b64 = base64.RawURLEncoding
var (
mu sync.RWMutex
master []byte
boxes map[string]cipher.AEAD
)
// Install sets the master key every Seal and Open derives from. The server
// calls it once at startup, before anything reads a credential.
func Install(key []byte) {
mu.Lock()
defer mu.Unlock()
master = append([]byte(nil), key...)
boxes = map[string]cipher.AEAD{}
}
// aead returns purpose's cipher, deriving it on first use.
func aead(purpose string) (cipher.AEAD, error) {
mu.RLock()
a, ok := boxes[purpose]
key := master
mu.RUnlock()
if ok {
return a, nil
}
if len(key) == 0 {
return nil, ErrNoKey
}
derived, err := hkdf.Key(sha256.New, key, nil, "minstrel/"+purpose, 32)
if err != nil {
return nil, fmt.Errorf("secret: derive: %w", err)
}
block, err := aes.NewCipher(derived)
if err != nil {
return nil, fmt.Errorf("secret: %w", err)
}
a, err = cipher.NewGCM(block)
if err != nil {
return nil, fmt.Errorf("secret: %w", err)
}
mu.Lock()
if boxes != nil {
boxes[purpose] = a
}
mu.Unlock()
return a, nil
}
// IsSealed reports whether stored is in the sealed format.
func IsSealed(stored string) bool { return strings.HasPrefix(stored, sealedPrefix) }
// Seal encrypts plain for purpose (the column it is stored in) and owner (the
// row it belongs to). The empty string stays empty, so "is one set?" stays a
// test for the empty string.
func Seal(purpose, owner, plain string) (string, error) {
if plain == "" {
return "", nil
}
a, err := aead(purpose)
if err != nil {
return "", err
}
nonce := make([]byte, a.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return "", fmt.Errorf("secret: nonce: %w", err)
}
out := a.Seal(nonce, nonce, []byte(plain), []byte(owner))
return sealedPrefix + b64.EncodeToString(out), nil
}
// Open decrypts what Seal stored for the same purpose and owner. The empty
// string opens to itself.
func Open(purpose, owner, stored string) (string, error) {
if stored == "" {
return "", nil
}
a, err := aead(purpose)
if err != nil {
return "", err
}
raw, ok := strings.CutPrefix(stored, sealedPrefix)
if !ok {
return "", ErrCorrupt
}
data, err := b64.DecodeString(raw)
n := a.NonceSize()
if err != nil || len(data) < n {
return "", ErrCorrupt
}
plain, err := a.Open(nil, data[:n], data[n:], []byte(owner))
if err != nil {
return "", ErrCorrupt
}
return string(plain), nil
}
// Reveal is Open for a caller about to use the credential: a value that does
// not open is logged and read as unset, which is what it has become. The
// admin sees "not set" in Settings and enters it again.
func Reveal(purpose, owner, stored string) string {
plain, err := Open(purpose, owner, stored)
if err != nil {
slog.Warn("secret: a stored credential does not open; it reads as unset until it is entered again",
"column", purpose, "err", err)
return ""
}
return plain
}
// RevealPtr is Reveal for a nullable column: nil stays nil, and a value that
// does not open reads as nil.
func RevealPtr(purpose, owner string, stored *string) *string {
if stored == nil {
return nil
}
plain := Reveal(purpose, owner, *stored)
if plain == "" {
return nil
}
return &plain
}
// SealPtr is Seal for a nullable column: nil and the empty string both store
// NULL.
func SealPtr(purpose, owner string, plain *string) (*string, error) {
if plain == nil || *plain == "" {
return nil, nil
}
sealed, err := Seal(purpose, owner, *plain)
if err != nil {
return nil, err
}
return &sealed, nil
}
-110
View File
@@ -1,110 +0,0 @@
package secret
import (
"errors"
"strings"
"testing"
)
func withKey(t *testing.T, key string) {
t.Helper()
mu.RLock()
prev := master
mu.RUnlock()
Install([]byte(key))
t.Cleanup(func() { Install(prev) })
}
func TestSealOpensForTheSamePurposeAndOwner(t *testing.T) {
withKey(t, "test master key")
sealed, err := Seal("t.col", "row-1", "hunter2")
if err != nil {
t.Fatal(err)
}
if !IsSealed(sealed) || strings.Contains(sealed, "hunter2") {
t.Fatalf("sealed = %q, want an opaque v1. value", sealed)
}
plain, err := Open("t.col", "row-1", sealed)
if err != nil || plain != "hunter2" {
t.Fatalf("Open = %q, %v; want hunter2", plain, err)
}
again, _ := Seal("t.col", "row-1", "hunter2")
if again == sealed {
t.Error("two seals of one value are identical; the nonce is not random")
}
}
func TestOpenRefusesAnythingElse(t *testing.T) {
withKey(t, "test master key")
sealed, _ := Seal("t.col", "row-1", "hunter2")
cases := map[string]func() (string, error){
"another owner": func() (string, error) { return Open("t.col", "row-2", sealed) },
"another purpose": func() (string, error) { return Open("t.other", "row-1", sealed) },
"plain text": func() (string, error) { return Open("t.col", "row-1", "hunter2") },
"truncated": func() (string, error) { return Open("t.col", "row-1", sealed[:len(sealed)-4]) },
"not base64": func() (string, error) { return Open("t.col", "row-1", "v1.!!!") },
"too short": func() (string, error) { return Open("t.col", "row-1", "v1.AAAA") },
}
for name, open := range cases {
if _, err := open(); !errors.Is(err, ErrCorrupt) {
t.Errorf("%s: err = %v, want ErrCorrupt", name, err)
}
}
Install([]byte("a different key"))
if _, err := Open("t.col", "row-1", sealed); !errors.Is(err, ErrCorrupt) {
t.Errorf("another master key: err = %v, want ErrCorrupt", err)
}
}
func TestEmptyStaysEmpty(t *testing.T) {
withKey(t, "test master key")
if s, err := Seal("t.col", "o", ""); s != "" || err != nil {
t.Errorf("Seal(\"\") = %q, %v; want the empty string", s, err)
}
if s, err := Open("t.col", "o", ""); s != "" || err != nil {
t.Errorf("Open(\"\") = %q, %v; want the empty string", s, err)
}
empty := ""
if p, err := SealPtr("t.col", "o", &empty); p != nil || err != nil {
t.Errorf("SealPtr(\"\") = %v, %v; want nil", p, err)
}
if p := RevealPtr("t.col", "o", nil); p != nil {
t.Errorf("RevealPtr(nil) = %v, want nil", p)
}
}
func TestNoKeyIsAnErrorNotAPanic(t *testing.T) {
withKey(t, "")
if _, err := Seal("t.col", "o", "x"); !errors.Is(err, ErrNoKey) {
t.Errorf("Seal: err = %v, want ErrNoKey", err)
}
if _, err := Open("t.col", "o", "v1.AAAA"); !errors.Is(err, ErrNoKey) {
t.Errorf("Open: err = %v, want ErrNoKey", err)
}
}
func TestRevealReadsAnUnopenableValueAsUnset(t *testing.T) {
withKey(t, "test master key")
if got := Reveal("t.col", "o", "not sealed"); got != "" {
t.Errorf("Reveal = %q, want the empty string", got)
}
bad := "not sealed"
if got := RevealPtr("t.col", "o", &bad); got != nil {
t.Errorf("RevealPtr = %q, want nil", *got)
}
}
func TestEveryColumnHasADistinctPurpose(t *testing.T) {
seen := map[string]bool{}
for _, c := range Columns {
if seen[c.Purpose()] {
t.Errorf("%s is listed twice", c.Purpose())
}
seen[c.Purpose()] = true
}
if !seen[AcquisitionAPIKey.Purpose()] || !seen[SubsonicPassword.Purpose()] {
t.Error("Columns is missing a declared column")
}
}
-112
View File
@@ -1,112 +0,0 @@
package secret
import (
"context"
"fmt"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
// Column is one database column that holds a sealed credential.
//
// Its Purpose ("table.column") is the key-derivation label every Seal and
// Open of that column passes. The owner a value is bound to is the row's key
// as text (KeyColumn::text), or "" for a single-row settings table.
type Column struct {
Table string
Column string
KeyColumn string
Singleton bool
}
// Purpose is the label Seal and Open take for this column.
func (c Column) Purpose() string { return c.Table + "." + c.Column }
// The sealed columns, and the purpose strings their callers pass. Every
// credential Minstrel stores for an outside system, or that must be read back
// in the clear, is listed here.
var (
AcquisitionAPIKey = Column{"acquisition_services", "api_key", "code", false}
AcquisitionPassword = Column{"acquisition_services", "password", "code", false}
AcoustIDAPIKey = Column{"acoustid_settings", "api_key", "id", true}
SMTPPassword = Column{"smtp_config", "password", "id", true}
ListenBrainzToken = Column{"users", "listenbrainz_token", "id", false}
SubsonicPassword = Column{"users", "subsonic_password", "id", false}
CoverArtAPIKey = Column{"cover_art_provider_settings", "api_key", "provider_id", false}
TagProviderAPIKey = Column{"tag_provider_settings", "api_key", "provider_id", false}
)
// Columns is every sealed column, for SealStored and for anything that has to
// know which columns hold credentials (a backup leaves them out).
var Columns = []Column{
AcquisitionAPIKey, AcquisitionPassword, AcoustIDAPIKey, SMTPPassword,
ListenBrainzToken, SubsonicPassword, CoverArtAPIKey, TagProviderAPIKey,
}
// Owner is the owner string for a row whose key is key. A single-row table's
// rows are owned by "".
func (c Column) Owner(key string) string {
if c.Singleton {
return ""
}
return key
}
// SealStored seals every value in Columns that is still stored in the clear,
// and returns how many it sealed. The server runs it at startup, after
// migrations and before anything reads a credential, so a database written
// before sealing existed is brought over in place. Each row is updated only
// if it still holds the value that was read, so a concurrent write wins.
func SealStored(ctx context.Context, pool *pgxpool.Pool) (int, error) {
total := 0
for _, c := range Columns {
n, err := sealColumn(ctx, pool, c)
total += n
if err != nil {
return total, fmt.Errorf("seal %s: %w", c.Purpose(), err)
}
}
return total, nil
}
func sealColumn(ctx context.Context, pool *pgxpool.Pool, c Column) (int, error) {
table := pgx.Identifier{c.Table}.Sanitize()
col := pgx.Identifier{c.Column}.Sanitize()
key := pgx.Identifier{c.KeyColumn}.Sanitize()
rows, err := pool.Query(ctx, fmt.Sprintf(
`SELECT %s::text, %s FROM %s WHERE %s IS NOT NULL AND %s <> '' AND %s NOT LIKE 'v1.%%'`,
key, col, table, col, col, col))
if err != nil {
return 0, err
}
type pending struct{ key, plain string }
var todo []pending
for rows.Next() {
var p pending
if err := rows.Scan(&p.key, &p.plain); err != nil {
rows.Close()
return 0, err
}
todo = append(todo, p)
}
rows.Close()
if err := rows.Err(); err != nil {
return 0, err
}
update := fmt.Sprintf(`UPDATE %s SET %s = $1 WHERE %s::text = $2 AND %s = $3`, table, col, key, col)
sealed := 0
for _, p := range todo {
v, err := Seal(c.Purpose(), c.Owner(p.key), p.plain)
if err != nil {
return sealed, err
}
if _, err := pool.Exec(ctx, update, v, p.key, p.plain); err != nil {
return sealed, err
}
sealed++
}
return sealed, nil
}
-154
View File
@@ -1,154 +0,0 @@
package secret_test
import (
"context"
"fmt"
"io"
"log/slog"
"os"
"testing"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
func newPool(t *testing.T) *pgxpool.Pool {
t.Helper()
if testing.Short() {
t.Skip("skipping integration test in -short mode")
}
dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
}
if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil {
t.Fatalf("migrate: %v", err)
}
pool, err := pgxpool.New(context.Background(), dsn)
if err != nil {
t.Fatalf("pool: %v", err)
}
t.Cleanup(pool.Close)
dbtest.ResetDB(t, pool)
return pool
}
// Every listed column exists, so a renamed or dropped column fails here
// rather than leaving a credential unsealed.
func TestColumnsExistInTheSchema_Integration(t *testing.T) {
pool := newPool(t)
for _, c := range secret.Columns {
var n int
err := pool.QueryRow(context.Background(),
`SELECT count(*) FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = $1 AND column_name IN ($2, $3)`,
c.Table, c.Column, c.KeyColumn).Scan(&n)
if err != nil || n != 2 {
t.Errorf("%s (key %s): found %d of 2 columns, err %v", c.Purpose(), c.KeyColumn, n, err)
}
}
}
// A database written before sealing existed is sealed in place at startup,
// every value opens to what it held, and a second pass has nothing to do.
func TestSealStoredSealsWhatIsInTheClear_Integration(t *testing.T) {
pool := newPool(t)
ctx := context.Background()
q := dbq.New(pool)
u, err := q.CreateUser(ctx, dbq.CreateUserParams{
Username: dbtest.TestUserPrefix + "sealer", PasswordHash: "x", ApiTokenHash: "sealer", IsAdmin: false,
})
if err != nil {
t.Fatalf("user: %v", err)
}
if _, err := q.CreateAcquisitionService(ctx, dbq.CreateAcquisitionServiceParams{
Code: "seal-test", Kind: "search", Protocol: "prowlarr", Label: "Seal test",
Category: "", TimeoutMs: 1000, SearchesPerHour: 0,
}); err != nil {
t.Fatalf("acquisition row: %v", err)
}
seed := []string{
`UPDATE acquisition_services SET api_key = 'acq-key', password = 'acq-pw' WHERE code = 'seal-test'`,
`UPDATE acoustid_settings SET api_key = 'acoustid-key' WHERE id = true`,
`UPDATE smtp_config SET password = 'smtp-pw' WHERE id = true`,
fmt.Sprintf(`UPDATE users SET listenbrainz_token = 'lb-token', subsonic_password = 'sub-pw' WHERE id = '%s'`, u.ID.String()),
`INSERT INTO cover_art_provider_settings (provider_id, enabled, display_order, api_key)
VALUES ('seal-test', false, 99, 'cover-key')
ON CONFLICT (provider_id) DO UPDATE SET api_key = EXCLUDED.api_key`,
`INSERT INTO tag_provider_settings (provider_id, enabled, display_order, api_key)
VALUES ('seal-test', false, 99, 'tag-key')
ON CONFLICT (provider_id) DO UPDATE SET api_key = EXCLUDED.api_key`,
}
for _, s := range seed {
if _, err := pool.Exec(ctx, s); err != nil {
t.Fatalf("seed %q: %v", s, err)
}
}
t.Cleanup(func() {
for _, s := range []string{
`DELETE FROM cover_art_provider_settings WHERE provider_id = 'seal-test'`,
`DELETE FROM tag_provider_settings WHERE provider_id = 'seal-test'`,
`DELETE FROM acquisition_services WHERE code = 'seal-test'`,
`UPDATE acoustid_settings SET api_key = NULL WHERE id = true`,
`UPDATE smtp_config SET password = '' WHERE id = true`,
} {
_, _ = pool.Exec(context.Background(), s)
}
})
want := []struct {
col secret.Column
key string
plain string
}{
{secret.AcquisitionAPIKey, "seal-test", "acq-key"},
{secret.AcquisitionPassword, "seal-test", "acq-pw"},
{secret.AcoustIDAPIKey, "true", "acoustid-key"},
{secret.SMTPPassword, "true", "smtp-pw"},
{secret.ListenBrainzToken, u.ID.String(), "lb-token"},
{secret.SubsonicPassword, u.ID.String(), "sub-pw"},
{secret.CoverArtAPIKey, "seal-test", "cover-key"},
{secret.TagProviderAPIKey, "seal-test", "tag-key"},
}
if len(want) != len(secret.Columns) {
t.Fatalf("the test covers %d columns, secret.Columns lists %d", len(want), len(secret.Columns))
}
n, err := secret.SealStored(ctx, pool)
if err != nil {
t.Fatalf("SealStored: %v", err)
}
if n < len(want) {
t.Errorf("sealed %d values, want at least %d", n, len(want))
}
for _, w := range want {
var stored string
query := fmt.Sprintf(`SELECT %s FROM %s WHERE %s::text = $1`,
pgx.Identifier{w.col.Column}.Sanitize(), pgx.Identifier{w.col.Table}.Sanitize(),
pgx.Identifier{w.col.KeyColumn}.Sanitize())
if err := pool.QueryRow(ctx, query, w.key).Scan(&stored); err != nil {
t.Errorf("%s: read: %v", w.col.Purpose(), err)
continue
}
if !secret.IsSealed(stored) {
t.Errorf("%s = %q, want it sealed", w.col.Purpose(), stored)
continue
}
got, err := secret.Open(w.col.Purpose(), w.col.Owner(w.key), stored)
if err != nil || got != w.plain {
t.Errorf("%s opens to %q (err %v), want %q", w.col.Purpose(), got, err, w.plain)
}
}
again, err := secret.SealStored(ctx, pool)
if err != nil || again != 0 {
t.Errorf("second pass sealed %d (err %v), want 0", again, err)
}
}
+64 -32
View File
@@ -272,41 +272,73 @@ func TestDevChannel_PublishesDevAloneAndNeverLatest(t *testing.T) {
}
}
// A merge to main is enough to put a new app in front of stable phones: main
// builds the signed APK from its own commit and bakes it into :latest, as dev
// does for :dev. The regression to guard is main falling back to a tag-only
// APK build, which leaves every merged app change waiting on a release while
// each build stays green.
func TestEveryBranchShipsTheAppBuiltFromItsOwnCommit(t *testing.T) {
jobs := releaseJobs(t)
build := strings.Join(strings.Fields(jobs["android-release"].If), " ")
for _, ref := range []string{"refs/heads/main", "refs/heads/dev", "refs/tags/v"} {
if !strings.Contains(build, ref) {
t.Errorf("android-release does not build on %s, so that ref's image carries no app of its own\n if: %s", ref, build)
}
// The two bundling paths must stay mutually exclusive. If the rebundle step's
// condition were relaxed back to "not a tag", a dev push would run BOTH: stage
// its freshly-built APK, then overwrite it with the previous release's. The
// image would still build and the sidecar would still parse — it would just
// quietly serve stale art to the channel whose whole job is being current.
func TestDevChannel_RebundlePathIsMainOnly(t *testing.T) {
body, err := os.ReadFile(filepath.Join(repoRoot(t), ".gitea", "workflows", "release.yml"))
if err != nil {
t.Fatal(err)
}
yaml := string(body)
image := strings.Join(strings.Fields(jobs["image-release"].If), " ")
if !strings.Contains(image, "needs.android-release.result == 'success'") || strings.Contains(image, "'skipped'") {
t.Errorf("image-release can publish without a freshly-built APK\n if: %s", image)
}
yaml := releaseYAML(t)
if strings.Contains(yaml, "releases/latest") {
t.Error("release.yml fetches the latest release; an image bundling a release's APK ships an app older than its own commit")
}
// Both bundling steps run on every ref. Sliced from image-release on,
// because release-assets has a step of the same name.
i := strings.Index(yaml, "\n image-release:")
i := strings.Index(yaml, "- name: Bundle latest release APK")
if i < 0 {
t.Fatal("no image-release job")
t.Fatal("no 'Bundle latest release APK' step")
}
for _, name := range []string{"Download signed APK artifact", "Stage bundled APK + version sidecar"} {
step := stepBody(t, yaml[i:], name)
if strings.Contains(step, "github.ref") {
t.Errorf("%q is gated by ref, so some ref's image ships without its APK\n%s", name, step)
step := yaml[i:]
if j := strings.Index(step, "\n - name:"); j >= 0 {
step = step[:j]
}
if !strings.Contains(step, "github.ref == 'refs/heads/main'") {
t.Errorf("the rebundle step is not gated to main; a dev push would overwrite its own APK\n%s", step)
}
}
// The runner invokes `shell: bash` as `bash -e -o pipefail`. That makes any
// command substitution whose grep matches NOTHING fatal at the assignment —
// pipefail reports the grep's non-zero status even though `head` succeeded —
// so the step dies before reaching the `if` written to handle the empty case.
//
// This is not hypothetical. It took down the first `main` build after the
// version rework: the newest release predated sidecar assets, its
// `.apk.version` grep matched nothing, and the step aborted instead of
// falling through to the name-only branch that exists precisely for it. The
// same latent hazard sat on the other two assignments and had simply never
// fired, because their greps always matched.
//
// Pins the property that makes every "degrades gracefully" branch in that
// step reachable at all.
func TestBundleStep_GrepsCannotKillTheStep(t *testing.T) {
body, err := os.ReadFile(filepath.Join(repoRoot(t), ".gitea", "workflows", "release.yml"))
if err != nil {
t.Fatal(err)
}
yaml := string(body)
i := strings.Index(yaml, "- name: Bundle latest release APK")
if i < 0 {
t.Fatal("no 'Bundle latest release APK' step")
}
step := yaml[i:]
if j := strings.Index(step, "\n - name:"); j >= 0 {
step = step[:j]
}
for _, line := range strings.Split(step, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "#") || !strings.Contains(trimmed, "grep") {
continue
}
// Only assignments from a command substitution can abort the step.
if !strings.Contains(trimmed, `="$(`) {
continue
}
if !strings.HasSuffix(trimmed, "|| true") {
t.Errorf("this assignment dies under pipefail when its grep matches nothing, "+
"skipping the empty-case branch below it:\n %s", trimmed)
}
}
}
@@ -344,7 +376,7 @@ func TestReleaseBuild_RefreshesTheChannelAndMintsNothingElse(t *testing.T) {
arm, _, _ := imageTagArms(t, releaseYAML(t))
if !strings.Contains(arm, "${IMAGE}:latest") {
t.Errorf("tag arm does not refresh :latest — the server would hand out a different APK build from the one attached to the Release\n%s", arm)
t.Errorf("tag arm does not refresh :latest — the channel would keep serving the PREVIOUS release's APK until someone pushed to main\n%s", arm)
}
if strings.Contains(arm, "GITHUB_SHA") {
t.Errorf("tag arm mints a :<sha> image; that would re-push main's immutable rollback target with different bundled contents\n%s", arm)
+4 -9
View File
@@ -17,7 +17,6 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
// Config controls wire-format concessions the server makes for Subsonic
@@ -102,15 +101,11 @@ func authenticate(r *http.Request, pool *pgxpool.Pool, cfg Config) (dbq.User, in
return dbq.User{}, ErrGeneric, "Auth lookup failed"
}
// The stored Subsonic password is sealed; both t+s and p= compare against
// it opened. One that does not open reads as not configured.
subsonicPassword := secret.RevealPtr(secret.SubsonicPassword.Purpose(), user.ID.String(), user.SubsonicPassword)
if token, salt := params.Get("t"), params.Get("s"); token != "" && salt != "" {
if subsonicPassword == nil {
if user.SubsonicPassword == nil {
return dbq.User{}, ErrTokenNotSupported, "Subsonic token auth not configured; use apiKey or set a Subsonic password"
}
sum := md5.Sum([]byte(*subsonicPassword + salt))
sum := md5.Sum([]byte(*user.SubsonicPassword + salt))
expected := hex.EncodeToString(sum[:])
if subtle.ConstantTimeCompare([]byte(expected), []byte(token)) != 1 {
return dbq.User{}, ErrWrongCredentials, "Wrong username or password"
@@ -122,14 +117,14 @@ func authenticate(r *http.Request, pool *pgxpool.Pool, cfg Config) (dbq.User, in
if !cfg.AllowPlaintextPassword {
return dbq.User{}, ErrTokenNotSupported, "Plaintext password auth is disabled; use apiKey or t+s"
}
if subsonicPassword == nil {
if user.SubsonicPassword == nil {
return dbq.User{}, ErrTokenNotSupported, "Subsonic password not configured for this user"
}
decoded, err := decodePassword(password)
if err != nil {
return dbq.User{}, ErrWrongCredentials, "Malformed password"
}
if subtle.ConstantTimeCompare([]byte(*subsonicPassword), []byte(decoded)) != 1 {
if subtle.ConstantTimeCompare([]byte(*user.SubsonicPassword), []byte(decoded)) != 1 {
return dbq.User{}, ErrWrongCredentials, "Wrong username or password"
}
return user, 0, ""
+3 -8
View File
@@ -13,7 +13,6 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)
// SettingsService reconciles the compiled-in provider registry with the
@@ -102,7 +101,7 @@ func (s *SettingsService) reconcile(ctx context.Context) error {
}
key := ""
if row.ApiKey != nil {
key = secret.Reveal(secret.TagProviderAPIKey.Purpose(), row.ProviderID, *row.ApiKey)
key = *row.ApiKey
}
s.enabledIDs[p.ID()] = row.Enabled
s.apiKeys[p.ID()] = key
@@ -182,15 +181,11 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
if patch.Enabled != nil {
enabledVal = *patch.Enabled
}
sealedKey, err := secret.SealPtr(secret.TagProviderAPIKey.Purpose(), providerID, patch.APIKey)
if err != nil {
return false, fmt.Errorf("seal provider api key: %w", err)
}
if err := q.UpdateTagProviderSettings(ctx, dbq.UpdateTagProviderSettingsParams{
ProviderID: providerID,
Enabled: enabledVal,
Column3: patch.APIKey != nil,
ApiKey: sealedKey,
ApiKey: patch.APIKey,
}); err != nil {
return false, fmt.Errorf("update provider settings: %w", err)
}
@@ -205,7 +200,7 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
for _, r := range rows {
key := ""
if r.ApiKey != nil {
key = secret.Reveal(secret.TagProviderAPIKey.Purpose(), r.ProviderID, *r.ApiKey)
key = *r.ApiKey
}
s.enabledIDs[r.ProviderID] = r.Enabled
s.apiKeys[r.ProviderID] = key