Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a67897a6cd | ||
|
|
a8a2d20307 | ||
|
|
be642cd53f |
+39
-102
@@ -2,9 +2,9 @@ name: release
|
||||
|
||||
# Builds and pushes the minstrel container image to the Gitea registry.
|
||||
#
|
||||
# push to dev → :dev (freshly-built dev APK bundled)
|
||||
# push to main → :latest + :<sha> (latest-release APK bundled)
|
||||
# push tag vYYYY.MM.DD.HHMM → :latest (fresh APK bundled)
|
||||
# push to dev → :dev (freshly-built dev APK bundled)
|
||||
# push to main → :latest + :<sha> (freshly-built stable APK bundled)
|
||||
# push tag vYYYY.MM.DD.HHMM → :latest (fresh APK bundled, and attached to the Release)
|
||||
# workflow_dispatch → manual trigger (same rules based on the ref)
|
||||
#
|
||||
# That is the whole tag map.
|
||||
@@ -41,6 +41,12 @@ name: release
|
||||
# image in the same run from the same commit, and the channel cannot go
|
||||
# stale against its own artifact.
|
||||
#
|
||||
# Every branch build carries the app built from its own commit, so a merge to
|
||||
# main is all it takes to put a new app in front of stable phones: `:latest`
|
||||
# serves it from /api/client/apk and the in-app update offers it. A release
|
||||
# is an optional bookmark — a named point in history with the APK attached
|
||||
# for download — never a step the app has to wait for.
|
||||
#
|
||||
# Release model: the tag IS the artifact's version name with a `v` in front.
|
||||
# `v2026.09.10.1432` and `2026.09.10.1432` are the same string, derived from
|
||||
# the tagged commit's UTC timestamp — so there is no mismatch to reconcile
|
||||
@@ -56,20 +62,14 @@ name: release
|
||||
# :latest is updated by every main push AND every tag push, so it always
|
||||
# reflects the newest blessed image.
|
||||
#
|
||||
# APK pipeline: on tag pushes the android-release job builds + signs the
|
||||
# Android APK and uploads it as a workflow artifact. The image-release
|
||||
# job declares `needs: android-release`, so the docker image cannot
|
||||
# start building until the APK is guaranteed-ready — no polling, no
|
||||
# race, no silent-failure mode. Attaching the APK to the gitea Release is
|
||||
# its own job (release-assets), behind the test gate below.
|
||||
#
|
||||
# :latest always carries an APK. Because every main push also moves
|
||||
# :latest (not just tags), a main build with no APK would silently strip
|
||||
# the in-app update channel off :latest until the next release. So on
|
||||
# non-tag builds image-release pulls the MOST RECENT release's signed APK
|
||||
# AND the version sidecar published beside it — the recorded values, not
|
||||
# recomputed ones — so no rebuild is needed, just a rebundle. Tag builds
|
||||
# keep bundling their own freshly-built APK.
|
||||
# APK pipeline: on every push — dev, main and tags alike — the
|
||||
# android-release job builds + signs the Android APK and uploads it as a
|
||||
# workflow artifact. The image-release job declares `needs: android-release`
|
||||
# and requires its success, so the docker image cannot start building until
|
||||
# the APK is guaranteed-ready — no polling, no race, no silent-failure mode,
|
||||
# and no image that moves a channel without an app inside it. Attaching the
|
||||
# APK to the gitea Release is its own job (release-assets), tags only, behind
|
||||
# the test gate below.
|
||||
#
|
||||
# THE GATE. Every verifying lane lives in this file —
|
||||
# Go (vet, lint, short tests), the Postgres integration suite, the web app
|
||||
@@ -400,11 +400,14 @@ jobs:
|
||||
# ------------------------------------------------------------ artifacts --
|
||||
|
||||
android-release:
|
||||
name: Build signed APK (releases and dev)
|
||||
# Also builds on `dev`, which is what makes a test channel possible at
|
||||
# all: without it the only way to get a build onto a phone is to cut a
|
||||
# release, which quietly turns `main` into the staging area.
|
||||
if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev'
|
||||
name: Build signed APK
|
||||
# Builds on every ref, so each channel's image carries the app from its
|
||||
# own commit: `dev` for the test channel, `main` for stable phones, and a
|
||||
# tag for the copy attached to its Release.
|
||||
if: >-
|
||||
startsWith(github.ref, 'refs/tags/v')
|
||||
|| github.ref == 'refs/heads/dev'
|
||||
|| github.ref == 'refs/heads/main'
|
||||
runs-on: flutter-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-android:36
|
||||
@@ -601,9 +604,9 @@ jobs:
|
||||
path: release-apk/
|
||||
|
||||
- name: Attach APK to gitea Release
|
||||
# Tag releases only. A dev build has no Release to hang assets on and
|
||||
# does not need one — the :dev image bundles the APK, and the server
|
||||
# serves it from /api/client/apk like any other.
|
||||
# Tag releases only. A branch build has no Release to hang assets on
|
||||
# and does not need one — its image bundles the APK, and the server
|
||||
# serves it from /api/client/apk.
|
||||
shell: bash
|
||||
env:
|
||||
CI_TOKEN: ${{ secrets.CI_TOKEN }}
|
||||
@@ -667,10 +670,9 @@ jobs:
|
||||
|
||||
image-release:
|
||||
name: Build + push container image
|
||||
# Every lane must have SUCCEEDED, each named here. Then the
|
||||
# APK: tag and dev pushes build one and it must have succeeded; main
|
||||
# pushes skip android-release and bundle the latest release's APK
|
||||
# instead, so for main alone a skipped android-release is expected.
|
||||
# Every lane must have SUCCEEDED, each named here, and so must the APK
|
||||
# build: every ref bundles the APK built from its own commit, so an
|
||||
# image without one would move a channel with no app for phones to take.
|
||||
needs: [go, integration, web, android, govulncheck, android-release]
|
||||
if: >-
|
||||
${{
|
||||
@@ -680,8 +682,7 @@ jobs:
|
||||
&& needs.web.result == 'success'
|
||||
&& needs.android.result == 'success'
|
||||
&& needs.govulncheck.result == 'success'
|
||||
&& (needs.android-release.result == 'success'
|
||||
|| (needs.android-release.result == 'skipped' && github.ref == 'refs/heads/main')) }}
|
||||
&& needs.android-release.result == 'success' }}
|
||||
runs-on: go-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-go:1.26
|
||||
@@ -747,9 +748,10 @@ jobs:
|
||||
# publish the moving channel tag and never a commit-addressable
|
||||
# one.
|
||||
#
|
||||
# :latest must move here rather than waiting for the next main
|
||||
# push, or the channel would carry the PREVIOUS release's APK
|
||||
# indefinitely — a channel that cannot refresh itself.
|
||||
# :latest moves here so the server hands out the same APK build
|
||||
# that is attached to the Release: a phone that installed from the
|
||||
# Release page and one that updated in-app hold one build, with one
|
||||
# ordering key. Cut a release on main's tip, as :latest follows it.
|
||||
CHANNEL=stable
|
||||
echo "args=-t ${IMAGE}:latest" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::Release build ${VERSION}: refreshing :latest around the new APK"
|
||||
@@ -782,12 +784,8 @@ jobs:
|
||||
| docker login git.fabledsword.com -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
- name: Download signed APK artifact
|
||||
# Tag and dev pushes — android-release just produced this. Only `main`
|
||||
# takes the "Bundle latest release APK" path below, because it is the
|
||||
# one ref that moves a channel without building an APK of its own.
|
||||
if: >-
|
||||
steps.guard.outputs.ready == 'true' &&
|
||||
(startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev')
|
||||
# android-release just produced this, on every ref.
|
||||
if: steps.guard.outputs.ready == 'true'
|
||||
# Consuming half of the pair: stock download-artifact, which works here for
|
||||
# the same reason as the upload (gitea/runner 3.x edits the GHES refusal
|
||||
# out of the bundle). v8 runs on node24, which every CI-runner image
|
||||
@@ -798,9 +796,7 @@ jobs:
|
||||
path: client/
|
||||
|
||||
- name: Stage bundled APK + version sidecar
|
||||
if: >-
|
||||
steps.guard.outputs.ready == 'true' &&
|
||||
(startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev')
|
||||
if: steps.guard.outputs.ready == 'true'
|
||||
shell: bash
|
||||
env:
|
||||
# All three pulled from android-release's outputs so the sidecar the
|
||||
@@ -821,65 +817,6 @@ jobs:
|
||||
cat client/minstrel.apk.version
|
||||
ls -lh client/
|
||||
|
||||
- name: Bundle latest release APK (non-tag :latest builds)
|
||||
# Main pushes don't build an APK, but they DO move :latest — so
|
||||
# without this the in-app update channel would vanish from :latest
|
||||
# until the next tag. Pull the most-recent release's signed APK and
|
||||
# the sidecar published beside it, so what the server reports is what
|
||||
# that build actually recorded rather than something re-derived here.
|
||||
# Degrades to an empty client/ (404 update channel) — never a wrong
|
||||
# version — if no release or APK asset can be resolved. That
|
||||
# degradation only actually works because the greps below carry
|
||||
# `|| true`; under the runner's default pipefail a non-matching grep
|
||||
# kills the step instead of falling through to the empty-case branch.
|
||||
if: steps.guard.outputs.ready == 'true' && github.ref == 'refs/heads/main'
|
||||
shell: bash
|
||||
env:
|
||||
CI_TOKEN: ${{ secrets.CI_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
REPO="${GITHUB_REPOSITORY}"
|
||||
REL_JSON="$(curl -fsSL -H "Authorization: token ${CI_TOKEN}" \
|
||||
"https://git.fabledsword.com/api/v1/repos/${REPO}/releases/latest" || true)"
|
||||
if [ -z "${REL_JSON}" ]; then
|
||||
echo "::notice::no published release — image ships without bundled APK"; exit 0
|
||||
fi
|
||||
# `|| true` on every one of these, and it is load-bearing rather
|
||||
# than defensive habit. The runner already invokes this shell as
|
||||
# `bash -e -o pipefail`, so a pipeline whose grep matches NOTHING
|
||||
# exits non-zero even though `head` succeeded — and the step dies at
|
||||
# the assignment, before ever reaching the `if` written to handle the
|
||||
# empty case. Every "degrades gracefully" branch below is unreachable
|
||||
# without this.
|
||||
TAG="$(printf '%s' "${REL_JSON}" | grep -oP '"tag_name":\s*"\K[^"]+' | head -1)" || true
|
||||
APK_URL="$(printf '%s' "${REL_JSON}" | grep -oP '"browser_download_url":\s*"\K[^"]+' | grep -E '\.apk$' | head -1)" || true
|
||||
if [ -z "${TAG}" ] || [ -z "${APK_URL}" ]; then
|
||||
echo "::notice::latest release '${TAG:-?}' has no APK asset — image ships without bundled APK"; exit 0
|
||||
fi
|
||||
curl -fsSL -H "Authorization: token ${CI_TOKEN}" -o client/minstrel.apk "${APK_URL}"
|
||||
|
||||
# Take the version the release RECORDED rather than recomputing it.
|
||||
# Re-deriving it from the tagged commit would keep the formula in two
|
||||
# files that must stay in step, and could only ever recover the name —
|
||||
# the ordering key is build-time minutes and exists nowhere after that
|
||||
# build ends.
|
||||
SIDECAR_URL="$(printf '%s' "${REL_JSON}" | grep -oP '"browser_download_url":\s*"\K[^"]+' | grep -E '\.apk\.version$' | head -1)" || true
|
||||
if [ -n "${SIDECAR_URL}" ]; then
|
||||
curl -fsSL -H "Authorization: token ${CI_TOKEN}" -o client/minstrel.apk.version "${SIDECAR_URL}"
|
||||
cat client/minstrel.apk.version
|
||||
else
|
||||
# Releases published before sidecars were attached. Their name is
|
||||
# still recoverable from the tag, but their ordering key genuinely
|
||||
# is not — so it is reported ABSENT rather than guessed. A wrong
|
||||
# key is an install the platform refuses; an absent one just tells
|
||||
# the client to fall back to comparing names, which is exactly
|
||||
# what those builds already do.
|
||||
echo "::notice::release ${TAG} predates the version sidecar — bundling with name only, no ordering key"
|
||||
printf '{"name":"%s","code":null,"channel":"stable"}\n' "${TAG#v}" > client/minstrel.apk.version
|
||||
fi
|
||||
echo "::notice::bundled release APK from ${TAG}"
|
||||
ls -lh client/
|
||||
|
||||
- name: Build and push
|
||||
if: steps.guard.outputs.ready == 'true'
|
||||
# --pull: the Dockerfile's base images are floating tags (golang:1.26,
|
||||
|
||||
@@ -35,6 +35,7 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/requests"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble/listenbrainz"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/server"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/similarity"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/subsonic"
|
||||
@@ -128,6 +129,18 @@ func run() error {
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
// Stored credentials are sealed under a key derived from the stream
|
||||
// secret. Install it before anything reads one, and seal any value still
|
||||
// stored in the clear.
|
||||
secret.Install(cfg.StreamSecret)
|
||||
sealed, err := secret.SealStored(ctx, pool)
|
||||
if err != nil {
|
||||
return fmt.Errorf("seal stored credentials: %w", err)
|
||||
}
|
||||
if sealed > 0 {
|
||||
logger.Info("sealed stored credentials", "count", sealed)
|
||||
}
|
||||
|
||||
// Fingerprinting settings: one instance, shared by the scanner,
|
||||
// the fingerprint backfill, the duplicate sweep and the admin API, so a save
|
||||
// reaches all of them without a restart. A load failure is logged, not fatal:
|
||||
|
||||
+26
-6
@@ -49,22 +49,42 @@ cross-origin preflight. Requests authenticated with a bearer token or the
|
||||
Subsonic query parameters carry nothing a browser attaches on its own, so
|
||||
they aren't checked.
|
||||
|
||||
## Subsonic sign-in, and the one password stored in plain text
|
||||
## Credentials Minstrel has to read back
|
||||
|
||||
Some credentials can't be hashed, because Minstrel has to send or compare the
|
||||
value itself:
|
||||
- the API keys and passwords for indexers, Prowlarr and download clients;
|
||||
- the AcoustID key, and the cover-art and tag provider keys;
|
||||
- the SMTP password;
|
||||
- each user's ListenBrainz token and Subsonic password.
|
||||
|
||||
These are encrypted in the database (AES-256-GCM). Each column has its own key,
|
||||
derived from the stream secret (`MINSTREL_STREAM_SECRET`, or the
|
||||
`stream_secret` file in the data directory), and each value is bound to its
|
||||
row. The stream secret is never stored in the database, so a copy of the
|
||||
database alone doesn't reveal any of them. None of them is ever sent back to a
|
||||
browser: Settings only shows whether one is set.
|
||||
|
||||
**Keep the stream secret with the data directory, and don't change it
|
||||
casually.** If it changes, every stored credential reads as not set, and each
|
||||
one has to be entered again in Settings (Subsonic passwords generated again).
|
||||
Minstrel logs a warning for each one that no longer opens.
|
||||
|
||||
## Subsonic sign-in
|
||||
|
||||
Classic Subsonic clients sign in with `t` and `s`: the MD5 of the password
|
||||
followed by a random salt. To check that, the server has to know the password
|
||||
itself, so supporting this sign-in method means storing a password Minstrel
|
||||
can read. That is the Subsonic password, and it is the only credential
|
||||
Minstrel keeps unhashed.
|
||||
can read: the Subsonic password, encrypted as described above.
|
||||
|
||||
- **The recommended way in is the API key.** Clients that support the
|
||||
OpenSubsonic `apiKey` should use it. The key is stored hashed and can be
|
||||
replaced at any time in Settings.
|
||||
- **The Subsonic password is never your login password.** Minstrel generates
|
||||
it (**Settings → Subsonic password**), shows it once, and lets you replace
|
||||
or turn it off. Because it is random, a copy of the database exposes access
|
||||
to this server's Subsonic API and nothing else: it can't be a password you
|
||||
also use somewhere else.
|
||||
or turn it off. Because it is random, even someone holding both the database
|
||||
and the stream secret gains access to this server's Subsonic API and nothing
|
||||
else: it can't be a password you also use somewhere else.
|
||||
- **`t`/`s` and `p=` sign-in are off for an account until it has a Subsonic
|
||||
password.** Plain `p=` sign-in is additionally off server-wide unless
|
||||
`subsonic.allow_plaintext_password` is enabled.
|
||||
|
||||
@@ -24,7 +24,7 @@ func Build(ctx context.Context, pool *pgxpool.Pool) (*Service, dbq.AcquisitionSe
|
||||
if err != nil {
|
||||
return nil, settings, fmt.Errorf("acquire: read search sources: %w", err)
|
||||
}
|
||||
clients, err := q.ListUsableAcquisitionServices(ctx, JobDownload)
|
||||
clients, err := usableServices(ctx, q, JobDownload)
|
||||
if err != nil {
|
||||
return nil, settings, fmt.Errorf("acquire: read download clients: %w", err)
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/exchange"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/prowlarr"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// Kinds: which list a row appears in.
|
||||
@@ -159,7 +160,7 @@ func DownloadClient(row dbq.AcquisitionService) (downloader.Driver, error) {
|
||||
// DownloadClientFor builds the download client a grab went to, by the code
|
||||
// of its acquisition_services row.
|
||||
func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (downloader.Driver, error) {
|
||||
row, err := dbq.New(pool).GetAcquisitionService(ctx, code)
|
||||
row, err := getService(ctx, dbq.New(pool), code)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, fmt.Errorf("the download client %q is no longer set up", code)
|
||||
@@ -171,7 +172,7 @@ func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (do
|
||||
|
||||
// Sources builds every enabled search source.
|
||||
func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) {
|
||||
rows, err := dbq.New(pool).ListUsableAcquisitionServices(ctx, JobSearch)
|
||||
rows, err := usableServices(ctx, dbq.New(pool), JobSearch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -182,4 +183,33 @@ func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// getService and usableServices are the only reads of a full
|
||||
// acquisition_services row. They hand back the credentials opened, so the
|
||||
// clients built from a row are given the key and password themselves.
|
||||
func getService(ctx context.Context, q *dbq.Queries, code string) (dbq.AcquisitionService, error) {
|
||||
row, err := q.GetAcquisitionService(ctx, code)
|
||||
if err != nil {
|
||||
return row, err
|
||||
}
|
||||
return revealed(row), nil
|
||||
}
|
||||
|
||||
func usableServices(ctx context.Context, q *dbq.Queries, job string) ([]dbq.AcquisitionService, error) {
|
||||
rows, err := q.ListUsableAcquisitionServices(ctx, job)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range rows {
|
||||
rows[i] = revealed(rows[i])
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// revealed is row with its sealed credentials opened.
|
||||
func revealed(row dbq.AcquisitionService) dbq.AcquisitionService {
|
||||
row.ApiKey = secret.Reveal(secret.AcquisitionAPIKey.Purpose(), row.Code, row.ApiKey)
|
||||
row.Password = secret.Reveal(secret.AcquisitionPassword.Purpose(), row.Code, row.Password)
|
||||
return row
|
||||
}
|
||||
|
||||
func msDuration(ms int32) time.Duration { return time.Duration(ms) * time.Millisecond }
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// ErrNotFound is a code with no row.
|
||||
@@ -49,7 +50,7 @@ func (s *Store) List(ctx context.Context) ([]dbq.ListAcquisitionServicesRow, err
|
||||
|
||||
// Get returns one row, secrets included: for building a client only.
|
||||
func (s *Store) Get(ctx context.Context, code string) (dbq.AcquisitionService, error) {
|
||||
row, err := dbq.New(s.pool).GetAcquisitionService(ctx, code)
|
||||
row, err := getService(ctx, dbq.New(s.pool), code)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return row, ErrNotFound
|
||||
}
|
||||
@@ -163,19 +164,27 @@ func (s *Store) Delete(ctx context.Context, code string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetAPIKey stores a key, trimmed; empty clears it.
|
||||
// SetAPIKey stores a key, trimmed and sealed; empty clears it.
|
||||
func (s *Store) SetAPIKey(ctx context.Context, code, key string) error {
|
||||
sealed, err := secret.Seal(secret.AcquisitionAPIKey.Purpose(), code, strings.TrimSpace(key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := dbq.New(s.pool).SetAcquisitionServiceAPIKey(ctx, dbq.SetAcquisitionServiceAPIKeyParams{
|
||||
Code: code, ApiKey: strings.TrimSpace(key),
|
||||
Code: code, ApiKey: sealed,
|
||||
})
|
||||
return affected(n, err)
|
||||
}
|
||||
|
||||
// SetPassword stores a password as typed (spaces are part of a password);
|
||||
// empty clears it.
|
||||
// SetPassword stores a password as typed (spaces are part of a password),
|
||||
// sealed; empty clears it.
|
||||
func (s *Store) SetPassword(ctx context.Context, code, password string) error {
|
||||
sealed, err := secret.Seal(secret.AcquisitionPassword.Purpose(), code, password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := dbq.New(s.pool).SetAcquisitionServicePassword(ctx, dbq.SetAcquisitionServicePasswordParams{
|
||||
Code: code, Password: password,
|
||||
Code: code, Password: sealed,
|
||||
})
|
||||
return affected(n, err)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
func newPool(t *testing.T) *pgxpool.Pool {
|
||||
@@ -83,6 +84,15 @@ func TestStore_AddsConfiguresAndRemoves_Integration(t *testing.T) {
|
||||
if err != nil || full.ApiKey != "key" {
|
||||
t.Errorf("stored key = %q (err %v), want it trimmed", full.ApiKey, err)
|
||||
}
|
||||
// The column holds the key sealed, never the key.
|
||||
var raw string
|
||||
if err := s.pool.QueryRow(ctx,
|
||||
"SELECT api_key FROM acquisition_services WHERE code = $1", first.Code).Scan(&raw); err != nil {
|
||||
t.Fatalf("read api_key: %v", err)
|
||||
}
|
||||
if raw == "key" || !secret.IsSealed(raw) {
|
||||
t.Errorf("api_key column = %q, want it sealed", raw)
|
||||
}
|
||||
rows, err := s.List(ctx)
|
||||
if err != nil || len(rows) != 2 {
|
||||
t.Fatalf("list: %d rows, err %v", len(rows), err)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
type smtpConfigResp struct {
|
||||
@@ -76,11 +77,15 @@ func (h *handlers) handleUpdateSMTPConfig(w http.ResponseWriter, r *http.Request
|
||||
writeErrWithLog(w, h.logger, "admin smtp: get failed", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
// Preserve existing password when the request sends "" (so the
|
||||
// admin doesn't have to re-enter it on every save).
|
||||
password := req.Password
|
||||
if password == "" {
|
||||
password = current.Password
|
||||
// Preserve the stored (sealed) password when the request sends "", so
|
||||
// the admin doesn't have to re-enter it on every save.
|
||||
password := current.Password
|
||||
if req.Password != "" {
|
||||
password, err = secret.Seal(secret.SMTPPassword.Purpose(), "", req.Password)
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "admin smtp: seal password", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := q.UpdateSMTPConfig(r.Context(), dbq.UpdateSMTPConfigParams{
|
||||
Enabled: req.Enabled,
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// LBStatus is the GET response body and the PUT response body. Token is
|
||||
@@ -53,10 +54,11 @@ func (h *handlers) handlePutListenBrainz(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
q := dbq.New(h.pool)
|
||||
if body.Token != nil {
|
||||
t := *body.Token
|
||||
var tokenPtr *string
|
||||
if t != "" {
|
||||
tokenPtr = &t
|
||||
tokenPtr, err := secret.SealPtr(secret.ListenBrainzToken.Purpose(), user.ID.String(), body.Token)
|
||||
if err != nil {
|
||||
h.logger.Error("api: seal listenbrainz token", "err", err)
|
||||
writeErr(w, apierror.InternalMsg("update failed", err))
|
||||
return
|
||||
}
|
||||
if err := q.SetListenBrainzToken(r.Context(), dbq.SetListenBrainzTokenParams{
|
||||
ID: user.ID, ListenbrainzToken: tokenPtr,
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// The Subsonic password is for clients that sign in with Subsonic's t/s
|
||||
@@ -51,9 +52,15 @@ func (h *handlers) handleGenerateMySubsonicPassword(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
pw := base64.RawURLEncoding.EncodeToString(b)
|
||||
sealed, err := secret.Seal(secret.SubsonicPassword.Purpose(), user.ID.String(), pw)
|
||||
if err != nil {
|
||||
h.logger.Error("generate subsonic password: seal failed", "err", err)
|
||||
writeErr(w, apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
if err := dbq.New(h.pool).SetSubsonicPassword(r.Context(), dbq.SetSubsonicPasswordParams{
|
||||
ID: user.ID,
|
||||
SubsonicPassword: &pw,
|
||||
SubsonicPassword: &sealed,
|
||||
}); err != nil {
|
||||
h.logger.Error("generate subsonic password: update failed", "err", err)
|
||||
writeErr(w, apierror.Internal(err))
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
func newMeSubsonicPasswordRouter(h *handlers) chi.Router {
|
||||
@@ -21,6 +22,8 @@ func newMeSubsonicPasswordRouter(h *handlers) chi.Router {
|
||||
return r
|
||||
}
|
||||
|
||||
// readSubsonicPassword returns the stored Subsonic password opened. It fails
|
||||
// the test if the column holds the password in the clear.
|
||||
func readSubsonicPassword(t *testing.T, h *handlers, user dbq.User) *string {
|
||||
t.Helper()
|
||||
var pw *string
|
||||
@@ -28,7 +31,17 @@ func readSubsonicPassword(t *testing.T, h *handlers, user dbq.User) *string {
|
||||
"SELECT subsonic_password FROM users WHERE id = $1", user.ID).Scan(&pw); err != nil {
|
||||
t.Fatalf("read subsonic_password: %v", err)
|
||||
}
|
||||
return pw
|
||||
if pw == nil {
|
||||
return nil
|
||||
}
|
||||
if !secret.IsSealed(*pw) {
|
||||
t.Fatalf("subsonic_password is stored in the clear")
|
||||
}
|
||||
plain, err := secret.Open(secret.SubsonicPassword.Purpose(), user.ID.String(), *pw)
|
||||
if err != nil {
|
||||
t.Fatalf("open subsonic_password: %v", err)
|
||||
}
|
||||
return &plain
|
||||
}
|
||||
|
||||
func TestSubsonicPassword_GenerateIsRandomAndNotTheLoginPassword(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package backup_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||
)
|
||||
|
||||
func newPool(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in -short mode")
|
||||
}
|
||||
dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
|
||||
}
|
||||
if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("pool: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
dbtest.ResetDB(t, pool)
|
||||
return pool
|
||||
}
|
||||
|
||||
func index(name string) int {
|
||||
return slices.IndexFunc(backup.Tables, func(t backup.Table) bool { return t.Name == name })
|
||||
}
|
||||
|
||||
// Every table in the schema is backed up or excluded with a reason, so a new
|
||||
// table fails here until someone decides which.
|
||||
func TestEveryTableIsAccountedFor_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
rows, err := pool.Query(context.Background(), "SELECT tablename FROM pg_tables WHERE schemaname = 'public'")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
schema, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range backup.Coverage(schema) {
|
||||
t.Error(p)
|
||||
}
|
||||
}
|
||||
|
||||
// A restore loads each table after the ones it references, and never needs a
|
||||
// table the backup leaves out.
|
||||
func TestTablesComeAfterWhatTheyReference_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
rows, err := pool.Query(context.Background(), `
|
||||
SELECT cl.relname, pc.relname
|
||||
FROM pg_constraint co
|
||||
JOIN pg_class cl ON cl.oid = co.conrelid
|
||||
JOIN pg_class pc ON pc.oid = co.confrelid
|
||||
JOIN pg_namespace n ON n.oid = cl.relnamespace
|
||||
WHERE co.contype = 'f' AND n.nspname = 'public'`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
type fk struct{ child, parent string }
|
||||
fks, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (fk, error) {
|
||||
var f fk
|
||||
err := r.Scan(&f.child, &f.parent)
|
||||
return f, err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(fks) == 0 {
|
||||
t.Fatal("found no foreign keys; the query is wrong")
|
||||
}
|
||||
for _, f := range fks {
|
||||
child, parent := index(f.child), index(f.parent)
|
||||
if child < 0 {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case parent < 0:
|
||||
t.Errorf("%s is backed up but references %s, which is excluded", f.child, f.parent)
|
||||
case parent >= child && f.parent != f.child:
|
||||
t.Errorf("%s references %s, so it must come after it in Tables", f.child, f.parent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every omitted column exists, so a renamed column fails here rather than a
|
||||
// sealed credential slipping into the file under its new name.
|
||||
func TestOmittedColumnsExist_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
for _, tbl := range backup.Tables {
|
||||
for _, col := range tbl.Omit {
|
||||
var n int
|
||||
if err := pool.QueryRow(context.Background(), `
|
||||
SELECT count(*) FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2`,
|
||||
tbl.Name, col).Scan(&n); err != nil || n != 1 {
|
||||
t.Errorf("%s.%s: found %d columns, err %v", tbl.Name, col, n, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustExec(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := pool.Exec(context.Background(), sql, args...); err != nil {
|
||||
t.Fatalf("%s: %v", sql, err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustID(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) string {
|
||||
t.Helper()
|
||||
var id string
|
||||
if err := pool.QueryRow(context.Background(), sql, args...).Scan(&id); err != nil {
|
||||
t.Fatalf("%s: %v", sql, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// A backup restored into an emptied instance gives back the same rows, byte
|
||||
// for byte, without the credentials and markers it leaves out.
|
||||
func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
ctx := context.Background()
|
||||
|
||||
user := mustID(t, pool, `INSERT INTO users (username, password_hash, api_token_hash, listenbrainz_token)
|
||||
VALUES ($1, 'hash', 'backup-token', 'lb-token') RETURNING id::text`, dbtest.TestUserPrefix+"backup")
|
||||
artist := mustID(t, pool, `INSERT INTO artists (name, sort_name, artist_art_source) VALUES ('Artist', 'artist', 'none') RETURNING id::text`)
|
||||
album := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id, cover_art_source, cover_art_path)
|
||||
VALUES ('Album', 'album', $1, 'mbcaa', '/data/covers/a.jpg') RETURNING id::text`, artist)
|
||||
track := mustID(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format, tag_source)
|
||||
VALUES ('Track', $1, $2, 180000, '/music/a.flac', 1000, 'flac', 'musicbrainz') RETURNING id::text`, album, artist)
|
||||
session := mustID(t, pool, `INSERT INTO play_sessions (user_id, started_at, last_event_at) VALUES ($1, now(), now()) RETURNING id::text`, user)
|
||||
mustExec(t, pool, `INSERT INTO play_events (user_id, track_id, session_id, started_at) VALUES ($1, $2, $3, now())`, user, track, session)
|
||||
mustExec(t, pool, `INSERT INTO general_likes (user_id, track_id) VALUES ($1, $2)`, user, track)
|
||||
playlist := mustID(t, pool, `INSERT INTO playlists (user_id, name) VALUES ($1, 'Mine') RETURNING id::text`, user)
|
||||
mustExec(t, pool, `INSERT INTO playlist_tracks (playlist_id, position, track_id, title, artist_name, album_title, duration_sec)
|
||||
VALUES ($1, 0, $2, 'Track', 'Artist', 'Album', 180)`, playlist, track)
|
||||
mustExec(t, pool, `INSERT INTO follows (artist_mbid, artist_name, profile_id, followed_by)
|
||||
SELECT 'mbid-1', 'Artist', id, $1 FROM release_type_profiles WHERE is_default`, user)
|
||||
mustExec(t, pool, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert')`, track)
|
||||
|
||||
first, err := backup.Snapshot(ctx, pool)
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot: %v", err)
|
||||
}
|
||||
for _, tbl := range backup.Tables {
|
||||
for _, line := range first.Tables[tbl.Name] {
|
||||
var row map[string]json.RawMessage
|
||||
if err := json.Unmarshal(line, &row); err != nil {
|
||||
t.Fatalf("%s: %v", tbl.Name, err)
|
||||
}
|
||||
for _, col := range tbl.Omit {
|
||||
if _, ok := row[col]; ok {
|
||||
t.Errorf("%s.%s is in the backup", tbl.Name, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, tbl := range []string{"users", "tracks", "play_events", "general_likes", "playlist_tracks", "follows", "library_changes"} {
|
||||
if len(first.Tables[tbl]) == 0 {
|
||||
t.Errorf("the backup holds no %s", tbl)
|
||||
}
|
||||
}
|
||||
|
||||
if err := backup.Restore(ctx, pool, first); !errors.Is(err, backup.ErrNotEmpty) {
|
||||
t.Fatalf("restore over data: err = %v, want ErrNotEmpty", err)
|
||||
}
|
||||
newer := first
|
||||
newer.Schema++
|
||||
if err := backup.Restore(ctx, pool, newer); !errors.Is(err, backup.ErrNewerSchema) {
|
||||
t.Fatalf("restore from a newer schema: err = %v, want ErrNewerSchema", err)
|
||||
}
|
||||
|
||||
mustExec(t, pool, "TRUNCATE users, artists, library_changes CASCADE")
|
||||
if err := backup.Restore(ctx, pool, first); err != nil {
|
||||
t.Fatalf("restore: %v", err)
|
||||
}
|
||||
|
||||
second, err := backup.Snapshot(ctx, pool)
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot after restore: %v", err)
|
||||
}
|
||||
for _, tbl := range backup.Tables {
|
||||
a, b := first.Tables[tbl.Name], second.Tables[tbl.Name]
|
||||
if len(a) != len(b) {
|
||||
t.Errorf("%s: %d rows restored, the backup held %d", tbl.Name, len(b), len(a))
|
||||
continue
|
||||
}
|
||||
for i := range a {
|
||||
if !bytes.Equal(a[i], b[i]) {
|
||||
t.Errorf("%s row %d:\nbacked up %s\nrestored %s", tbl.Name, i+1, a[i], b[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the backup leaves out comes back unset, so the server asks for the
|
||||
// credential again and redoes the art and tags.
|
||||
var lbToken, coverSource, tagSource *string
|
||||
if err := pool.QueryRow(ctx, `SELECT u.listenbrainz_token, al.cover_art_source, tr.tag_source
|
||||
FROM users u, albums al, tracks tr WHERE u.id = $1 AND al.id = $2 AND tr.id = $3`,
|
||||
user, album, track).Scan(&lbToken, &coverSource, &tagSource); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if lbToken != nil || coverSource != nil || tagSource != nil {
|
||||
t.Errorf("token %v, cover source %v, tag source %v; want all unset", lbToken, coverSource, tagSource)
|
||||
}
|
||||
|
||||
// The change feed carries on after the restored ids.
|
||||
var restoredMax, next int64
|
||||
if err := pool.QueryRow(ctx, "SELECT max(id) FROM library_changes").Scan(&restoredMax); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op)
|
||||
VALUES ('track', $1, 'upsert') RETURNING id`, track).Scan(&next); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if next <= restoredMax {
|
||||
t.Errorf("next change id %d, want it after the restored %d", next, restoredMax)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNotEmpty is a restore into an instance that already has accounts or
|
||||
// a library.
|
||||
ErrNotEmpty = errors.New("backup: this Minstrel already has accounts or a library; a backup restores into an empty one")
|
||||
|
||||
// ErrNewerSchema is a backup written by a newer Minstrel than this one.
|
||||
ErrNewerSchema = errors.New("backup: the backup was written by a newer version of Minstrel; upgrade this one first")
|
||||
|
||||
// ErrNotABackup is contents that aren't rows a backup writes.
|
||||
ErrNotABackup = errors.New("backup: not a Minstrel backup")
|
||||
)
|
||||
|
||||
// batchRows is how many rows one INSERT carries.
|
||||
const batchRows = 500
|
||||
|
||||
// Restore loads a backup into an empty instance, in one transaction: either
|
||||
// all of it lands or none of it does.
|
||||
//
|
||||
// The instance must have no accounts and no tracks. Its other rows in the
|
||||
// tables the backup holds are replaced, which is what a fresh instance's
|
||||
// seeded settings need. A table the backup doesn't hold, because it was
|
||||
// written before that table existed, keeps what the instance has.
|
||||
//
|
||||
// A row's columns are those the backup and the table both have: an older
|
||||
// backup's rows take the defaults of columns added since, and a column dropped
|
||||
// since is left out.
|
||||
func Restore(ctx context.Context, pool *pgxpool.Pool, c Contents) error {
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: begin: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
schema, err := schemaVersion(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c.Schema > schema {
|
||||
return ErrNewerSchema
|
||||
}
|
||||
// Taken before the emptiness check, so a first account or a scan starting
|
||||
// at the same moment waits for the restore, then finds the data.
|
||||
if _, err := tx.Exec(ctx, "LOCK TABLE users, tracks IN EXCLUSIVE MODE"); err != nil {
|
||||
return fmt.Errorf("backup: lock: %w", err)
|
||||
}
|
||||
var occupied bool
|
||||
if err := tx.QueryRow(ctx,
|
||||
"SELECT EXISTS (SELECT 1 FROM users) OR EXISTS (SELECT 1 FROM tracks)").Scan(&occupied); err != nil {
|
||||
return fmt.Errorf("backup: check the instance is empty: %w", err)
|
||||
}
|
||||
if occupied {
|
||||
return ErrNotEmpty
|
||||
}
|
||||
|
||||
// Children first, so no row is left pointing at one already gone.
|
||||
for _, t := range slices.Backward(Tables) {
|
||||
if _, ok := c.Tables[t.Name]; !ok {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(ctx, "DELETE FROM "+pgx.Identifier{t.Name}.Sanitize()); err != nil {
|
||||
return fmt.Errorf("backup: clear %s: %w", t.Name, err)
|
||||
}
|
||||
}
|
||||
for _, t := range Tables {
|
||||
rows, ok := c.Tables[t.Name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if err := restoreTable(ctx, tx, t.Name, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("backup: commit: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type column struct {
|
||||
name string
|
||||
identity bool
|
||||
sequence bool
|
||||
}
|
||||
|
||||
func restoreTable(ctx context.Context, tx pgx.Tx, table string, rows [][]byte) error {
|
||||
cols, err := writableColumns(ctx, tx, table)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ident := pgx.Identifier{table}.Sanitize()
|
||||
overriding := ""
|
||||
if slices.ContainsFunc(cols, func(c column) bool { return c.identity }) {
|
||||
overriding = " OVERRIDING SYSTEM VALUE"
|
||||
}
|
||||
|
||||
// Rows go in batches that share one key set; a backup's rows for a table
|
||||
// all do, since one read wrote them.
|
||||
var (
|
||||
batch []json.RawMessage
|
||||
list string
|
||||
)
|
||||
flush := func() error {
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
payload, err := json.Marshal(batch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sql := fmt.Sprintf("INSERT INTO %s (%s)%s SELECT %s FROM jsonb_populate_recordset(NULL::%s, $1::jsonb)",
|
||||
ident, list, overriding, list, ident)
|
||||
if _, err := tx.Exec(ctx, sql, string(payload)); err != nil {
|
||||
return fmt.Errorf("backup: restore %s: %w", table, err)
|
||||
}
|
||||
batch = batch[:0]
|
||||
return nil
|
||||
}
|
||||
for i, raw := range rows {
|
||||
var keys map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &keys); err != nil {
|
||||
return fmt.Errorf("%w: row %d of %s is not an object", ErrNotABackup, i+1, table)
|
||||
}
|
||||
var names []string
|
||||
for _, c := range cols {
|
||||
if _, ok := keys[c.name]; ok {
|
||||
names = append(names, pgx.Identifier{c.name}.Sanitize())
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
continue
|
||||
}
|
||||
slices.Sort(names)
|
||||
rowList := strings.Join(names, ", ")
|
||||
if rowList != list || len(batch) == batchRows {
|
||||
if err := flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
list = rowList
|
||||
}
|
||||
batch = append(batch, raw)
|
||||
}
|
||||
if err := flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// A sequence carries on after the highest restored value, so the next row
|
||||
// doesn't collide with one from the backup.
|
||||
for _, c := range cols {
|
||||
if !c.sequence {
|
||||
continue
|
||||
}
|
||||
col := pgx.Identifier{c.name}.Sanitize()
|
||||
if _, err := tx.Exec(ctx, fmt.Sprintf(
|
||||
"SELECT setval(pg_get_serial_sequence($1, $2), COALESCE((SELECT max(%s) FROM %s), 0) + 1, false)", col, ident),
|
||||
table, c.name); err != nil {
|
||||
return fmt.Errorf("backup: carry on %s's sequence: %w", table, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writableColumns returns the columns of a table an INSERT can set: all but
|
||||
// generated ones, which Postgres computes.
|
||||
func writableColumns(ctx context.Context, tx pgx.Tx, table string) ([]column, error) {
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT column_name, is_identity = 'YES', pg_get_serial_sequence($1::text, column_name) IS NOT NULL
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = $1::text AND is_generated = 'NEVER'
|
||||
ORDER BY ordinal_position`, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read %s's columns: %w", table, err)
|
||||
}
|
||||
cols, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (column, error) {
|
||||
var c column
|
||||
err := row.Scan(&c.name, &c.identity, &c.sequence)
|
||||
return c, err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read %s's columns: %w", table, err)
|
||||
}
|
||||
return cols, nil
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Contents is a backup's data: the schema version that wrote it, and each
|
||||
// table's rows as JSON objects with their keys sorted, in primary-key order.
|
||||
// An unchanged row is the same bytes in every backup.
|
||||
type Contents struct {
|
||||
Schema int64
|
||||
Tables map[string][][]byte
|
||||
}
|
||||
|
||||
// Snapshot reads every table in Tables as of one moment.
|
||||
func Snapshot(ctx context.Context, pool *pgxpool.Pool) (Contents, error) {
|
||||
c := Contents{Tables: make(map[string][][]byte, len(Tables))}
|
||||
tx, err := pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly})
|
||||
if err != nil {
|
||||
return c, fmt.Errorf("backup: begin: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
if c.Schema, err = schemaVersion(ctx, tx); err != nil {
|
||||
return c, err
|
||||
}
|
||||
for _, t := range Tables {
|
||||
lines, err := readTable(ctx, tx, t)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
c.Tables[t.Name] = lines
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func readTable(ctx context.Context, tx pgx.Tx, t Table) ([][]byte, error) {
|
||||
pk, err := primaryKey(ctx, tx, t.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
order := make([]string, len(pk))
|
||||
for i, col := range pk {
|
||||
order[i] = "t." + pgx.Identifier{col}.Sanitize()
|
||||
}
|
||||
omit := t.Omit
|
||||
if omit == nil {
|
||||
omit = []string{}
|
||||
}
|
||||
rows, err := tx.Query(ctx, fmt.Sprintf("SELECT to_jsonb(t) - $1::text[] FROM %s t ORDER BY %s",
|
||||
pgx.Identifier{t.Name}.Sanitize(), strings.Join(order, ", ")), omit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read %s: %w", t.Name, err)
|
||||
}
|
||||
lines, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) ([]byte, error) {
|
||||
var raw []byte
|
||||
if err := row.Scan(&raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return canonical(raw)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read %s: %w", t.Name, err)
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// canonical re-encodes a row with its keys sorted. Numbers pass through as
|
||||
// written, so a bigint or a float stays exact.
|
||||
func canonical(raw []byte) ([]byte, error) {
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.UseNumber()
|
||||
var row map[string]any
|
||||
if err := dec.Decode(&row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(row)
|
||||
}
|
||||
|
||||
// primaryKey returns a table's primary-key columns in key order.
|
||||
func primaryKey(ctx context.Context, tx pgx.Tx, table string) ([]string, error) {
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT a.attname
|
||||
FROM pg_index i
|
||||
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY (i.indkey)
|
||||
WHERE i.indrelid = to_regclass($1) AND i.indisprimary
|
||||
ORDER BY array_position(i.indkey, a.attnum)`, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read %s's primary key: %w", table, err)
|
||||
}
|
||||
cols, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read %s's primary key: %w", table, err)
|
||||
}
|
||||
if len(cols) == 0 {
|
||||
return nil, fmt.Errorf("backup: %s has no primary key to order its rows by", table)
|
||||
}
|
||||
return cols, nil
|
||||
}
|
||||
|
||||
func schemaVersion(ctx context.Context, tx pgx.Tx) (int64, error) {
|
||||
var v int64
|
||||
if err := tx.QueryRow(ctx, "SELECT version FROM schema_migrations").Scan(&v); err != nil {
|
||||
return 0, fmt.Errorf("backup: read the schema version: %w", err)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
// Package backup reads Minstrel's irreplaceable data out of Postgres as JSON
|
||||
// Lines, one table at a time, and loads it back into an empty instance.
|
||||
//
|
||||
// What goes in is everything a person made or decided: accounts, likes, play
|
||||
// and skip history, playlists, follows, wants and requests, download history,
|
||||
// admin decisions, and settings. What stays out is everything the restored
|
||||
// server rebuilds by itself: what it computes from the music files, what it
|
||||
// fetches again from MusicBrainz, ListenBrainz and the other providers, and
|
||||
// logs of past runs. Excluded says why for each, and the coverage test fails
|
||||
// on a table that is in neither list.
|
||||
package backup
|
||||
|
||||
// Table is one table a backup holds.
|
||||
//
|
||||
// Omit names columns left out of the file. They restore to their defaults:
|
||||
// - a credential Minstrel stores sealed, which only the server holding the
|
||||
// stream secret could open, so the restored server asks for it again;
|
||||
// - a marker saying work on an excluded table is done (a tag source, an art
|
||||
// source), so the restored server does that work again rather than
|
||||
// believing it has the result.
|
||||
type Table struct {
|
||||
Name string
|
||||
Omit []string
|
||||
}
|
||||
|
||||
// Tables is every table a backup holds, in the order a restore loads them:
|
||||
// each after every table it references. There are no self-referencing
|
||||
// tables, so no table needs its rows loaded in depth order.
|
||||
var Tables = []Table{
|
||||
{Name: "users", Omit: []string{"subsonic_password", "listenbrainz_token"}},
|
||||
|
||||
// The library's identity. Every like, play and playlist entry hangs off a
|
||||
// track id, so the rows travel even though a scan could rebuild most of
|
||||
// their columns; the scan after a restore finds each file at its path and
|
||||
// keeps its id.
|
||||
{Name: "artists", Omit: []string{"artist_thumb_path", "artist_fanart_path", "artist_art_source", "artist_art_sources_version"}},
|
||||
{Name: "albums", Omit: []string{"cover_art_path", "cover_art_source", "cover_art_sources_version"}},
|
||||
{Name: "tracks", Omit: []string{"tag_source", "tag_sources_version"}},
|
||||
|
||||
// Settings.
|
||||
{Name: "library_settings"},
|
||||
{Name: "network_settings"},
|
||||
{Name: "registration_settings"},
|
||||
{Name: "smtp_config", Omit: []string{"password"}},
|
||||
{Name: "notification_email_settings"},
|
||||
{Name: "acoustid_settings", Omit: []string{"api_key"}},
|
||||
{Name: "fingerprint_settings"},
|
||||
{Name: "loudness_settings"},
|
||||
{Name: "cover_art_provider_settings", Omit: []string{"api_key"}},
|
||||
{Name: "cover_art_sources_meta"},
|
||||
{Name: "tag_provider_settings", Omit: []string{"api_key"}},
|
||||
{Name: "tag_sources_meta"},
|
||||
{Name: "acquisition_services", Omit: []string{"api_key", "password"}},
|
||||
{Name: "acquisition_settings"},
|
||||
{Name: "reacquisition_settings"},
|
||||
{Name: "discover_tuning"},
|
||||
{Name: "taste_tuning"},
|
||||
{Name: "recommendation_weight_profiles"},
|
||||
{Name: "recommendation_tuning_audit"},
|
||||
|
||||
// Follows, wants, requests and downloads.
|
||||
{Name: "release_type_profiles"},
|
||||
{Name: "follows"},
|
||||
{Name: "follow_seen_groups"},
|
||||
{Name: "wants"},
|
||||
{Name: "grabs"},
|
||||
{Name: "download_interventions"},
|
||||
{Name: "release_blocks"},
|
||||
{Name: "requests"},
|
||||
|
||||
// Listening history and likes.
|
||||
{Name: "play_sessions"},
|
||||
{Name: "play_events"},
|
||||
{Name: "skip_events"},
|
||||
{Name: "scrobble_queue"},
|
||||
{Name: "contextual_likes"},
|
||||
{Name: "general_likes"},
|
||||
{Name: "general_likes_albums"},
|
||||
{Name: "general_likes_artists"},
|
||||
|
||||
// Playlists and what is built from history. The taste profile and the
|
||||
// "you might like" rows are rebuilt nightly from the history above; they
|
||||
// travel so a restored Home is full before that first nightly pass.
|
||||
{Name: "playlists"},
|
||||
{Name: "playlist_tracks"},
|
||||
{Name: "system_playlist_runs"},
|
||||
{Name: "system_playlist_rotation_state"},
|
||||
{Name: "taste_profile_artists"},
|
||||
{Name: "taste_profile_tags"},
|
||||
{Name: "taste_profile_eras"},
|
||||
{Name: "taste_profile_moods"},
|
||||
{Name: "you_might_like_albums"},
|
||||
{Name: "you_might_like_artists"},
|
||||
|
||||
// Per-user preferences and notifications.
|
||||
{Name: "user_normalization_prefs"},
|
||||
{Name: "user_notification_prefs"},
|
||||
{Name: "user_notification_email_state"},
|
||||
{Name: "user_notifications"},
|
||||
{Name: "suggestion_snoozes"},
|
||||
{Name: "quarantine"},
|
||||
{Name: "quarantine_actions"},
|
||||
|
||||
// Admin decisions: an album taken off the wanted listing stays off, and a
|
||||
// dismissed duplicate group stays dismissed when the sweep finds it again.
|
||||
{Name: "library_album_matches"},
|
||||
{Name: "duplicate_sweeps"},
|
||||
{Name: "duplicate_groups"},
|
||||
{Name: "duplicate_group_members"},
|
||||
|
||||
// The library change feed is how the apps sync, including a first sync,
|
||||
// so a restored server carries it on from where the backup left it.
|
||||
{Name: "library_changes"},
|
||||
{Name: "audit_log"},
|
||||
{Name: "playback_errors"},
|
||||
}
|
||||
|
||||
// Excluded is every table a backup leaves out, with the reason.
|
||||
var Excluded = map[string]string{
|
||||
"schema_migrations": "the restored server's own migrations write it; the backup's schema version is in its manifest",
|
||||
"sessions": "sign-ins; everyone signs in again on the restored server",
|
||||
"password_resets": "one-time reset links, which expire within hours",
|
||||
"user_invites": "open sign-up links; an admin sends an invite again from the restored server",
|
||||
|
||||
"track_fingerprints": "computed from the music files",
|
||||
"track_loudness": "computed from the music files",
|
||||
"album_loudness": "computed from the music files",
|
||||
"track_acoustid_lookups": "looked up again from each track's fingerprint",
|
||||
"track_tags": "fetched again from the tag providers",
|
||||
"candidate_artist_tags": "fetched again from the tag providers",
|
||||
"candidate_artist_tag_state": "fetched again from the tag providers",
|
||||
|
||||
"mb_artists": "MusicBrainz cache, fetched again",
|
||||
"mb_release_groups": "MusicBrainz cache, fetched again",
|
||||
"mb_releases": "MusicBrainz cache, fetched again",
|
||||
"mb_release_tracks": "MusicBrainz cache, fetched again",
|
||||
"mb_artist_release_groups": "MusicBrainz cache, fetched again",
|
||||
|
||||
"listenbrainz_similar_recordings": "similarity, fetched again from ListenBrainz",
|
||||
"track_similarity": "similarity, fetched again from the similarity providers",
|
||||
"track_similarity_fetches": "similarity, fetched again from the similarity providers",
|
||||
"artist_similarity": "similarity, fetched again from the similarity providers",
|
||||
"artist_similarity_fetches": "similarity, fetched again from the similarity providers",
|
||||
"artist_similarity_unmatched": "similarity, fetched again from the similarity providers",
|
||||
|
||||
"want_candidates": "the last search's results; the next search finds them again",
|
||||
"scan_runs": "a log of past library scans",
|
||||
"diagnostic_events": "debug-mode diagnostics",
|
||||
}
|
||||
|
||||
// carried reports whether a table is in Tables.
|
||||
func carried(name string) bool {
|
||||
for _, t := range Tables {
|
||||
if t.Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Coverage checks a schema's tables against Tables and Excluded. It returns
|
||||
// one line per problem: a table in neither list, in both, or listed twice,
|
||||
// and a listed table the schema doesn't have.
|
||||
func Coverage(schema []string) []string {
|
||||
var problems []string
|
||||
inSchema := map[string]bool{}
|
||||
for _, name := range schema {
|
||||
inSchema[name] = true
|
||||
_, excluded := Excluded[name]
|
||||
switch {
|
||||
case carried(name) && excluded:
|
||||
problems = append(problems, name+": both backed up and excluded")
|
||||
case !carried(name) && !excluded:
|
||||
problems = append(problems, name+": neither backed up nor excluded; add it to Tables, or to Excluded with the reason")
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, t := range Tables {
|
||||
if seen[t.Name] {
|
||||
problems = append(problems, t.Name+": listed twice in Tables")
|
||||
}
|
||||
seen[t.Name] = true
|
||||
if !inSchema[t.Name] {
|
||||
problems = append(problems, t.Name+": in Tables but not in the schema")
|
||||
}
|
||||
}
|
||||
for name := range Excluded {
|
||||
if !inSchema[name] {
|
||||
problems = append(problems, name+": in Excluded but not in the schema")
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func everyListedTable() []string {
|
||||
var names []string
|
||||
for _, t := range Tables {
|
||||
names = append(names, t.Name)
|
||||
}
|
||||
for name := range Excluded {
|
||||
names = append(names, name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func TestCoverageAcceptsASchemaThatMatchesTheLists(t *testing.T) {
|
||||
if problems := Coverage(everyListedTable()); len(problems) != 0 {
|
||||
t.Fatalf("problems = %v, want none", problems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageNamesATableInNeitherList(t *testing.T) {
|
||||
problems := Coverage(append(everyListedTable(), "new_table"))
|
||||
if len(problems) != 1 || !strings.HasPrefix(problems[0], "new_table: neither") {
|
||||
t.Fatalf("problems = %v, want one naming new_table", problems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageNamesAListedTableTheSchemaLacks(t *testing.T) {
|
||||
schema := slices.DeleteFunc(everyListedTable(), func(n string) bool { return n == "playlists" || n == "scan_runs" })
|
||||
problems := Coverage(schema)
|
||||
if len(problems) != 2 {
|
||||
t.Fatalf("problems = %v, want playlists and scan_runs named", problems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryExclusionHasAReason(t *testing.T) {
|
||||
for name, why := range Excluded {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
t.Errorf("%s is excluded without a reason", name)
|
||||
}
|
||||
if carried(name) {
|
||||
t.Errorf("%s is both backed up and excluded", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// SettingsService reconciles the registry of compiled-in providers
|
||||
@@ -112,7 +113,7 @@ func (s *SettingsService) reconcile(ctx context.Context) error {
|
||||
}
|
||||
key := ""
|
||||
if row.ApiKey != nil {
|
||||
key = *row.ApiKey
|
||||
key = secret.Reveal(secret.CoverArtAPIKey.Purpose(), row.ProviderID, *row.ApiKey)
|
||||
}
|
||||
s.enabledIDs[p.ID()] = row.Enabled
|
||||
s.apiKeys[p.ID()] = key
|
||||
@@ -204,11 +205,15 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
|
||||
// (apiKeyChanged flag) and $4 (api_key value). The actual generated
|
||||
// field for $4 is ApiKey *string.
|
||||
apiKeyChanged := patch.APIKey != nil
|
||||
sealedKey, err := secret.SealPtr(secret.CoverArtAPIKey.Purpose(), providerID, patch.APIKey)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("seal provider api key: %w", err)
|
||||
}
|
||||
if err := q.UpdateProviderSettings(ctx, dbq.UpdateProviderSettingsParams{
|
||||
ProviderID: providerID,
|
||||
Enabled: enabledVal,
|
||||
Column3: apiKeyChanged,
|
||||
ApiKey: patch.APIKey,
|
||||
ApiKey: sealedKey,
|
||||
}); err != nil {
|
||||
return false, fmt.Errorf("update provider settings: %w", err)
|
||||
}
|
||||
@@ -225,7 +230,7 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
|
||||
for _, r := range rows {
|
||||
key := ""
|
||||
if r.ApiKey != nil {
|
||||
key = *r.ApiKey
|
||||
key = secret.Reveal(secret.CoverArtAPIKey.Purpose(), r.ProviderID, *r.ApiKey)
|
||||
}
|
||||
s.enabledIDs[r.ProviderID] = r.Enabled
|
||||
s.apiKeys[r.ProviderID] = key
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package dbtest
|
||||
|
||||
import "git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
|
||||
// Integration tests run with a master key installed, as the server runs
|
||||
// after startup, so code that seals or opens a stored credential behaves as
|
||||
// it does in production.
|
||||
func init() {
|
||||
secret.Install([]byte("minstrel integration tests: not a deployment key"))
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// AcoustID lookup settings: a database row, changed without a restart, shared
|
||||
@@ -97,8 +98,12 @@ func (s *AcoustIDSettingsService) Set(ctx context.Context, in AcoustIDSettingsUp
|
||||
}
|
||||
params := dbq.UpdateAcoustIDSettingsParams{Enabled: in.Enabled, MinScore: in.MinScore}
|
||||
if in.APIKey != nil {
|
||||
sealed, err := secret.Seal(secret.AcoustIDAPIKey.Purpose(), "", *in.APIKey)
|
||||
if err != nil {
|
||||
return AcoustIDSettings{}, fmt.Errorf("acoustid settings: seal key: %w", err)
|
||||
}
|
||||
params.SetApiKey = true
|
||||
params.ApiKey = *in.APIKey
|
||||
params.ApiKey = sealed
|
||||
}
|
||||
row, err := dbq.New(s.pool).UpdateAcoustIDSettings(ctx, params)
|
||||
if err != nil {
|
||||
@@ -126,7 +131,7 @@ func (s *AcoustIDSettingsService) Changed() <-chan struct{} {
|
||||
func acoustIDSettingsFromRow(row dbq.AcoustidSetting) AcoustIDSettings {
|
||||
out := AcoustIDSettings{Enabled: row.Enabled, MinScore: row.MinScore, UpdatedAt: row.UpdatedAt.Time}
|
||||
if row.ApiKey != nil {
|
||||
out.APIKey = *row.ApiKey
|
||||
out.APIKey = secret.Reveal(secret.AcoustIDAPIKey.Purpose(), "", *row.ApiKey)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// ErrNotConfigured is returned when smtp_config.enabled is false or
|
||||
@@ -68,7 +69,8 @@ func (s *SMTPSender) Send(ctx context.Context, to, subject, textBody, htmlBody s
|
||||
|
||||
var auth smtp.Auth
|
||||
if cfg.Username != "" {
|
||||
auth = smtp.PlainAuth("", cfg.Username, cfg.Password, cfg.Host)
|
||||
password := secret.Reveal(secret.SMTPPassword.Purpose(), "", cfg.Password)
|
||||
auth = smtp.PlainAuth("", cfg.Username, password, cfg.Host)
|
||||
}
|
||||
|
||||
// We respect the use_tls flag: if true, we require STARTTLS via
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
func testPool(t *testing.T) (*pgxpool.Pool, *dbq.Queries) {
|
||||
@@ -63,8 +64,13 @@ func seed(t *testing.T, opts seedOpts) setup {
|
||||
t.Fatalf("user: %v", err)
|
||||
}
|
||||
if opts.lbToken != "" {
|
||||
// Stored sealed, as the settings route stores it.
|
||||
sealed, err := secret.Seal(secret.ListenBrainzToken.Purpose(), u.ID.String(), opts.lbToken)
|
||||
if err != nil {
|
||||
t.Fatalf("seal token: %v", err)
|
||||
}
|
||||
if err := q.SetListenBrainzToken(ctx, dbq.SetListenBrainzTokenParams{
|
||||
ID: u.ID, ListenbrainzToken: &opts.lbToken,
|
||||
ID: u.ID, ListenbrainzToken: &sealed,
|
||||
}); err != nil {
|
||||
t.Fatalf("token: %v", err)
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble/listenbrainz"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// backoffSchedule maps the failure count (1-indexed) to the delay before
|
||||
@@ -98,7 +99,11 @@ func (w *Worker) tickOnce(ctx context.Context) error {
|
||||
key := r.UserID.String()
|
||||
ub := batches[key]
|
||||
if ub == nil {
|
||||
ub = &userBatch{token: *r.LbToken, userID: r.UserID}
|
||||
token := secret.Reveal(secret.ListenBrainzToken.Purpose(), key, *r.LbToken)
|
||||
if token == "" {
|
||||
continue
|
||||
}
|
||||
ub = &userBatch{token: token, userID: r.UserID}
|
||||
batches[key] = ub
|
||||
}
|
||||
ub.queueIDs = append(ub.queueIDs, r.QueueID)
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
// Package secret seals the credentials Minstrel keeps for outside systems
|
||||
// (indexer and download-client keys, the SMTP password, provider API keys,
|
||||
// ListenBrainz tokens, Subsonic passwords) so the database holds ciphertext,
|
||||
// not the credential.
|
||||
//
|
||||
// Each column has its own key, derived from the install's master key with
|
||||
// HKDF-SHA256 and the column as the info string, so a value sealed for one
|
||||
// column cannot be opened as another, and the master key never touches a
|
||||
// cipher. Each value is also bound to the row it belongs to (its owner), so a
|
||||
// sealed value copied onto another row does not open there.
|
||||
//
|
||||
// The master key is the stream secret (MINSTREL_STREAM_SECRET, or the
|
||||
// stream_secret file in the data directory). It lives outside the database,
|
||||
// so a copy of the database alone opens nothing. Changing it makes every
|
||||
// sealed value unreadable: each one reads as unset and has to be entered
|
||||
// again in Settings.
|
||||
package secret
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/hkdf"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// ErrNoKey is a seal or open before Install has given the package a key.
|
||||
var ErrNoKey = errors.New("secret: no master key installed")
|
||||
|
||||
// ErrCorrupt is a stored value that does not open: the master key changed,
|
||||
// the value was edited by hand, or it belongs to a different row.
|
||||
var ErrCorrupt = errors.New("secret: the stored value does not open")
|
||||
|
||||
// sealedPrefix versions the format, so a future change of cipher can still
|
||||
// read what this one wrote. Nothing a person types starts with it by chance
|
||||
// often enough to matter, and Open rejects anything without it.
|
||||
const sealedPrefix = "v1."
|
||||
|
||||
var b64 = base64.RawURLEncoding
|
||||
|
||||
var (
|
||||
mu sync.RWMutex
|
||||
master []byte
|
||||
boxes map[string]cipher.AEAD
|
||||
)
|
||||
|
||||
// Install sets the master key every Seal and Open derives from. The server
|
||||
// calls it once at startup, before anything reads a credential.
|
||||
func Install(key []byte) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
master = append([]byte(nil), key...)
|
||||
boxes = map[string]cipher.AEAD{}
|
||||
}
|
||||
|
||||
// aead returns purpose's cipher, deriving it on first use.
|
||||
func aead(purpose string) (cipher.AEAD, error) {
|
||||
mu.RLock()
|
||||
a, ok := boxes[purpose]
|
||||
key := master
|
||||
mu.RUnlock()
|
||||
if ok {
|
||||
return a, nil
|
||||
}
|
||||
if len(key) == 0 {
|
||||
return nil, ErrNoKey
|
||||
}
|
||||
derived, err := hkdf.Key(sha256.New, key, nil, "minstrel/"+purpose, 32)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("secret: derive: %w", err)
|
||||
}
|
||||
block, err := aes.NewCipher(derived)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("secret: %w", err)
|
||||
}
|
||||
a, err = cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("secret: %w", err)
|
||||
}
|
||||
mu.Lock()
|
||||
if boxes != nil {
|
||||
boxes[purpose] = a
|
||||
}
|
||||
mu.Unlock()
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// IsSealed reports whether stored is in the sealed format.
|
||||
func IsSealed(stored string) bool { return strings.HasPrefix(stored, sealedPrefix) }
|
||||
|
||||
// Seal encrypts plain for purpose (the column it is stored in) and owner (the
|
||||
// row it belongs to). The empty string stays empty, so "is one set?" stays a
|
||||
// test for the empty string.
|
||||
func Seal(purpose, owner, plain string) (string, error) {
|
||||
if plain == "" {
|
||||
return "", nil
|
||||
}
|
||||
a, err := aead(purpose)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
nonce := make([]byte, a.NonceSize())
|
||||
if _, err := rand.Read(nonce); err != nil {
|
||||
return "", fmt.Errorf("secret: nonce: %w", err)
|
||||
}
|
||||
out := a.Seal(nonce, nonce, []byte(plain), []byte(owner))
|
||||
return sealedPrefix + b64.EncodeToString(out), nil
|
||||
}
|
||||
|
||||
// Open decrypts what Seal stored for the same purpose and owner. The empty
|
||||
// string opens to itself.
|
||||
func Open(purpose, owner, stored string) (string, error) {
|
||||
if stored == "" {
|
||||
return "", nil
|
||||
}
|
||||
a, err := aead(purpose)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
raw, ok := strings.CutPrefix(stored, sealedPrefix)
|
||||
if !ok {
|
||||
return "", ErrCorrupt
|
||||
}
|
||||
data, err := b64.DecodeString(raw)
|
||||
n := a.NonceSize()
|
||||
if err != nil || len(data) < n {
|
||||
return "", ErrCorrupt
|
||||
}
|
||||
plain, err := a.Open(nil, data[:n], data[n:], []byte(owner))
|
||||
if err != nil {
|
||||
return "", ErrCorrupt
|
||||
}
|
||||
return string(plain), nil
|
||||
}
|
||||
|
||||
// Reveal is Open for a caller about to use the credential: a value that does
|
||||
// not open is logged and read as unset, which is what it has become. The
|
||||
// admin sees "not set" in Settings and enters it again.
|
||||
func Reveal(purpose, owner, stored string) string {
|
||||
plain, err := Open(purpose, owner, stored)
|
||||
if err != nil {
|
||||
slog.Warn("secret: a stored credential does not open; it reads as unset until it is entered again",
|
||||
"column", purpose, "err", err)
|
||||
return ""
|
||||
}
|
||||
return plain
|
||||
}
|
||||
|
||||
// RevealPtr is Reveal for a nullable column: nil stays nil, and a value that
|
||||
// does not open reads as nil.
|
||||
func RevealPtr(purpose, owner string, stored *string) *string {
|
||||
if stored == nil {
|
||||
return nil
|
||||
}
|
||||
plain := Reveal(purpose, owner, *stored)
|
||||
if plain == "" {
|
||||
return nil
|
||||
}
|
||||
return &plain
|
||||
}
|
||||
|
||||
// SealPtr is Seal for a nullable column: nil and the empty string both store
|
||||
// NULL.
|
||||
func SealPtr(purpose, owner string, plain *string) (*string, error) {
|
||||
if plain == nil || *plain == "" {
|
||||
return nil, nil
|
||||
}
|
||||
sealed, err := Seal(purpose, owner, *plain)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &sealed, nil
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package secret
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func withKey(t *testing.T, key string) {
|
||||
t.Helper()
|
||||
mu.RLock()
|
||||
prev := master
|
||||
mu.RUnlock()
|
||||
Install([]byte(key))
|
||||
t.Cleanup(func() { Install(prev) })
|
||||
}
|
||||
|
||||
func TestSealOpensForTheSamePurposeAndOwner(t *testing.T) {
|
||||
withKey(t, "test master key")
|
||||
sealed, err := Seal("t.col", "row-1", "hunter2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !IsSealed(sealed) || strings.Contains(sealed, "hunter2") {
|
||||
t.Fatalf("sealed = %q, want an opaque v1. value", sealed)
|
||||
}
|
||||
plain, err := Open("t.col", "row-1", sealed)
|
||||
if err != nil || plain != "hunter2" {
|
||||
t.Fatalf("Open = %q, %v; want hunter2", plain, err)
|
||||
}
|
||||
again, _ := Seal("t.col", "row-1", "hunter2")
|
||||
if again == sealed {
|
||||
t.Error("two seals of one value are identical; the nonce is not random")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenRefusesAnythingElse(t *testing.T) {
|
||||
withKey(t, "test master key")
|
||||
sealed, _ := Seal("t.col", "row-1", "hunter2")
|
||||
|
||||
cases := map[string]func() (string, error){
|
||||
"another owner": func() (string, error) { return Open("t.col", "row-2", sealed) },
|
||||
"another purpose": func() (string, error) { return Open("t.other", "row-1", sealed) },
|
||||
"plain text": func() (string, error) { return Open("t.col", "row-1", "hunter2") },
|
||||
"truncated": func() (string, error) { return Open("t.col", "row-1", sealed[:len(sealed)-4]) },
|
||||
"not base64": func() (string, error) { return Open("t.col", "row-1", "v1.!!!") },
|
||||
"too short": func() (string, error) { return Open("t.col", "row-1", "v1.AAAA") },
|
||||
}
|
||||
for name, open := range cases {
|
||||
if _, err := open(); !errors.Is(err, ErrCorrupt) {
|
||||
t.Errorf("%s: err = %v, want ErrCorrupt", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
Install([]byte("a different key"))
|
||||
if _, err := Open("t.col", "row-1", sealed); !errors.Is(err, ErrCorrupt) {
|
||||
t.Errorf("another master key: err = %v, want ErrCorrupt", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyStaysEmpty(t *testing.T) {
|
||||
withKey(t, "test master key")
|
||||
if s, err := Seal("t.col", "o", ""); s != "" || err != nil {
|
||||
t.Errorf("Seal(\"\") = %q, %v; want the empty string", s, err)
|
||||
}
|
||||
if s, err := Open("t.col", "o", ""); s != "" || err != nil {
|
||||
t.Errorf("Open(\"\") = %q, %v; want the empty string", s, err)
|
||||
}
|
||||
empty := ""
|
||||
if p, err := SealPtr("t.col", "o", &empty); p != nil || err != nil {
|
||||
t.Errorf("SealPtr(\"\") = %v, %v; want nil", p, err)
|
||||
}
|
||||
if p := RevealPtr("t.col", "o", nil); p != nil {
|
||||
t.Errorf("RevealPtr(nil) = %v, want nil", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoKeyIsAnErrorNotAPanic(t *testing.T) {
|
||||
withKey(t, "")
|
||||
if _, err := Seal("t.col", "o", "x"); !errors.Is(err, ErrNoKey) {
|
||||
t.Errorf("Seal: err = %v, want ErrNoKey", err)
|
||||
}
|
||||
if _, err := Open("t.col", "o", "v1.AAAA"); !errors.Is(err, ErrNoKey) {
|
||||
t.Errorf("Open: err = %v, want ErrNoKey", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRevealReadsAnUnopenableValueAsUnset(t *testing.T) {
|
||||
withKey(t, "test master key")
|
||||
if got := Reveal("t.col", "o", "not sealed"); got != "" {
|
||||
t.Errorf("Reveal = %q, want the empty string", got)
|
||||
}
|
||||
bad := "not sealed"
|
||||
if got := RevealPtr("t.col", "o", &bad); got != nil {
|
||||
t.Errorf("RevealPtr = %q, want nil", *got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryColumnHasADistinctPurpose(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
for _, c := range Columns {
|
||||
if seen[c.Purpose()] {
|
||||
t.Errorf("%s is listed twice", c.Purpose())
|
||||
}
|
||||
seen[c.Purpose()] = true
|
||||
}
|
||||
if !seen[AcquisitionAPIKey.Purpose()] || !seen[SubsonicPassword.Purpose()] {
|
||||
t.Error("Columns is missing a declared column")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package secret
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Column is one database column that holds a sealed credential.
|
||||
//
|
||||
// Its Purpose ("table.column") is the key-derivation label every Seal and
|
||||
// Open of that column passes. The owner a value is bound to is the row's key
|
||||
// as text (KeyColumn::text), or "" for a single-row settings table.
|
||||
type Column struct {
|
||||
Table string
|
||||
Column string
|
||||
KeyColumn string
|
||||
Singleton bool
|
||||
}
|
||||
|
||||
// Purpose is the label Seal and Open take for this column.
|
||||
func (c Column) Purpose() string { return c.Table + "." + c.Column }
|
||||
|
||||
// The sealed columns, and the purpose strings their callers pass. Every
|
||||
// credential Minstrel stores for an outside system, or that must be read back
|
||||
// in the clear, is listed here.
|
||||
var (
|
||||
AcquisitionAPIKey = Column{"acquisition_services", "api_key", "code", false}
|
||||
AcquisitionPassword = Column{"acquisition_services", "password", "code", false}
|
||||
AcoustIDAPIKey = Column{"acoustid_settings", "api_key", "id", true}
|
||||
SMTPPassword = Column{"smtp_config", "password", "id", true}
|
||||
ListenBrainzToken = Column{"users", "listenbrainz_token", "id", false}
|
||||
SubsonicPassword = Column{"users", "subsonic_password", "id", false}
|
||||
CoverArtAPIKey = Column{"cover_art_provider_settings", "api_key", "provider_id", false}
|
||||
TagProviderAPIKey = Column{"tag_provider_settings", "api_key", "provider_id", false}
|
||||
)
|
||||
|
||||
// Columns is every sealed column, for SealStored and for anything that has to
|
||||
// know which columns hold credentials (a backup leaves them out).
|
||||
var Columns = []Column{
|
||||
AcquisitionAPIKey, AcquisitionPassword, AcoustIDAPIKey, SMTPPassword,
|
||||
ListenBrainzToken, SubsonicPassword, CoverArtAPIKey, TagProviderAPIKey,
|
||||
}
|
||||
|
||||
// Owner is the owner string for a row whose key is key. A single-row table's
|
||||
// rows are owned by "".
|
||||
func (c Column) Owner(key string) string {
|
||||
if c.Singleton {
|
||||
return ""
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// SealStored seals every value in Columns that is still stored in the clear,
|
||||
// and returns how many it sealed. The server runs it at startup, after
|
||||
// migrations and before anything reads a credential, so a database written
|
||||
// before sealing existed is brought over in place. Each row is updated only
|
||||
// if it still holds the value that was read, so a concurrent write wins.
|
||||
func SealStored(ctx context.Context, pool *pgxpool.Pool) (int, error) {
|
||||
total := 0
|
||||
for _, c := range Columns {
|
||||
n, err := sealColumn(ctx, pool, c)
|
||||
total += n
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("seal %s: %w", c.Purpose(), err)
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func sealColumn(ctx context.Context, pool *pgxpool.Pool, c Column) (int, error) {
|
||||
table := pgx.Identifier{c.Table}.Sanitize()
|
||||
col := pgx.Identifier{c.Column}.Sanitize()
|
||||
key := pgx.Identifier{c.KeyColumn}.Sanitize()
|
||||
|
||||
rows, err := pool.Query(ctx, fmt.Sprintf(
|
||||
`SELECT %s::text, %s FROM %s WHERE %s IS NOT NULL AND %s <> '' AND %s NOT LIKE 'v1.%%'`,
|
||||
key, col, table, col, col, col))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
type pending struct{ key, plain string }
|
||||
var todo []pending
|
||||
for rows.Next() {
|
||||
var p pending
|
||||
if err := rows.Scan(&p.key, &p.plain); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
todo = append(todo, p)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
update := fmt.Sprintf(`UPDATE %s SET %s = $1 WHERE %s::text = $2 AND %s = $3`, table, col, key, col)
|
||||
sealed := 0
|
||||
for _, p := range todo {
|
||||
v, err := Seal(c.Purpose(), c.Owner(p.key), p.plain)
|
||||
if err != nil {
|
||||
return sealed, err
|
||||
}
|
||||
if _, err := pool.Exec(ctx, update, v, p.key, p.plain); err != nil {
|
||||
return sealed, err
|
||||
}
|
||||
sealed++
|
||||
}
|
||||
return sealed, nil
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package secret_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
func newPool(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in -short mode")
|
||||
}
|
||||
dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
|
||||
}
|
||||
if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("pool: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
dbtest.ResetDB(t, pool)
|
||||
return pool
|
||||
}
|
||||
|
||||
// Every listed column exists, so a renamed or dropped column fails here
|
||||
// rather than leaving a credential unsealed.
|
||||
func TestColumnsExistInTheSchema_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
for _, c := range secret.Columns {
|
||||
var n int
|
||||
err := pool.QueryRow(context.Background(),
|
||||
`SELECT count(*) FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = $1 AND column_name IN ($2, $3)`,
|
||||
c.Table, c.Column, c.KeyColumn).Scan(&n)
|
||||
if err != nil || n != 2 {
|
||||
t.Errorf("%s (key %s): found %d of 2 columns, err %v", c.Purpose(), c.KeyColumn, n, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A database written before sealing existed is sealed in place at startup,
|
||||
// every value opens to what it held, and a second pass has nothing to do.
|
||||
func TestSealStoredSealsWhatIsInTheClear_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
ctx := context.Background()
|
||||
q := dbq.New(pool)
|
||||
|
||||
u, err := q.CreateUser(ctx, dbq.CreateUserParams{
|
||||
Username: dbtest.TestUserPrefix + "sealer", PasswordHash: "x", ApiTokenHash: "sealer", IsAdmin: false,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("user: %v", err)
|
||||
}
|
||||
if _, err := q.CreateAcquisitionService(ctx, dbq.CreateAcquisitionServiceParams{
|
||||
Code: "seal-test", Kind: "search", Protocol: "prowlarr", Label: "Seal test",
|
||||
Category: "", TimeoutMs: 1000, SearchesPerHour: 0,
|
||||
}); err != nil {
|
||||
t.Fatalf("acquisition row: %v", err)
|
||||
}
|
||||
seed := []string{
|
||||
`UPDATE acquisition_services SET api_key = 'acq-key', password = 'acq-pw' WHERE code = 'seal-test'`,
|
||||
`UPDATE acoustid_settings SET api_key = 'acoustid-key' WHERE id = true`,
|
||||
`UPDATE smtp_config SET password = 'smtp-pw' WHERE id = true`,
|
||||
fmt.Sprintf(`UPDATE users SET listenbrainz_token = 'lb-token', subsonic_password = 'sub-pw' WHERE id = '%s'`, u.ID.String()),
|
||||
`INSERT INTO cover_art_provider_settings (provider_id, enabled, display_order, api_key)
|
||||
VALUES ('seal-test', false, 99, 'cover-key')
|
||||
ON CONFLICT (provider_id) DO UPDATE SET api_key = EXCLUDED.api_key`,
|
||||
`INSERT INTO tag_provider_settings (provider_id, enabled, display_order, api_key)
|
||||
VALUES ('seal-test', false, 99, 'tag-key')
|
||||
ON CONFLICT (provider_id) DO UPDATE SET api_key = EXCLUDED.api_key`,
|
||||
}
|
||||
for _, s := range seed {
|
||||
if _, err := pool.Exec(ctx, s); err != nil {
|
||||
t.Fatalf("seed %q: %v", s, err)
|
||||
}
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
for _, s := range []string{
|
||||
`DELETE FROM cover_art_provider_settings WHERE provider_id = 'seal-test'`,
|
||||
`DELETE FROM tag_provider_settings WHERE provider_id = 'seal-test'`,
|
||||
`DELETE FROM acquisition_services WHERE code = 'seal-test'`,
|
||||
`UPDATE acoustid_settings SET api_key = NULL WHERE id = true`,
|
||||
`UPDATE smtp_config SET password = '' WHERE id = true`,
|
||||
} {
|
||||
_, _ = pool.Exec(context.Background(), s)
|
||||
}
|
||||
})
|
||||
|
||||
want := []struct {
|
||||
col secret.Column
|
||||
key string
|
||||
plain string
|
||||
}{
|
||||
{secret.AcquisitionAPIKey, "seal-test", "acq-key"},
|
||||
{secret.AcquisitionPassword, "seal-test", "acq-pw"},
|
||||
{secret.AcoustIDAPIKey, "true", "acoustid-key"},
|
||||
{secret.SMTPPassword, "true", "smtp-pw"},
|
||||
{secret.ListenBrainzToken, u.ID.String(), "lb-token"},
|
||||
{secret.SubsonicPassword, u.ID.String(), "sub-pw"},
|
||||
{secret.CoverArtAPIKey, "seal-test", "cover-key"},
|
||||
{secret.TagProviderAPIKey, "seal-test", "tag-key"},
|
||||
}
|
||||
if len(want) != len(secret.Columns) {
|
||||
t.Fatalf("the test covers %d columns, secret.Columns lists %d", len(want), len(secret.Columns))
|
||||
}
|
||||
|
||||
n, err := secret.SealStored(ctx, pool)
|
||||
if err != nil {
|
||||
t.Fatalf("SealStored: %v", err)
|
||||
}
|
||||
if n < len(want) {
|
||||
t.Errorf("sealed %d values, want at least %d", n, len(want))
|
||||
}
|
||||
|
||||
for _, w := range want {
|
||||
var stored string
|
||||
query := fmt.Sprintf(`SELECT %s FROM %s WHERE %s::text = $1`,
|
||||
pgx.Identifier{w.col.Column}.Sanitize(), pgx.Identifier{w.col.Table}.Sanitize(),
|
||||
pgx.Identifier{w.col.KeyColumn}.Sanitize())
|
||||
if err := pool.QueryRow(ctx, query, w.key).Scan(&stored); err != nil {
|
||||
t.Errorf("%s: read: %v", w.col.Purpose(), err)
|
||||
continue
|
||||
}
|
||||
if !secret.IsSealed(stored) {
|
||||
t.Errorf("%s = %q, want it sealed", w.col.Purpose(), stored)
|
||||
continue
|
||||
}
|
||||
got, err := secret.Open(w.col.Purpose(), w.col.Owner(w.key), stored)
|
||||
if err != nil || got != w.plain {
|
||||
t.Errorf("%s opens to %q (err %v), want %q", w.col.Purpose(), got, err, w.plain)
|
||||
}
|
||||
}
|
||||
|
||||
again, err := secret.SealStored(ctx, pool)
|
||||
if err != nil || again != 0 {
|
||||
t.Errorf("second pass sealed %d (err %v), want 0", again, err)
|
||||
}
|
||||
}
|
||||
@@ -272,73 +272,41 @@ func TestDevChannel_PublishesDevAloneAndNeverLatest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The two bundling paths must stay mutually exclusive. If the rebundle step's
|
||||
// condition were relaxed back to "not a tag", a dev push would run BOTH: stage
|
||||
// its freshly-built APK, then overwrite it with the previous release's. The
|
||||
// image would still build and the sidecar would still parse — it would just
|
||||
// quietly serve stale art to the channel whose whole job is being current.
|
||||
func TestDevChannel_RebundlePathIsMainOnly(t *testing.T) {
|
||||
body, err := os.ReadFile(filepath.Join(repoRoot(t), ".gitea", "workflows", "release.yml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
yaml := string(body)
|
||||
// A merge to main is enough to put a new app in front of stable phones: main
|
||||
// builds the signed APK from its own commit and bakes it into :latest, as dev
|
||||
// does for :dev. The regression to guard is main falling back to a tag-only
|
||||
// APK build, which leaves every merged app change waiting on a release while
|
||||
// each build stays green.
|
||||
func TestEveryBranchShipsTheAppBuiltFromItsOwnCommit(t *testing.T) {
|
||||
jobs := releaseJobs(t)
|
||||
|
||||
i := strings.Index(yaml, "- name: Bundle latest release APK")
|
||||
if i < 0 {
|
||||
t.Fatal("no 'Bundle latest release APK' step")
|
||||
}
|
||||
step := yaml[i:]
|
||||
if j := strings.Index(step, "\n - name:"); j >= 0 {
|
||||
step = step[:j]
|
||||
}
|
||||
if !strings.Contains(step, "github.ref == 'refs/heads/main'") {
|
||||
t.Errorf("the rebundle step is not gated to main; a dev push would overwrite its own APK\n%s", step)
|
||||
}
|
||||
}
|
||||
|
||||
// The runner invokes `shell: bash` as `bash -e -o pipefail`. That makes any
|
||||
// command substitution whose grep matches NOTHING fatal at the assignment —
|
||||
// pipefail reports the grep's non-zero status even though `head` succeeded —
|
||||
// so the step dies before reaching the `if` written to handle the empty case.
|
||||
//
|
||||
// This is not hypothetical. It took down the first `main` build after the
|
||||
// version rework: the newest release predated sidecar assets, its
|
||||
// `.apk.version` grep matched nothing, and the step aborted instead of
|
||||
// falling through to the name-only branch that exists precisely for it. The
|
||||
// same latent hazard sat on the other two assignments and had simply never
|
||||
// fired, because their greps always matched.
|
||||
//
|
||||
// Pins the property that makes every "degrades gracefully" branch in that
|
||||
// step reachable at all.
|
||||
func TestBundleStep_GrepsCannotKillTheStep(t *testing.T) {
|
||||
body, err := os.ReadFile(filepath.Join(repoRoot(t), ".gitea", "workflows", "release.yml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
yaml := string(body)
|
||||
|
||||
i := strings.Index(yaml, "- name: Bundle latest release APK")
|
||||
if i < 0 {
|
||||
t.Fatal("no 'Bundle latest release APK' step")
|
||||
}
|
||||
step := yaml[i:]
|
||||
if j := strings.Index(step, "\n - name:"); j >= 0 {
|
||||
step = step[:j]
|
||||
}
|
||||
|
||||
for _, line := range strings.Split(step, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "#") || !strings.Contains(trimmed, "grep") {
|
||||
continue
|
||||
build := strings.Join(strings.Fields(jobs["android-release"].If), " ")
|
||||
for _, ref := range []string{"refs/heads/main", "refs/heads/dev", "refs/tags/v"} {
|
||||
if !strings.Contains(build, ref) {
|
||||
t.Errorf("android-release does not build on %s, so that ref's image carries no app of its own\n if: %s", ref, build)
|
||||
}
|
||||
// Only assignments from a command substitution can abort the step.
|
||||
if !strings.Contains(trimmed, `="$(`) {
|
||||
continue
|
||||
}
|
||||
if !strings.HasSuffix(trimmed, "|| true") {
|
||||
t.Errorf("this assignment dies under pipefail when its grep matches nothing, "+
|
||||
"skipping the empty-case branch below it:\n %s", trimmed)
|
||||
}
|
||||
|
||||
image := strings.Join(strings.Fields(jobs["image-release"].If), " ")
|
||||
if !strings.Contains(image, "needs.android-release.result == 'success'") || strings.Contains(image, "'skipped'") {
|
||||
t.Errorf("image-release can publish without a freshly-built APK\n if: %s", image)
|
||||
}
|
||||
|
||||
yaml := releaseYAML(t)
|
||||
if strings.Contains(yaml, "releases/latest") {
|
||||
t.Error("release.yml fetches the latest release; an image bundling a release's APK ships an app older than its own commit")
|
||||
}
|
||||
|
||||
// Both bundling steps run on every ref. Sliced from image-release on,
|
||||
// because release-assets has a step of the same name.
|
||||
i := strings.Index(yaml, "\n image-release:")
|
||||
if i < 0 {
|
||||
t.Fatal("no image-release job")
|
||||
}
|
||||
for _, name := range []string{"Download signed APK artifact", "Stage bundled APK + version sidecar"} {
|
||||
step := stepBody(t, yaml[i:], name)
|
||||
if strings.Contains(step, "github.ref") {
|
||||
t.Errorf("%q is gated by ref, so some ref's image ships without its APK\n%s", name, step)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -376,7 +344,7 @@ func TestReleaseBuild_RefreshesTheChannelAndMintsNothingElse(t *testing.T) {
|
||||
arm, _, _ := imageTagArms(t, releaseYAML(t))
|
||||
|
||||
if !strings.Contains(arm, "${IMAGE}:latest") {
|
||||
t.Errorf("tag arm does not refresh :latest — the channel would keep serving the PREVIOUS release's APK until someone pushed to main\n%s", arm)
|
||||
t.Errorf("tag arm does not refresh :latest — the server would hand out a different APK build from the one attached to the Release\n%s", arm)
|
||||
}
|
||||
if strings.Contains(arm, "GITHUB_SHA") {
|
||||
t.Errorf("tag arm mints a :<sha> image; that would re-push main's immutable rollback target with different bundled contents\n%s", arm)
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// Config controls wire-format concessions the server makes for Subsonic
|
||||
@@ -101,11 +102,15 @@ func authenticate(r *http.Request, pool *pgxpool.Pool, cfg Config) (dbq.User, in
|
||||
return dbq.User{}, ErrGeneric, "Auth lookup failed"
|
||||
}
|
||||
|
||||
// The stored Subsonic password is sealed; both t+s and p= compare against
|
||||
// it opened. One that does not open reads as not configured.
|
||||
subsonicPassword := secret.RevealPtr(secret.SubsonicPassword.Purpose(), user.ID.String(), user.SubsonicPassword)
|
||||
|
||||
if token, salt := params.Get("t"), params.Get("s"); token != "" && salt != "" {
|
||||
if user.SubsonicPassword == nil {
|
||||
if subsonicPassword == nil {
|
||||
return dbq.User{}, ErrTokenNotSupported, "Subsonic token auth not configured; use apiKey or set a Subsonic password"
|
||||
}
|
||||
sum := md5.Sum([]byte(*user.SubsonicPassword + salt))
|
||||
sum := md5.Sum([]byte(*subsonicPassword + salt))
|
||||
expected := hex.EncodeToString(sum[:])
|
||||
if subtle.ConstantTimeCompare([]byte(expected), []byte(token)) != 1 {
|
||||
return dbq.User{}, ErrWrongCredentials, "Wrong username or password"
|
||||
@@ -117,14 +122,14 @@ func authenticate(r *http.Request, pool *pgxpool.Pool, cfg Config) (dbq.User, in
|
||||
if !cfg.AllowPlaintextPassword {
|
||||
return dbq.User{}, ErrTokenNotSupported, "Plaintext password auth is disabled; use apiKey or t+s"
|
||||
}
|
||||
if user.SubsonicPassword == nil {
|
||||
if subsonicPassword == nil {
|
||||
return dbq.User{}, ErrTokenNotSupported, "Subsonic password not configured for this user"
|
||||
}
|
||||
decoded, err := decodePassword(password)
|
||||
if err != nil {
|
||||
return dbq.User{}, ErrWrongCredentials, "Malformed password"
|
||||
}
|
||||
if subtle.ConstantTimeCompare([]byte(*user.SubsonicPassword), []byte(decoded)) != 1 {
|
||||
if subtle.ConstantTimeCompare([]byte(*subsonicPassword), []byte(decoded)) != 1 {
|
||||
return dbq.User{}, ErrWrongCredentials, "Wrong username or password"
|
||||
}
|
||||
return user, 0, ""
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// SettingsService reconciles the compiled-in provider registry with the
|
||||
@@ -101,7 +102,7 @@ func (s *SettingsService) reconcile(ctx context.Context) error {
|
||||
}
|
||||
key := ""
|
||||
if row.ApiKey != nil {
|
||||
key = *row.ApiKey
|
||||
key = secret.Reveal(secret.TagProviderAPIKey.Purpose(), row.ProviderID, *row.ApiKey)
|
||||
}
|
||||
s.enabledIDs[p.ID()] = row.Enabled
|
||||
s.apiKeys[p.ID()] = key
|
||||
@@ -181,11 +182,15 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
|
||||
if patch.Enabled != nil {
|
||||
enabledVal = *patch.Enabled
|
||||
}
|
||||
sealedKey, err := secret.SealPtr(secret.TagProviderAPIKey.Purpose(), providerID, patch.APIKey)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("seal provider api key: %w", err)
|
||||
}
|
||||
if err := q.UpdateTagProviderSettings(ctx, dbq.UpdateTagProviderSettingsParams{
|
||||
ProviderID: providerID,
|
||||
Enabled: enabledVal,
|
||||
Column3: patch.APIKey != nil,
|
||||
ApiKey: patch.APIKey,
|
||||
ApiKey: sealedKey,
|
||||
}); err != nil {
|
||||
return false, fmt.Errorf("update provider settings: %w", err)
|
||||
}
|
||||
@@ -200,7 +205,7 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
|
||||
for _, r := range rows {
|
||||
key := ""
|
||||
if r.ApiKey != nil {
|
||||
key = *r.ApiKey
|
||||
key = secret.Reveal(secret.TagProviderAPIKey.Purpose(), r.ProviderID, *r.ApiKey)
|
||||
}
|
||||
s.enabledIDs[r.ProviderID] = r.Enabled
|
||||
s.apiKeys[r.ProviderID] = key
|
||||
|
||||
Reference in New Issue
Block a user