Compare commits
35
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7529266f5b | ||
|
|
165c87047e | ||
|
|
05bc85f2b8 | ||
|
|
d1fe3758d5 | ||
|
|
664b129947 | ||
|
|
c9fa18ecb7 | ||
|
|
3077ed75fe | ||
|
|
97e84a79eb | ||
|
|
a29ceaaeab | ||
|
|
54b2720d76 | ||
|
|
fe142e3451 | ||
|
|
3bdfa94f39 | ||
|
|
22ea27efff | ||
|
|
09bd71629a | ||
|
|
c83670d216 | ||
|
|
1e9aed214b | ||
|
|
3638d1d822 | ||
|
|
18618bd135 | ||
|
|
d2985f3841 | ||
|
|
f70df9f827 | ||
|
|
439c8625d5 | ||
|
|
1d67c160b2 | ||
|
|
237380b122 | ||
|
|
4f077736b6 | ||
|
|
727f68950e | ||
|
|
aa9f534f3c | ||
|
|
011b4d9a9c | ||
|
|
d5aa081157 | ||
|
|
a99f855e98 | ||
|
|
7e4727fc49 | ||
|
|
1b7fa635d8 | ||
|
|
57d2299180 | ||
|
|
fa7ea41ccf | ||
|
|
324059b2bd | ||
|
|
1138d75a45 |
+102
-39
@@ -2,9 +2,9 @@ name: release
|
||||
|
||||
# Builds and pushes the minstrel container image to the Gitea registry.
|
||||
#
|
||||
# push to dev → :dev (freshly-built dev APK bundled)
|
||||
# push to main → :latest + :<sha> (freshly-built stable APK bundled)
|
||||
# push tag vYYYY.MM.DD.HHMM → :latest (fresh APK bundled, and attached to the Release)
|
||||
# push to dev → :dev (freshly-built dev APK bundled)
|
||||
# push to main → :latest + :<sha> (latest-release APK bundled)
|
||||
# push tag vYYYY.MM.DD.HHMM → :latest (fresh APK bundled)
|
||||
# workflow_dispatch → manual trigger (same rules based on the ref)
|
||||
#
|
||||
# That is the whole tag map.
|
||||
@@ -41,12 +41,6 @@ name: release
|
||||
# image in the same run from the same commit, and the channel cannot go
|
||||
# stale against its own artifact.
|
||||
#
|
||||
# Every branch build carries the app built from its own commit, so a merge to
|
||||
# main is all it takes to put a new app in front of stable phones: `:latest`
|
||||
# serves it from /api/client/apk and the in-app update offers it. A release
|
||||
# is an optional bookmark — a named point in history with the APK attached
|
||||
# for download — never a step the app has to wait for.
|
||||
#
|
||||
# Release model: the tag IS the artifact's version name with a `v` in front.
|
||||
# `v2026.09.10.1432` and `2026.09.10.1432` are the same string, derived from
|
||||
# the tagged commit's UTC timestamp — so there is no mismatch to reconcile
|
||||
@@ -62,14 +56,20 @@ name: release
|
||||
# :latest is updated by every main push AND every tag push, so it always
|
||||
# reflects the newest blessed image.
|
||||
#
|
||||
# APK pipeline: on every push — dev, main and tags alike — the
|
||||
# android-release job builds + signs the Android APK and uploads it as a
|
||||
# workflow artifact. The image-release job declares `needs: android-release`
|
||||
# and requires its success, so the docker image cannot start building until
|
||||
# the APK is guaranteed-ready — no polling, no race, no silent-failure mode,
|
||||
# and no image that moves a channel without an app inside it. Attaching the
|
||||
# APK to the gitea Release is its own job (release-assets), tags only, behind
|
||||
# the test gate below.
|
||||
# APK pipeline: on tag pushes the android-release job builds + signs the
|
||||
# Android APK and uploads it as a workflow artifact. The image-release
|
||||
# job declares `needs: android-release`, so the docker image cannot
|
||||
# start building until the APK is guaranteed-ready — no polling, no
|
||||
# race, no silent-failure mode. Attaching the APK to the gitea Release is
|
||||
# its own job (release-assets), behind the test gate below.
|
||||
#
|
||||
# :latest always carries an APK. Because every main push also moves
|
||||
# :latest (not just tags), a main build with no APK would silently strip
|
||||
# the in-app update channel off :latest until the next release. So on
|
||||
# non-tag builds image-release pulls the MOST RECENT release's signed APK
|
||||
# AND the version sidecar published beside it — the recorded values, not
|
||||
# recomputed ones — so no rebuild is needed, just a rebundle. Tag builds
|
||||
# keep bundling their own freshly-built APK.
|
||||
#
|
||||
# THE GATE. Every verifying lane lives in this file —
|
||||
# Go (vet, lint, short tests), the Postgres integration suite, the web app
|
||||
@@ -400,14 +400,11 @@ jobs:
|
||||
# ------------------------------------------------------------ artifacts --
|
||||
|
||||
android-release:
|
||||
name: Build signed APK
|
||||
# Builds on every ref, so each channel's image carries the app from its
|
||||
# own commit: `dev` for the test channel, `main` for stable phones, and a
|
||||
# tag for the copy attached to its Release.
|
||||
if: >-
|
||||
startsWith(github.ref, 'refs/tags/v')
|
||||
|| github.ref == 'refs/heads/dev'
|
||||
|| github.ref == 'refs/heads/main'
|
||||
name: Build signed APK (releases and dev)
|
||||
# Also builds on `dev`, which is what makes a test channel possible at
|
||||
# all: without it the only way to get a build onto a phone is to cut a
|
||||
# release, which quietly turns `main` into the staging area.
|
||||
if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev'
|
||||
runs-on: flutter-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-android:36
|
||||
@@ -604,9 +601,9 @@ jobs:
|
||||
path: release-apk/
|
||||
|
||||
- name: Attach APK to gitea Release
|
||||
# Tag releases only. A branch build has no Release to hang assets on
|
||||
# and does not need one — its image bundles the APK, and the server
|
||||
# serves it from /api/client/apk.
|
||||
# Tag releases only. A dev build has no Release to hang assets on and
|
||||
# does not need one — the :dev image bundles the APK, and the server
|
||||
# serves it from /api/client/apk like any other.
|
||||
shell: bash
|
||||
env:
|
||||
CI_TOKEN: ${{ secrets.CI_TOKEN }}
|
||||
@@ -670,9 +667,10 @@ jobs:
|
||||
|
||||
image-release:
|
||||
name: Build + push container image
|
||||
# Every lane must have SUCCEEDED, each named here, and so must the APK
|
||||
# build: every ref bundles the APK built from its own commit, so an
|
||||
# image without one would move a channel with no app for phones to take.
|
||||
# Every lane must have SUCCEEDED, each named here. Then the
|
||||
# APK: tag and dev pushes build one and it must have succeeded; main
|
||||
# pushes skip android-release and bundle the latest release's APK
|
||||
# instead, so for main alone a skipped android-release is expected.
|
||||
needs: [go, integration, web, android, govulncheck, android-release]
|
||||
if: >-
|
||||
${{
|
||||
@@ -682,7 +680,8 @@ jobs:
|
||||
&& needs.web.result == 'success'
|
||||
&& needs.android.result == 'success'
|
||||
&& needs.govulncheck.result == 'success'
|
||||
&& needs.android-release.result == 'success' }}
|
||||
&& (needs.android-release.result == 'success'
|
||||
|| (needs.android-release.result == 'skipped' && github.ref == 'refs/heads/main')) }}
|
||||
runs-on: go-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-go:1.26
|
||||
@@ -748,10 +747,9 @@ jobs:
|
||||
# publish the moving channel tag and never a commit-addressable
|
||||
# one.
|
||||
#
|
||||
# :latest moves here so the server hands out the same APK build
|
||||
# that is attached to the Release: a phone that installed from the
|
||||
# Release page and one that updated in-app hold one build, with one
|
||||
# ordering key. Cut a release on main's tip, as :latest follows it.
|
||||
# :latest must move here rather than waiting for the next main
|
||||
# push, or the channel would carry the PREVIOUS release's APK
|
||||
# indefinitely — a channel that cannot refresh itself.
|
||||
CHANNEL=stable
|
||||
echo "args=-t ${IMAGE}:latest" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::Release build ${VERSION}: refreshing :latest around the new APK"
|
||||
@@ -784,8 +782,12 @@ jobs:
|
||||
| docker login git.fabledsword.com -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
- name: Download signed APK artifact
|
||||
# android-release just produced this, on every ref.
|
||||
if: steps.guard.outputs.ready == 'true'
|
||||
# Tag and dev pushes — android-release just produced this. Only `main`
|
||||
# takes the "Bundle latest release APK" path below, because it is the
|
||||
# one ref that moves a channel without building an APK of its own.
|
||||
if: >-
|
||||
steps.guard.outputs.ready == 'true' &&
|
||||
(startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev')
|
||||
# Consuming half of the pair: stock download-artifact, which works here for
|
||||
# the same reason as the upload (gitea/runner 3.x edits the GHES refusal
|
||||
# out of the bundle). v8 runs on node24, which every CI-runner image
|
||||
@@ -796,7 +798,9 @@ jobs:
|
||||
path: client/
|
||||
|
||||
- name: Stage bundled APK + version sidecar
|
||||
if: steps.guard.outputs.ready == 'true'
|
||||
if: >-
|
||||
steps.guard.outputs.ready == 'true' &&
|
||||
(startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev')
|
||||
shell: bash
|
||||
env:
|
||||
# All three pulled from android-release's outputs so the sidecar the
|
||||
@@ -817,6 +821,65 @@ jobs:
|
||||
cat client/minstrel.apk.version
|
||||
ls -lh client/
|
||||
|
||||
- name: Bundle latest release APK (non-tag :latest builds)
|
||||
# Main pushes don't build an APK, but they DO move :latest — so
|
||||
# without this the in-app update channel would vanish from :latest
|
||||
# until the next tag. Pull the most-recent release's signed APK and
|
||||
# the sidecar published beside it, so what the server reports is what
|
||||
# that build actually recorded rather than something re-derived here.
|
||||
# Degrades to an empty client/ (404 update channel) — never a wrong
|
||||
# version — if no release or APK asset can be resolved. That
|
||||
# degradation only actually works because the greps below carry
|
||||
# `|| true`; under the runner's default pipefail a non-matching grep
|
||||
# kills the step instead of falling through to the empty-case branch.
|
||||
if: steps.guard.outputs.ready == 'true' && github.ref == 'refs/heads/main'
|
||||
shell: bash
|
||||
env:
|
||||
CI_TOKEN: ${{ secrets.CI_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
REPO="${GITHUB_REPOSITORY}"
|
||||
REL_JSON="$(curl -fsSL -H "Authorization: token ${CI_TOKEN}" \
|
||||
"https://git.fabledsword.com/api/v1/repos/${REPO}/releases/latest" || true)"
|
||||
if [ -z "${REL_JSON}" ]; then
|
||||
echo "::notice::no published release — image ships without bundled APK"; exit 0
|
||||
fi
|
||||
# `|| true` on every one of these, and it is load-bearing rather
|
||||
# than defensive habit. The runner already invokes this shell as
|
||||
# `bash -e -o pipefail`, so a pipeline whose grep matches NOTHING
|
||||
# exits non-zero even though `head` succeeded — and the step dies at
|
||||
# the assignment, before ever reaching the `if` written to handle the
|
||||
# empty case. Every "degrades gracefully" branch below is unreachable
|
||||
# without this.
|
||||
TAG="$(printf '%s' "${REL_JSON}" | grep -oP '"tag_name":\s*"\K[^"]+' | head -1)" || true
|
||||
APK_URL="$(printf '%s' "${REL_JSON}" | grep -oP '"browser_download_url":\s*"\K[^"]+' | grep -E '\.apk$' | head -1)" || true
|
||||
if [ -z "${TAG}" ] || [ -z "${APK_URL}" ]; then
|
||||
echo "::notice::latest release '${TAG:-?}' has no APK asset — image ships without bundled APK"; exit 0
|
||||
fi
|
||||
curl -fsSL -H "Authorization: token ${CI_TOKEN}" -o client/minstrel.apk "${APK_URL}"
|
||||
|
||||
# Take the version the release RECORDED rather than recomputing it.
|
||||
# Re-deriving it from the tagged commit would keep the formula in two
|
||||
# files that must stay in step, and could only ever recover the name —
|
||||
# the ordering key is build-time minutes and exists nowhere after that
|
||||
# build ends.
|
||||
SIDECAR_URL="$(printf '%s' "${REL_JSON}" | grep -oP '"browser_download_url":\s*"\K[^"]+' | grep -E '\.apk\.version$' | head -1)" || true
|
||||
if [ -n "${SIDECAR_URL}" ]; then
|
||||
curl -fsSL -H "Authorization: token ${CI_TOKEN}" -o client/minstrel.apk.version "${SIDECAR_URL}"
|
||||
cat client/minstrel.apk.version
|
||||
else
|
||||
# Releases published before sidecars were attached. Their name is
|
||||
# still recoverable from the tag, but their ordering key genuinely
|
||||
# is not — so it is reported ABSENT rather than guessed. A wrong
|
||||
# key is an install the platform refuses; an absent one just tells
|
||||
# the client to fall back to comparing names, which is exactly
|
||||
# what those builds already do.
|
||||
echo "::notice::release ${TAG} predates the version sidecar — bundling with name only, no ordering key"
|
||||
printf '{"name":"%s","code":null,"channel":"stable"}\n' "${TAG#v}" > client/minstrel.apk.version
|
||||
fi
|
||||
echo "::notice::bundled release APK from ${TAG}"
|
||||
ls -lh client/
|
||||
|
||||
- name: Build and push
|
||||
if: steps.guard.outputs.ready == 'true'
|
||||
# --pull: the Dockerfile's base images are floating tags (golang:1.26,
|
||||
|
||||
@@ -35,7 +35,6 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/requests"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble/listenbrainz"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/server"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/similarity"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/subsonic"
|
||||
@@ -129,18 +128,6 @@ func run() error {
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
// Stored credentials are sealed under a key derived from the stream
|
||||
// secret. Install it before anything reads one, and seal any value still
|
||||
// stored in the clear.
|
||||
secret.Install(cfg.StreamSecret)
|
||||
sealed, err := secret.SealStored(ctx, pool)
|
||||
if err != nil {
|
||||
return fmt.Errorf("seal stored credentials: %w", err)
|
||||
}
|
||||
if sealed > 0 {
|
||||
logger.Info("sealed stored credentials", "count", sealed)
|
||||
}
|
||||
|
||||
// Fingerprinting settings: one instance, shared by the scanner,
|
||||
// the fingerprint backfill, the duplicate sweep and the admin API, so a save
|
||||
// reaches all of them without a restart. A load failure is logged, not fatal:
|
||||
|
||||
+6
-26
@@ -49,42 +49,22 @@ cross-origin preflight. Requests authenticated with a bearer token or the
|
||||
Subsonic query parameters carry nothing a browser attaches on its own, so
|
||||
they aren't checked.
|
||||
|
||||
## Credentials Minstrel has to read back
|
||||
|
||||
Some credentials can't be hashed, because Minstrel has to send or compare the
|
||||
value itself:
|
||||
- the API keys and passwords for indexers, Prowlarr and download clients;
|
||||
- the AcoustID key, and the cover-art and tag provider keys;
|
||||
- the SMTP password;
|
||||
- each user's ListenBrainz token and Subsonic password.
|
||||
|
||||
These are encrypted in the database (AES-256-GCM). Each column has its own key,
|
||||
derived from the stream secret (`MINSTREL_STREAM_SECRET`, or the
|
||||
`stream_secret` file in the data directory), and each value is bound to its
|
||||
row. The stream secret is never stored in the database, so a copy of the
|
||||
database alone doesn't reveal any of them. None of them is ever sent back to a
|
||||
browser: Settings only shows whether one is set.
|
||||
|
||||
**Keep the stream secret with the data directory, and don't change it
|
||||
casually.** If it changes, every stored credential reads as not set, and each
|
||||
one has to be entered again in Settings (Subsonic passwords generated again).
|
||||
Minstrel logs a warning for each one that no longer opens.
|
||||
|
||||
## Subsonic sign-in
|
||||
## Subsonic sign-in, and the one password stored in plain text
|
||||
|
||||
Classic Subsonic clients sign in with `t` and `s`: the MD5 of the password
|
||||
followed by a random salt. To check that, the server has to know the password
|
||||
itself, so supporting this sign-in method means storing a password Minstrel
|
||||
can read: the Subsonic password, encrypted as described above.
|
||||
can read. That is the Subsonic password, and it is the only credential
|
||||
Minstrel keeps unhashed.
|
||||
|
||||
- **The recommended way in is the API key.** Clients that support the
|
||||
OpenSubsonic `apiKey` should use it. The key is stored hashed and can be
|
||||
replaced at any time in Settings.
|
||||
- **The Subsonic password is never your login password.** Minstrel generates
|
||||
it (**Settings → Subsonic password**), shows it once, and lets you replace
|
||||
or turn it off. Because it is random, even someone holding both the database
|
||||
and the stream secret gains access to this server's Subsonic API and nothing
|
||||
else: it can't be a password you also use somewhere else.
|
||||
or turn it off. Because it is random, a copy of the database exposes access
|
||||
to this server's Subsonic API and nothing else: it can't be a password you
|
||||
also use somewhere else.
|
||||
- **`t`/`s` and `p=` sign-in are off for an account until it has a Subsonic
|
||||
password.** Plain `p=` sign-in is additionally off server-wide unless
|
||||
`subsonic.allow_plaintext_password` is enabled.
|
||||
|
||||
@@ -24,7 +24,7 @@ func Build(ctx context.Context, pool *pgxpool.Pool) (*Service, dbq.AcquisitionSe
|
||||
if err != nil {
|
||||
return nil, settings, fmt.Errorf("acquire: read search sources: %w", err)
|
||||
}
|
||||
clients, err := usableServices(ctx, q, JobDownload)
|
||||
clients, err := q.ListUsableAcquisitionServices(ctx, JobDownload)
|
||||
if err != nil {
|
||||
return nil, settings, fmt.Errorf("acquire: read download clients: %w", err)
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/exchange"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/prowlarr"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// Kinds: which list a row appears in.
|
||||
@@ -160,7 +159,7 @@ func DownloadClient(row dbq.AcquisitionService) (downloader.Driver, error) {
|
||||
// DownloadClientFor builds the download client a grab went to, by the code
|
||||
// of its acquisition_services row.
|
||||
func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (downloader.Driver, error) {
|
||||
row, err := getService(ctx, dbq.New(pool), code)
|
||||
row, err := dbq.New(pool).GetAcquisitionService(ctx, code)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, fmt.Errorf("the download client %q is no longer set up", code)
|
||||
@@ -172,7 +171,7 @@ func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (do
|
||||
|
||||
// Sources builds every enabled search source.
|
||||
func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) {
|
||||
rows, err := usableServices(ctx, dbq.New(pool), JobSearch)
|
||||
rows, err := dbq.New(pool).ListUsableAcquisitionServices(ctx, JobSearch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -183,33 +182,4 @@ func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// getService and usableServices are the only reads of a full
|
||||
// acquisition_services row. They hand back the credentials opened, so the
|
||||
// clients built from a row are given the key and password themselves.
|
||||
func getService(ctx context.Context, q *dbq.Queries, code string) (dbq.AcquisitionService, error) {
|
||||
row, err := q.GetAcquisitionService(ctx, code)
|
||||
if err != nil {
|
||||
return row, err
|
||||
}
|
||||
return revealed(row), nil
|
||||
}
|
||||
|
||||
func usableServices(ctx context.Context, q *dbq.Queries, job string) ([]dbq.AcquisitionService, error) {
|
||||
rows, err := q.ListUsableAcquisitionServices(ctx, job)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range rows {
|
||||
rows[i] = revealed(rows[i])
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// revealed is row with its sealed credentials opened.
|
||||
func revealed(row dbq.AcquisitionService) dbq.AcquisitionService {
|
||||
row.ApiKey = secret.Reveal(secret.AcquisitionAPIKey.Purpose(), row.Code, row.ApiKey)
|
||||
row.Password = secret.Reveal(secret.AcquisitionPassword.Purpose(), row.Code, row.Password)
|
||||
return row
|
||||
}
|
||||
|
||||
func msDuration(ms int32) time.Duration { return time.Duration(ms) * time.Millisecond }
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// ErrNotFound is a code with no row.
|
||||
@@ -50,7 +49,7 @@ func (s *Store) List(ctx context.Context) ([]dbq.ListAcquisitionServicesRow, err
|
||||
|
||||
// Get returns one row, secrets included: for building a client only.
|
||||
func (s *Store) Get(ctx context.Context, code string) (dbq.AcquisitionService, error) {
|
||||
row, err := getService(ctx, dbq.New(s.pool), code)
|
||||
row, err := dbq.New(s.pool).GetAcquisitionService(ctx, code)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return row, ErrNotFound
|
||||
}
|
||||
@@ -164,27 +163,19 @@ func (s *Store) Delete(ctx context.Context, code string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetAPIKey stores a key, trimmed and sealed; empty clears it.
|
||||
// SetAPIKey stores a key, trimmed; empty clears it.
|
||||
func (s *Store) SetAPIKey(ctx context.Context, code, key string) error {
|
||||
sealed, err := secret.Seal(secret.AcquisitionAPIKey.Purpose(), code, strings.TrimSpace(key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := dbq.New(s.pool).SetAcquisitionServiceAPIKey(ctx, dbq.SetAcquisitionServiceAPIKeyParams{
|
||||
Code: code, ApiKey: sealed,
|
||||
Code: code, ApiKey: strings.TrimSpace(key),
|
||||
})
|
||||
return affected(n, err)
|
||||
}
|
||||
|
||||
// SetPassword stores a password as typed (spaces are part of a password),
|
||||
// sealed; empty clears it.
|
||||
// SetPassword stores a password as typed (spaces are part of a password);
|
||||
// empty clears it.
|
||||
func (s *Store) SetPassword(ctx context.Context, code, password string) error {
|
||||
sealed, err := secret.Seal(secret.AcquisitionPassword.Purpose(), code, password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := dbq.New(s.pool).SetAcquisitionServicePassword(ctx, dbq.SetAcquisitionServicePasswordParams{
|
||||
Code: code, Password: sealed,
|
||||
Code: code, Password: password,
|
||||
})
|
||||
return affected(n, err)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
func newPool(t *testing.T) *pgxpool.Pool {
|
||||
@@ -84,15 +83,6 @@ func TestStore_AddsConfiguresAndRemoves_Integration(t *testing.T) {
|
||||
if err != nil || full.ApiKey != "key" {
|
||||
t.Errorf("stored key = %q (err %v), want it trimmed", full.ApiKey, err)
|
||||
}
|
||||
// The column holds the key sealed, never the key.
|
||||
var raw string
|
||||
if err := s.pool.QueryRow(ctx,
|
||||
"SELECT api_key FROM acquisition_services WHERE code = $1", first.Code).Scan(&raw); err != nil {
|
||||
t.Fatalf("read api_key: %v", err)
|
||||
}
|
||||
if raw == "key" || !secret.IsSealed(raw) {
|
||||
t.Errorf("api_key column = %q, want it sealed", raw)
|
||||
}
|
||||
rows, err := s.List(ctx)
|
||||
if err != nil || len(rows) != 2 {
|
||||
t.Fatalf("list: %d rows, err %v", len(rows), err)
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
type smtpConfigResp struct {
|
||||
@@ -77,15 +76,11 @@ func (h *handlers) handleUpdateSMTPConfig(w http.ResponseWriter, r *http.Request
|
||||
writeErrWithLog(w, h.logger, "admin smtp: get failed", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
// Preserve the stored (sealed) password when the request sends "", so
|
||||
// the admin doesn't have to re-enter it on every save.
|
||||
password := current.Password
|
||||
if req.Password != "" {
|
||||
password, err = secret.Seal(secret.SMTPPassword.Purpose(), "", req.Password)
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "admin smtp: seal password", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
// Preserve existing password when the request sends "" (so the
|
||||
// admin doesn't have to re-enter it on every save).
|
||||
password := req.Password
|
||||
if password == "" {
|
||||
password = current.Password
|
||||
}
|
||||
if err := q.UpdateSMTPConfig(r.Context(), dbq.UpdateSMTPConfigParams{
|
||||
Enabled: req.Enabled,
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// LBStatus is the GET response body and the PUT response body. Token is
|
||||
@@ -54,11 +53,10 @@ func (h *handlers) handlePutListenBrainz(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
q := dbq.New(h.pool)
|
||||
if body.Token != nil {
|
||||
tokenPtr, err := secret.SealPtr(secret.ListenBrainzToken.Purpose(), user.ID.String(), body.Token)
|
||||
if err != nil {
|
||||
h.logger.Error("api: seal listenbrainz token", "err", err)
|
||||
writeErr(w, apierror.InternalMsg("update failed", err))
|
||||
return
|
||||
t := *body.Token
|
||||
var tokenPtr *string
|
||||
if t != "" {
|
||||
tokenPtr = &t
|
||||
}
|
||||
if err := q.SetListenBrainzToken(r.Context(), dbq.SetListenBrainzTokenParams{
|
||||
ID: user.ID, ListenbrainzToken: tokenPtr,
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// The Subsonic password is for clients that sign in with Subsonic's t/s
|
||||
@@ -52,15 +51,9 @@ func (h *handlers) handleGenerateMySubsonicPassword(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
pw := base64.RawURLEncoding.EncodeToString(b)
|
||||
sealed, err := secret.Seal(secret.SubsonicPassword.Purpose(), user.ID.String(), pw)
|
||||
if err != nil {
|
||||
h.logger.Error("generate subsonic password: seal failed", "err", err)
|
||||
writeErr(w, apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
if err := dbq.New(h.pool).SetSubsonicPassword(r.Context(), dbq.SetSubsonicPasswordParams{
|
||||
ID: user.ID,
|
||||
SubsonicPassword: &sealed,
|
||||
SubsonicPassword: &pw,
|
||||
}); err != nil {
|
||||
h.logger.Error("generate subsonic password: update failed", "err", err)
|
||||
writeErr(w, apierror.Internal(err))
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
func newMeSubsonicPasswordRouter(h *handlers) chi.Router {
|
||||
@@ -22,8 +21,6 @@ func newMeSubsonicPasswordRouter(h *handlers) chi.Router {
|
||||
return r
|
||||
}
|
||||
|
||||
// readSubsonicPassword returns the stored Subsonic password opened. It fails
|
||||
// the test if the column holds the password in the clear.
|
||||
func readSubsonicPassword(t *testing.T, h *handlers, user dbq.User) *string {
|
||||
t.Helper()
|
||||
var pw *string
|
||||
@@ -31,17 +28,7 @@ func readSubsonicPassword(t *testing.T, h *handlers, user dbq.User) *string {
|
||||
"SELECT subsonic_password FROM users WHERE id = $1", user.ID).Scan(&pw); err != nil {
|
||||
t.Fatalf("read subsonic_password: %v", err)
|
||||
}
|
||||
if pw == nil {
|
||||
return nil
|
||||
}
|
||||
if !secret.IsSealed(*pw) {
|
||||
t.Fatalf("subsonic_password is stored in the clear")
|
||||
}
|
||||
plain, err := secret.Open(secret.SubsonicPassword.Purpose(), user.ID.String(), *pw)
|
||||
if err != nil {
|
||||
t.Fatalf("open subsonic_password: %v", err)
|
||||
}
|
||||
return &plain
|
||||
return pw
|
||||
}
|
||||
|
||||
func TestSubsonicPassword_GenerateIsRandomAndNotTheLoginPassword(t *testing.T) {
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The backup file is laid out so that what didn't change between two days is
|
||||
// the same bytes both times, which suits a deduplicating off-site copy such as
|
||||
// Fabled Archives' restic:
|
||||
//
|
||||
// - Never compressed. Compression rewrites every byte after the first
|
||||
// change, so no chunk would match again.
|
||||
// - One member per table, tables/<table>.jsonl: one row per line, in
|
||||
// primary-key order, keys sorted, no spacing.
|
||||
// - What changes every run lives only in manifest.json, the first member.
|
||||
// Member headers are fixed (mtime 0, owner 0, mode 0644).
|
||||
// - A new dated name each run, so the target keeps a history and a sender
|
||||
// only ever adds.
|
||||
const (
|
||||
FormatName = "minstrel-backup-tar"
|
||||
FormatVersion = 1
|
||||
|
||||
manifestName = "manifest.json"
|
||||
tablesDir = "tables/"
|
||||
tableSuffix = ".jsonl"
|
||||
namePrefix = "minstrel-"
|
||||
nameSuffix = ".tar"
|
||||
|
||||
// maxMember bounds any one member a read takes in: far past the largest
|
||||
// library's tracks table, small enough that a hostile file can't fill
|
||||
// memory.
|
||||
maxMember = 1 << 30
|
||||
)
|
||||
|
||||
// Manifest is the file's first member.
|
||||
type Manifest struct {
|
||||
Format string `json:"format"`
|
||||
FormatVersion int `json:"format_version"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
// Build is the Minstrel version that wrote the file.
|
||||
Build string `json:"build"`
|
||||
// Schema is the database's migration version when the file was written.
|
||||
// A restore refuses a file from a newer schema than its own.
|
||||
Schema int64 `json:"schema"`
|
||||
// Rows is each table's row count, so a read can tell a whole member from
|
||||
// a cut-off one.
|
||||
Rows map[string]int `json:"rows"`
|
||||
}
|
||||
|
||||
// FileName is a backup's file name: minstrel-20261102T030000Z.tar, in UTC, so
|
||||
// names sort by time.
|
||||
func FileName(at time.Time) string {
|
||||
return namePrefix + at.UTC().Format("20060102T150405Z") + nameSuffix
|
||||
}
|
||||
|
||||
// IsFileName reports whether a name is one FileName gives.
|
||||
func IsFileName(name string) bool {
|
||||
stamp, ok := strings.CutPrefix(name, namePrefix)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
stamp, ok = strings.CutSuffix(stamp, nameSuffix)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, err := time.Parse("20060102T150405Z", stamp)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// Write writes c as a backup file to w and returns its manifest.
|
||||
func Write(w io.Writer, c Contents, at time.Time, build string) (Manifest, error) {
|
||||
names := make([]string, 0, len(c.Tables))
|
||||
m := Manifest{
|
||||
Format: FormatName,
|
||||
FormatVersion: FormatVersion,
|
||||
CreatedAt: at.UTC(),
|
||||
Build: build,
|
||||
Schema: c.Schema,
|
||||
Rows: make(map[string]int, len(c.Tables)),
|
||||
}
|
||||
for name, lines := range c.Tables {
|
||||
names = append(names, name)
|
||||
m.Rows[name] = len(lines)
|
||||
}
|
||||
slices.Sort(names)
|
||||
|
||||
manifest, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
tw := tar.NewWriter(w)
|
||||
if err := member(tw, manifestName, append(manifest, '\n')); err != nil {
|
||||
return m, err
|
||||
}
|
||||
for _, name := range names {
|
||||
var body bytes.Buffer
|
||||
for _, line := range c.Tables[name] {
|
||||
body.Write(line)
|
||||
body.WriteByte('\n')
|
||||
}
|
||||
if err := member(tw, tablesDir+name+tableSuffix, body.Bytes()); err != nil {
|
||||
return m, err
|
||||
}
|
||||
}
|
||||
return m, tw.Close()
|
||||
}
|
||||
|
||||
// WriteFile writes c into dir under its dated name. The bytes go to a hidden
|
||||
// .part file first, are synced, and only then take the backup's name, so
|
||||
// anything watching the folder never sees half a backup under a backup's
|
||||
// name, and a run that fails leaves nothing behind.
|
||||
func WriteFile(dir string, c Contents, at time.Time, build string) (string, Manifest, error) {
|
||||
final := filepath.Join(dir, FileName(at))
|
||||
part := filepath.Join(dir, "."+FileName(at)+".part")
|
||||
f, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
|
||||
if err != nil {
|
||||
return "", Manifest{}, fmt.Errorf("backup: create %s: %w", part, err)
|
||||
}
|
||||
m, err := Write(f, c, at, build)
|
||||
if err == nil {
|
||||
err = f.Sync()
|
||||
}
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err == nil {
|
||||
err = os.Rename(part, final)
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(part)
|
||||
return "", m, fmt.Errorf("backup: write %s: %w", final, err)
|
||||
}
|
||||
if d, err := os.Open(dir); err == nil {
|
||||
_ = d.Sync()
|
||||
_ = d.Close()
|
||||
}
|
||||
return final, m, nil
|
||||
}
|
||||
|
||||
func member(tw *tar.Writer, name string, data []byte) error {
|
||||
hdr := &tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(data)),
|
||||
ModTime: time.Unix(0, 0),
|
||||
Typeflag: tar.TypeReg,
|
||||
Format: tar.FormatUSTAR,
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tw.Write(data)
|
||||
return err
|
||||
}
|
||||
|
||||
// Read reads a backup file, refusing anything that isn't a whole one.
|
||||
func Read(r io.Reader) (Manifest, Contents, error) {
|
||||
var m Manifest
|
||||
c := Contents{Tables: map[string][][]byte{}}
|
||||
tr := tar.NewReader(r)
|
||||
first := true
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err)
|
||||
}
|
||||
if hdr.Size > maxMember {
|
||||
return m, c, fmt.Errorf("%w: %s is too large", ErrNotABackup, hdr.Name)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(tr, maxMember))
|
||||
if err != nil {
|
||||
return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err)
|
||||
}
|
||||
if first {
|
||||
first = false
|
||||
if hdr.Name != manifestName || json.Unmarshal(data, &m) != nil || m.Format != FormatName {
|
||||
return m, c, ErrNotABackup
|
||||
}
|
||||
if m.FormatVersion > FormatVersion {
|
||||
return m, c, fmt.Errorf("%w: it was written by a newer Minstrel (format %d)", ErrNotABackup, m.FormatVersion)
|
||||
}
|
||||
c.Schema = m.Schema
|
||||
continue
|
||||
}
|
||||
table, ok := strings.CutPrefix(hdr.Name, tablesDir)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if table, ok = strings.CutSuffix(table, tableSuffix); !ok || table == "" {
|
||||
continue
|
||||
}
|
||||
var lines [][]byte
|
||||
for line := range bytes.SplitSeq(data, []byte("\n")) {
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
if !json.Valid(line) {
|
||||
return m, c, fmt.Errorf("%w: a row in %s is not JSON", ErrNotABackup, table)
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
c.Tables[table] = lines
|
||||
}
|
||||
if first {
|
||||
return m, c, ErrNotABackup
|
||||
}
|
||||
for table, want := range m.Rows {
|
||||
if got := len(c.Tables[table]); got != want {
|
||||
return m, c, fmt.Errorf("%w: %s holds %d rows, the manifest says %d", ErrNotABackup, table, got, want)
|
||||
}
|
||||
}
|
||||
return m, c, nil
|
||||
}
|
||||
@@ -1,190 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func sample() Contents {
|
||||
return Contents{
|
||||
Schema: 98,
|
||||
Tables: map[string][][]byte{
|
||||
"users": {[]byte(`{"id":"u1","username":"ada"}`), []byte(`{"id":"u2","username":"bo"}`)},
|
||||
"tracks": {[]byte(`{"file_path":"/music/a.flac","id":"t1"}`)},
|
||||
"wants": {},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func members(t *testing.T, file []byte) map[string][]byte {
|
||||
t.Helper()
|
||||
out := map[string][]byte{}
|
||||
tr := tar.NewReader(bytes.NewReader(file))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return out
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, _ := io.ReadAll(tr)
|
||||
out[hdr.Name] = data
|
||||
}
|
||||
}
|
||||
|
||||
func TestABackupFileReadsBackAsWritten(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC)
|
||||
if _, err := Write(&buf, sample(), at, "2026.11.02.0300"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, c, err := Read(&buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Schema != 98 || c.Schema != 98 || m.Build != "2026.11.02.0300" || !m.CreatedAt.Equal(at) {
|
||||
t.Errorf("manifest = %+v", m)
|
||||
}
|
||||
want := sample()
|
||||
for name, lines := range want.Tables {
|
||||
got := c.Tables[name]
|
||||
if len(got) != len(lines) {
|
||||
t.Errorf("%s: %d rows, want %d", name, len(got), len(lines))
|
||||
continue
|
||||
}
|
||||
for i := range lines {
|
||||
if !bytes.Equal(got[i], lines[i]) {
|
||||
t.Errorf("%s row %d = %s, want %s", name, i, got[i], lines[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Two days over unchanged data differ only in the manifest, so a
|
||||
// deduplicating copy stores the tables once.
|
||||
func TestOnlyTheManifestChangesBetweenDays(t *testing.T) {
|
||||
var day1, day2 bytes.Buffer
|
||||
if _, err := Write(&day1, sample(), time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC), "a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Write(&day2, sample(), time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC), "b"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, b := members(t, day1.Bytes()), members(t, day2.Bytes())
|
||||
if len(a) != 4 {
|
||||
t.Fatalf("members = %d, want the manifest and three tables", len(a))
|
||||
}
|
||||
for name, data := range a {
|
||||
same := bytes.Equal(data, b[name])
|
||||
if name == manifestName && same {
|
||||
t.Error("the manifest is identical across days; it should carry the run's time")
|
||||
}
|
||||
if name != manifestName && !same {
|
||||
t.Errorf("%s differs between days over the same rows", name)
|
||||
}
|
||||
}
|
||||
// The table members' bytes, headers included, sit unchanged in both files.
|
||||
tracks := a[tablesDir+"tracks"+tableSuffix]
|
||||
if !bytes.Contains(day1.Bytes(), tracks) || !bytes.Contains(day2.Bytes(), tracks) {
|
||||
t.Error("the tracks member is not stored verbatim")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadRefusesWhatIsNotAWholeBackup(t *testing.T) {
|
||||
var good bytes.Buffer
|
||||
if _, err := Write(&good, sample(), time.Now(), "x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tarOf := func(name string, data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = member(tw, name, data)
|
||||
_ = tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
short := func() []byte {
|
||||
// The manifest promises two users; the member holds one.
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = member(tw, manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":1,"rows":{"users":2}}`))
|
||||
_ = member(tw, tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`+"\n"))
|
||||
_ = tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
cases := map[string][]byte{
|
||||
"empty": nil,
|
||||
"not a tar": []byte("hello"),
|
||||
"cut off": good.Bytes()[:good.Len()/2],
|
||||
"no manifest": tarOf(tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`)),
|
||||
"another format": tarOf(manifestName, []byte(`{"format":"fablednotary-backup-tar","format_version":1}`)),
|
||||
"a newer format": tarOf(manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":99}`)),
|
||||
"a member short": short(),
|
||||
}
|
||||
for name, file := range cases {
|
||||
if _, _, err := Read(bytes.NewReader(file)); !errors.Is(err, ErrNotABackup) {
|
||||
t.Errorf("%s: err = %v, want ErrNotABackup", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteFileLandsUnderTheDatedNameOnly(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC)
|
||||
path, _, err := WriteFile(dir, sample(), at, "x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if filepath.Base(path) != "minstrel-20261102T030000Z.tar" {
|
||||
t.Errorf("path = %s", path)
|
||||
}
|
||||
entries, _ := os.ReadDir(dir)
|
||||
if len(entries) != 1 || entries[0].Name() != filepath.Base(path) {
|
||||
t.Errorf("the folder holds %v, want only the backup", entries)
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, _, err := Read(f); err != nil {
|
||||
t.Errorf("the written file doesn't read: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedWriteFileLeavesNothingBehind(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "missing")
|
||||
if _, _, err := WriteFile(dir, sample(), time.Now(), "x"); err == nil {
|
||||
t.Fatal("writing into a missing folder succeeded")
|
||||
}
|
||||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||
t.Errorf("the failed write created %s", dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileNamesSortByTimeAndAreRecognised(t *testing.T) {
|
||||
earlier := FileName(time.Date(2026, 1, 9, 23, 0, 0, 0, time.UTC))
|
||||
later := FileName(time.Date(2026, 1, 10, 1, 0, 0, 0, time.FixedZone("x", 5*3600)))
|
||||
if earlier >= later {
|
||||
t.Errorf("%s does not sort before %s", earlier, later)
|
||||
}
|
||||
for name, want := range map[string]bool{
|
||||
earlier: true,
|
||||
"minstrel-20261102T030000Z.tar": true,
|
||||
".minstrel-20261102T030000Z.tar.part": false,
|
||||
"minstrel-yesterday.tar": false,
|
||||
"fablednotary-20261102T030000Z.tar": false,
|
||||
} {
|
||||
if IsFileName(name) != want {
|
||||
t.Errorf("IsFileName(%q) = %v, want %v", name, !want, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,238 +0,0 @@
|
||||
package backup_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||
)
|
||||
|
||||
func newPool(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in -short mode")
|
||||
}
|
||||
dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
|
||||
}
|
||||
if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("pool: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
dbtest.ResetDB(t, pool)
|
||||
return pool
|
||||
}
|
||||
|
||||
func index(name string) int {
|
||||
return slices.IndexFunc(backup.Tables, func(t backup.Table) bool { return t.Name == name })
|
||||
}
|
||||
|
||||
// Every table in the schema is backed up or excluded with a reason, so a new
|
||||
// table fails here until someone decides which.
|
||||
func TestEveryTableIsAccountedFor_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
rows, err := pool.Query(context.Background(), "SELECT tablename FROM pg_tables WHERE schemaname = 'public'")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
schema, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range backup.Coverage(schema) {
|
||||
t.Error(p)
|
||||
}
|
||||
}
|
||||
|
||||
// A restore loads each table after the ones it references, and never needs a
|
||||
// table the backup leaves out.
|
||||
func TestTablesComeAfterWhatTheyReference_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
rows, err := pool.Query(context.Background(), `
|
||||
SELECT cl.relname, pc.relname
|
||||
FROM pg_constraint co
|
||||
JOIN pg_class cl ON cl.oid = co.conrelid
|
||||
JOIN pg_class pc ON pc.oid = co.confrelid
|
||||
JOIN pg_namespace n ON n.oid = cl.relnamespace
|
||||
WHERE co.contype = 'f' AND n.nspname = 'public'`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
type fk struct{ child, parent string }
|
||||
fks, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (fk, error) {
|
||||
var f fk
|
||||
err := r.Scan(&f.child, &f.parent)
|
||||
return f, err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(fks) == 0 {
|
||||
t.Fatal("found no foreign keys; the query is wrong")
|
||||
}
|
||||
for _, f := range fks {
|
||||
child, parent := index(f.child), index(f.parent)
|
||||
if child < 0 {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case parent < 0:
|
||||
t.Errorf("%s is backed up but references %s, which is excluded", f.child, f.parent)
|
||||
case parent >= child && f.parent != f.child:
|
||||
t.Errorf("%s references %s, so it must come after it in Tables", f.child, f.parent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every omitted column exists, so a renamed column fails here rather than a
|
||||
// sealed credential slipping into the file under its new name.
|
||||
func TestOmittedColumnsExist_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
for _, tbl := range backup.Tables {
|
||||
for _, col := range tbl.Omit {
|
||||
var n int
|
||||
if err := pool.QueryRow(context.Background(), `
|
||||
SELECT count(*) FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2`,
|
||||
tbl.Name, col).Scan(&n); err != nil || n != 1 {
|
||||
t.Errorf("%s.%s: found %d columns, err %v", tbl.Name, col, n, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustExec(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := pool.Exec(context.Background(), sql, args...); err != nil {
|
||||
t.Fatalf("%s: %v", sql, err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustID(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) string {
|
||||
t.Helper()
|
||||
var id string
|
||||
if err := pool.QueryRow(context.Background(), sql, args...).Scan(&id); err != nil {
|
||||
t.Fatalf("%s: %v", sql, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// A backup restored into an emptied instance gives back the same rows, byte
|
||||
// for byte, without the credentials and markers it leaves out.
|
||||
func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
ctx := context.Background()
|
||||
|
||||
user := mustID(t, pool, `INSERT INTO users (username, password_hash, api_token_hash, listenbrainz_token)
|
||||
VALUES ($1, 'hash', 'backup-token', 'lb-token') RETURNING id::text`, dbtest.TestUserPrefix+"backup")
|
||||
artist := mustID(t, pool, `INSERT INTO artists (name, sort_name, artist_art_source) VALUES ('Artist', 'artist', 'none') RETURNING id::text`)
|
||||
album := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id, cover_art_source, cover_art_path)
|
||||
VALUES ('Album', 'album', $1, 'mbcaa', '/data/covers/a.jpg') RETURNING id::text`, artist)
|
||||
track := mustID(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format, tag_source)
|
||||
VALUES ('Track', $1, $2, 180000, '/music/a.flac', 1000, 'flac', 'musicbrainz') RETURNING id::text`, album, artist)
|
||||
session := mustID(t, pool, `INSERT INTO play_sessions (user_id, started_at, last_event_at) VALUES ($1, now(), now()) RETURNING id::text`, user)
|
||||
mustExec(t, pool, `INSERT INTO play_events (user_id, track_id, session_id, started_at) VALUES ($1, $2, $3, now())`, user, track, session)
|
||||
mustExec(t, pool, `INSERT INTO general_likes (user_id, track_id) VALUES ($1, $2)`, user, track)
|
||||
playlist := mustID(t, pool, `INSERT INTO playlists (user_id, name) VALUES ($1, 'Mine') RETURNING id::text`, user)
|
||||
mustExec(t, pool, `INSERT INTO playlist_tracks (playlist_id, position, track_id, title, artist_name, album_title, duration_sec)
|
||||
VALUES ($1, 0, $2, 'Track', 'Artist', 'Album', 180)`, playlist, track)
|
||||
mustExec(t, pool, `INSERT INTO follows (artist_mbid, artist_name, profile_id, followed_by)
|
||||
SELECT 'mbid-1', 'Artist', id, $1 FROM release_type_profiles WHERE is_default`, user)
|
||||
mustExec(t, pool, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert')`, track)
|
||||
|
||||
first, err := backup.Snapshot(ctx, pool)
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot: %v", err)
|
||||
}
|
||||
for _, tbl := range backup.Tables {
|
||||
for _, line := range first.Tables[tbl.Name] {
|
||||
var row map[string]json.RawMessage
|
||||
if err := json.Unmarshal(line, &row); err != nil {
|
||||
t.Fatalf("%s: %v", tbl.Name, err)
|
||||
}
|
||||
for _, col := range tbl.Omit {
|
||||
if _, ok := row[col]; ok {
|
||||
t.Errorf("%s.%s is in the backup", tbl.Name, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, tbl := range []string{"users", "tracks", "play_events", "general_likes", "playlist_tracks", "follows", "library_changes"} {
|
||||
if len(first.Tables[tbl]) == 0 {
|
||||
t.Errorf("the backup holds no %s", tbl)
|
||||
}
|
||||
}
|
||||
|
||||
if err := backup.Restore(ctx, pool, first); !errors.Is(err, backup.ErrNotEmpty) {
|
||||
t.Fatalf("restore over data: err = %v, want ErrNotEmpty", err)
|
||||
}
|
||||
newer := first
|
||||
newer.Schema++
|
||||
if err := backup.Restore(ctx, pool, newer); !errors.Is(err, backup.ErrNewerSchema) {
|
||||
t.Fatalf("restore from a newer schema: err = %v, want ErrNewerSchema", err)
|
||||
}
|
||||
|
||||
mustExec(t, pool, "TRUNCATE users, artists, library_changes CASCADE")
|
||||
if err := backup.Restore(ctx, pool, first); err != nil {
|
||||
t.Fatalf("restore: %v", err)
|
||||
}
|
||||
|
||||
second, err := backup.Snapshot(ctx, pool)
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot after restore: %v", err)
|
||||
}
|
||||
for _, tbl := range backup.Tables {
|
||||
a, b := first.Tables[tbl.Name], second.Tables[tbl.Name]
|
||||
if len(a) != len(b) {
|
||||
t.Errorf("%s: %d rows restored, the backup held %d", tbl.Name, len(b), len(a))
|
||||
continue
|
||||
}
|
||||
for i := range a {
|
||||
if !bytes.Equal(a[i], b[i]) {
|
||||
t.Errorf("%s row %d:\nbacked up %s\nrestored %s", tbl.Name, i+1, a[i], b[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the backup leaves out comes back unset, so the server asks for the
|
||||
// credential again and redoes the art and tags.
|
||||
var lbToken, coverSource, tagSource *string
|
||||
if err := pool.QueryRow(ctx, `SELECT u.listenbrainz_token, al.cover_art_source, tr.tag_source
|
||||
FROM users u, albums al, tracks tr WHERE u.id = $1 AND al.id = $2 AND tr.id = $3`,
|
||||
user, album, track).Scan(&lbToken, &coverSource, &tagSource); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if lbToken != nil || coverSource != nil || tagSource != nil {
|
||||
t.Errorf("token %v, cover source %v, tag source %v; want all unset", lbToken, coverSource, tagSource)
|
||||
}
|
||||
|
||||
// The change feed carries on after the restored ids.
|
||||
var restoredMax, next int64
|
||||
if err := pool.QueryRow(ctx, "SELECT max(id) FROM library_changes").Scan(&restoredMax); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op)
|
||||
VALUES ('track', $1, 'upsert') RETURNING id`, track).Scan(&next); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if next <= restoredMax {
|
||||
t.Errorf("next change id %d, want it after the restored %d", next, restoredMax)
|
||||
}
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNotEmpty is a restore into an instance that already has accounts or
|
||||
// a library.
|
||||
ErrNotEmpty = errors.New("backup: this Minstrel already has accounts or a library; a backup restores into an empty one")
|
||||
|
||||
// ErrNewerSchema is a backup written by a newer Minstrel than this one.
|
||||
ErrNewerSchema = errors.New("backup: the backup was written by a newer version of Minstrel; upgrade this one first")
|
||||
|
||||
// ErrNotABackup is contents that aren't rows a backup writes.
|
||||
ErrNotABackup = errors.New("backup: not a Minstrel backup")
|
||||
)
|
||||
|
||||
// batchRows is how many rows one INSERT carries.
|
||||
const batchRows = 500
|
||||
|
||||
// Restore loads a backup into an empty instance, in one transaction: either
|
||||
// all of it lands or none of it does.
|
||||
//
|
||||
// The instance must have no accounts and no tracks. Its other rows in the
|
||||
// tables the backup holds are replaced, which is what a fresh instance's
|
||||
// seeded settings need. A table the backup doesn't hold, because it was
|
||||
// written before that table existed, keeps what the instance has.
|
||||
//
|
||||
// A row's columns are those the backup and the table both have: an older
|
||||
// backup's rows take the defaults of columns added since, and a column dropped
|
||||
// since is left out.
|
||||
func Restore(ctx context.Context, pool *pgxpool.Pool, c Contents) error {
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: begin: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
schema, err := schemaVersion(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c.Schema > schema {
|
||||
return ErrNewerSchema
|
||||
}
|
||||
// Taken before the emptiness check, so a first account or a scan starting
|
||||
// at the same moment waits for the restore, then finds the data.
|
||||
if _, err := tx.Exec(ctx, "LOCK TABLE users, tracks IN EXCLUSIVE MODE"); err != nil {
|
||||
return fmt.Errorf("backup: lock: %w", err)
|
||||
}
|
||||
var occupied bool
|
||||
if err := tx.QueryRow(ctx,
|
||||
"SELECT EXISTS (SELECT 1 FROM users) OR EXISTS (SELECT 1 FROM tracks)").Scan(&occupied); err != nil {
|
||||
return fmt.Errorf("backup: check the instance is empty: %w", err)
|
||||
}
|
||||
if occupied {
|
||||
return ErrNotEmpty
|
||||
}
|
||||
|
||||
// Children first, so no row is left pointing at one already gone.
|
||||
for _, t := range slices.Backward(Tables) {
|
||||
if _, ok := c.Tables[t.Name]; !ok {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(ctx, "DELETE FROM "+pgx.Identifier{t.Name}.Sanitize()); err != nil {
|
||||
return fmt.Errorf("backup: clear %s: %w", t.Name, err)
|
||||
}
|
||||
}
|
||||
for _, t := range Tables {
|
||||
rows, ok := c.Tables[t.Name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if err := restoreTable(ctx, tx, t.Name, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("backup: commit: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type column struct {
|
||||
name string
|
||||
identity bool
|
||||
sequence bool
|
||||
}
|
||||
|
||||
func restoreTable(ctx context.Context, tx pgx.Tx, table string, rows [][]byte) error {
|
||||
cols, err := writableColumns(ctx, tx, table)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ident := pgx.Identifier{table}.Sanitize()
|
||||
overriding := ""
|
||||
if slices.ContainsFunc(cols, func(c column) bool { return c.identity }) {
|
||||
overriding = " OVERRIDING SYSTEM VALUE"
|
||||
}
|
||||
|
||||
// Rows go in batches that share one key set; a backup's rows for a table
|
||||
// all do, since one read wrote them.
|
||||
var (
|
||||
batch []json.RawMessage
|
||||
list string
|
||||
)
|
||||
flush := func() error {
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
payload, err := json.Marshal(batch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sql := fmt.Sprintf("INSERT INTO %s (%s)%s SELECT %s FROM jsonb_populate_recordset(NULL::%s, $1::jsonb)",
|
||||
ident, list, overriding, list, ident)
|
||||
if _, err := tx.Exec(ctx, sql, string(payload)); err != nil {
|
||||
return fmt.Errorf("backup: restore %s: %w", table, err)
|
||||
}
|
||||
batch = batch[:0]
|
||||
return nil
|
||||
}
|
||||
for i, raw := range rows {
|
||||
var keys map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &keys); err != nil {
|
||||
return fmt.Errorf("%w: row %d of %s is not an object", ErrNotABackup, i+1, table)
|
||||
}
|
||||
var names []string
|
||||
for _, c := range cols {
|
||||
if _, ok := keys[c.name]; ok {
|
||||
names = append(names, pgx.Identifier{c.name}.Sanitize())
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
continue
|
||||
}
|
||||
slices.Sort(names)
|
||||
rowList := strings.Join(names, ", ")
|
||||
if rowList != list || len(batch) == batchRows {
|
||||
if err := flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
list = rowList
|
||||
}
|
||||
batch = append(batch, raw)
|
||||
}
|
||||
if err := flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// A sequence carries on after the highest restored value, so the next row
|
||||
// doesn't collide with one from the backup.
|
||||
for _, c := range cols {
|
||||
if !c.sequence {
|
||||
continue
|
||||
}
|
||||
col := pgx.Identifier{c.name}.Sanitize()
|
||||
if _, err := tx.Exec(ctx, fmt.Sprintf(
|
||||
"SELECT setval(pg_get_serial_sequence($1, $2), COALESCE((SELECT max(%s) FROM %s), 0) + 1, false)", col, ident),
|
||||
table, c.name); err != nil {
|
||||
return fmt.Errorf("backup: carry on %s's sequence: %w", table, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writableColumns returns the columns of a table an INSERT can set: all but
|
||||
// generated ones, which Postgres computes.
|
||||
func writableColumns(ctx context.Context, tx pgx.Tx, table string) ([]column, error) {
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT column_name, is_identity = 'YES', pg_get_serial_sequence($1::text, column_name) IS NOT NULL
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = $1::text AND is_generated = 'NEVER'
|
||||
ORDER BY ordinal_position`, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read %s's columns: %w", table, err)
|
||||
}
|
||||
cols, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (column, error) {
|
||||
var c column
|
||||
err := row.Scan(&c.name, &c.identity, &c.sequence)
|
||||
return c, err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read %s's columns: %w", table, err)
|
||||
}
|
||||
return cols, nil
|
||||
}
|
||||
@@ -1,201 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrS3 is a request the bucket refused or never answered.
|
||||
var ErrS3 = errors.New("backup: the S3 bucket refused the request")
|
||||
|
||||
// Bucket is a path-style S3 bucket: an endpoint such as Fabled Archives'
|
||||
// backup port or MinIO, a bucket name, and a key pair. A backup only ever
|
||||
// adds objects and lists them, so an append-only key is enough.
|
||||
type Bucket struct {
|
||||
Endpoint string
|
||||
Name string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
// Region defaults to us-east-1, which S3-compatible servers accept.
|
||||
Region string
|
||||
Client *http.Client
|
||||
// now is the signing clock; tests pin it.
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// Object is one stored object.
|
||||
type Object struct {
|
||||
Key string
|
||||
Size int64
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
func (b Bucket) region() string {
|
||||
if b.Region == "" {
|
||||
return "us-east-1"
|
||||
}
|
||||
return b.Region
|
||||
}
|
||||
|
||||
func (b Bucket) client() *http.Client {
|
||||
if b.Client != nil {
|
||||
return b.Client
|
||||
}
|
||||
return http.DefaultClient
|
||||
}
|
||||
|
||||
func (b Bucket) clock() time.Time {
|
||||
if b.now != nil {
|
||||
return b.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// objectURL is the path-style URL of key, encoded exactly as it is signed.
|
||||
func (b Bucket) objectURL(key string) (*url.URL, error) {
|
||||
u, err := url.Parse(strings.TrimRight(b.Endpoint, "/"))
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return nil, fmt.Errorf("%w: the endpoint %q is not an http(s) address", ErrS3, b.Endpoint)
|
||||
}
|
||||
if b.Name == "" {
|
||||
return nil, fmt.Errorf("%w: no bucket name", ErrS3)
|
||||
}
|
||||
raw := "/" + uriEncode(b.Name)
|
||||
if key != "" {
|
||||
for s := range strings.SplitSeq(key, "/") {
|
||||
raw += "/" + uriEncode(s)
|
||||
}
|
||||
}
|
||||
u.RawPath = u.EscapedPath() + raw
|
||||
u.Path, _ = url.PathUnescape(u.RawPath)
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (b Bucket) do(req *http.Request, payloadSHA256 string) (*http.Response, error) {
|
||||
sign(req, payloadSHA256, b.AccessKey, b.SecretKey, b.region(), b.clock())
|
||||
resp, err := b.client().Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s is unreachable: %v", ErrS3, req.URL.Host, err)
|
||||
}
|
||||
if resp.StatusCode/100 != 2 {
|
||||
defer resp.Body.Close()
|
||||
return nil, refusal(resp)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Put stores body under key. size is the body's length and payloadSHA256
|
||||
// its hex SHA-256, which the request is signed over.
|
||||
func (b Bucket) Put(ctx context.Context, key string, body io.Reader, size int64, payloadSHA256 string) error {
|
||||
u, err := b.objectURL(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.URL = u
|
||||
req.ContentLength = size
|
||||
if size == 0 {
|
||||
req.Body = http.NoBody
|
||||
}
|
||||
resp, err := b.do(req, payloadSHA256)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
drain(resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns up to limit objects whose keys start with prefix, in key
|
||||
// order.
|
||||
func (b Bucket) List(ctx context.Context, prefix string, limit int) ([]Object, error) {
|
||||
if limit < 1 {
|
||||
limit = 1000
|
||||
}
|
||||
var out []Object
|
||||
token := ""
|
||||
for {
|
||||
u, err := b.objectURL("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := url.Values{"list-type": {"2"}, "prefix": {prefix}}
|
||||
if page := limit - len(out); page < 1000 {
|
||||
q.Set("max-keys", strconv.Itoa(page))
|
||||
}
|
||||
if token != "" {
|
||||
q.Set("continuation-token", token)
|
||||
}
|
||||
u.RawQuery = canonicalQuery(q)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.URL = u
|
||||
resp, err := b.do(req, emptySHA256)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var page struct {
|
||||
Contents []struct {
|
||||
Key string `xml:"Key"`
|
||||
Size int64 `xml:"Size"`
|
||||
LastModified string `xml:"LastModified"`
|
||||
} `xml:"Contents"`
|
||||
IsTruncated bool `xml:"IsTruncated"`
|
||||
NextContinuationToken string `xml:"NextContinuationToken"`
|
||||
}
|
||||
err = xml.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&page)
|
||||
drain(resp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: the listing is not S3 XML: %v", ErrS3, err)
|
||||
}
|
||||
for _, c := range page.Contents {
|
||||
modified, _ := time.Parse(time.RFC3339, c.LastModified)
|
||||
out = append(out, Object{Key: c.Key, Size: c.Size, LastModified: modified})
|
||||
}
|
||||
if !page.IsTruncated || page.NextContinuationToken == "" || len(out) >= limit {
|
||||
return out, nil
|
||||
}
|
||||
token = page.NextContinuationToken
|
||||
}
|
||||
}
|
||||
|
||||
// refusal turns an S3 error response into words an admin can act on.
|
||||
func refusal(resp *http.Response) error {
|
||||
var e struct {
|
||||
Code string `xml:"Code"`
|
||||
Message string `xml:"Message"`
|
||||
}
|
||||
_ = xml.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&e)
|
||||
hint := map[string]string{
|
||||
"SignatureDoesNotMatch": "the secret key is wrong",
|
||||
"InvalidAccessKeyId": "the access key is not known to this server",
|
||||
"AccessDenied": "the key may not do this in this bucket",
|
||||
"NoSuchBucket": "there is no bucket by that name",
|
||||
"SlowDown": "the server asks to slow down",
|
||||
}[e.Code]
|
||||
switch {
|
||||
case hint != "":
|
||||
return fmt.Errorf("%w: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, hint)
|
||||
case e.Code != "":
|
||||
return fmt.Errorf("%w: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, e.Message)
|
||||
default:
|
||||
return fmt.Errorf("%w: HTTP %d", ErrS3, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func drain(resp *http.Response) {
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20))
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AWS's published examples ("Examples: Signature Calculations in AWS
|
||||
// Signature Version 4", Amazon S3 API reference).
|
||||
const (
|
||||
awsAccessKey = "AKIAIOSFODNN7EXAMPLE"
|
||||
awsSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
awsCred = "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"
|
||||
)
|
||||
|
||||
var awsTime = time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
func TestSignMatchesAWSsGetObjectExample(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/test.txt", nil)
|
||||
req.Header.Set("Range", "bytes=0-9")
|
||||
sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
|
||||
want := awsCred + ", SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
|
||||
if got := req.Header.Get("Authorization"); got != want {
|
||||
t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignMatchesAWSsPutObjectExample(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodPut, "http://examplebucket.s3.amazonaws.com/test$file.text", strings.NewReader("Welcome to Amazon S3."))
|
||||
req.Header.Set("Date", "Fri, 24 May 2013 00:00:00 GMT")
|
||||
req.Header.Set("X-Amz-Storage-Class", "REDUCED_REDUNDANCY")
|
||||
payload := hexSHA256([]byte("Welcome to Amazon S3."))
|
||||
if payload != "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072" {
|
||||
t.Fatalf("payload hash = %s", payload)
|
||||
}
|
||||
sign(req, payload, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
|
||||
want := awsCred + ", SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd"
|
||||
if got := req.Header.Get("Authorization"); got != want {
|
||||
t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignMatchesAWSsListObjectsExample(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
||||
sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
|
||||
want := awsCred + ", SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
|
||||
if got := req.Header.Get("Authorization"); got != want {
|
||||
t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUriEncodeKeepsOnlyUnreservedCharacters(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"01 - Song.flac": "01%20-%20Song.flac",
|
||||
"a+b=c&d": "a%2Bb%3Dc%26d",
|
||||
"Björk~_.": "Bj%C3%B6rk~_.",
|
||||
"a/b": "a%2Fb",
|
||||
} {
|
||||
if got := uriEncode(in); got != want {
|
||||
t.Errorf("uriEncode(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testBucket(url string) Bucket {
|
||||
return Bucket{Endpoint: url, Name: "minstrel", AccessKey: "ak", SecretKey: "sk",
|
||||
now: func() time.Time { return awsTime }}
|
||||
}
|
||||
|
||||
func TestPutSendsTheBodyToItsPathStyleKey(t *testing.T) {
|
||||
var gotPath, gotSHA, gotBody, gotAuth string
|
||||
var gotLen int64
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath, gotSHA, gotAuth, gotLen = r.URL.EscapedPath(), r.Header.Get("X-Amz-Content-Sha256"), r.Header.Get("Authorization"), r.ContentLength
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
gotBody = string(b)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
body := "flac bytes"
|
||||
sum := hexSHA256([]byte(body))
|
||||
if err := testBucket(srv.URL).Put(context.Background(), "music/A B/01 + x.flac", strings.NewReader(body), int64(len(body)), sum); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gotPath != "/minstrel/music/A%20B/01%20%2B%20x.flac" {
|
||||
t.Errorf("path = %s", gotPath)
|
||||
}
|
||||
if gotSHA != sum || gotBody != body || gotLen != int64(len(body)) {
|
||||
t.Errorf("sha %s body %q length %d", gotSHA, gotBody, gotLen)
|
||||
}
|
||||
if !strings.HasPrefix(gotAuth, "AWS4-HMAC-SHA256 Credential=ak/20130524/us-east-1/s3/aws4_request") {
|
||||
t.Errorf("Authorization = %s", gotAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFollowsContinuationTokens(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
if r.URL.Path != "/minstrel" || q.Get("list-type") != "2" || q.Get("prefix") != "database/" {
|
||||
http.Error(w, "unexpected "+r.URL.String(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if q.Get("continuation-token") == "" {
|
||||
fmt.Fprint(w, `<ListBucketResult><IsTruncated>true</IsTruncated><NextContinuationToken>next</NextContinuationToken>
|
||||
<Contents><Key>database/minstrel-20261101T030000Z.tar</Key><Size>10</Size><LastModified>2026-11-01T03:00:05.000Z</LastModified></Contents>
|
||||
<Contents><Key>database/minstrel-20261102T030000Z.tar</Key><Size>11</Size><LastModified>2026-11-02T03:00:05.000Z</LastModified></Contents>
|
||||
</ListBucketResult>`)
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, `<ListBucketResult><IsTruncated>false</IsTruncated>
|
||||
<Contents><Key>database/minstrel-20261103T030000Z.tar</Key><Size>12</Size><LastModified>2026-11-03T03:00:05.000Z</LastModified></Contents>
|
||||
</ListBucketResult>`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
objs, err := testBucket(srv.URL).List(context.Background(), "database/", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(objs) != 3 || objs[2].Key != "database/minstrel-20261103T030000Z.tar" || objs[1].Size != 11 ||
|
||||
!objs[0].LastModified.Equal(time.Date(2026, 11, 1, 3, 0, 5, 0, time.UTC)) {
|
||||
t.Errorf("objects = %+v", objs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARefusalSaysWhatIsWrong(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
fmt.Fprint(w, `<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match</Message></Error>`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
_, err := testBucket(srv.URL).List(context.Background(), "", 1)
|
||||
if !errors.Is(err, ErrS3) || !strings.Contains(err.Error(), "HTTP 403 SignatureDoesNotMatch: the secret key is wrong") {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreachableOrMalformedEndpointIsAnS3Error(t *testing.T) {
|
||||
srv := httptest.NewServer(http.NotFoundHandler())
|
||||
srv.Close()
|
||||
for name, endpoint := range map[string]string{"closed": srv.URL, "not http": "ftp://archives", "empty": ""} {
|
||||
if _, err := testBucket(endpoint).List(context.Background(), "", 1); !errors.Is(err, ErrS3) {
|
||||
t.Errorf("%s: err = %v, want ErrS3", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// emptySHA256 is the payload hash of a request with no body.
|
||||
const emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
|
||||
// sign adds AWS Signature Version 4 headers for S3 to req: X-Amz-Date,
|
||||
// X-Amz-Content-Sha256 and Authorization. Every header already on the
|
||||
// request is signed, along with the host.
|
||||
func sign(req *http.Request, payloadSHA256, accessKey, secretKey, region string, at time.Time) {
|
||||
at = at.UTC()
|
||||
stamp := at.Format("20060102T150405Z")
|
||||
date := at.Format("20060102")
|
||||
req.Header.Set("X-Amz-Date", stamp)
|
||||
req.Header.Set("X-Amz-Content-Sha256", payloadSHA256)
|
||||
|
||||
host := req.Host
|
||||
if host == "" {
|
||||
host = req.URL.Host
|
||||
}
|
||||
headers := map[string]string{"host": host}
|
||||
for name, values := range req.Header {
|
||||
headers[strings.ToLower(name)] = strings.TrimSpace(strings.Join(values, ","))
|
||||
}
|
||||
names := make([]string, 0, len(headers))
|
||||
for name := range headers {
|
||||
names = append(names, name)
|
||||
}
|
||||
slices.Sort(names)
|
||||
var canonHeaders strings.Builder
|
||||
for _, name := range names {
|
||||
canonHeaders.WriteString(name + ":" + headers[name] + "\n")
|
||||
}
|
||||
signed := strings.Join(names, ";")
|
||||
|
||||
canonical := strings.Join([]string{
|
||||
req.Method,
|
||||
canonicalPath(req.URL),
|
||||
canonicalQuery(req.URL.Query()),
|
||||
canonHeaders.String(),
|
||||
signed,
|
||||
payloadSHA256,
|
||||
}, "\n")
|
||||
scope := date + "/" + region + "/s3/aws4_request"
|
||||
toSign := "AWS4-HMAC-SHA256\n" + stamp + "\n" + scope + "\n" + hexSHA256([]byte(canonical))
|
||||
|
||||
key := hmacSHA256([]byte("AWS4"+secretKey), date)
|
||||
key = hmacSHA256(key, region)
|
||||
key = hmacSHA256(key, "s3")
|
||||
key = hmacSHA256(key, "aws4_request")
|
||||
signature := hex.EncodeToString(hmacSHA256(key, toSign))
|
||||
|
||||
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+accessKey+"/"+scope+
|
||||
", SignedHeaders="+signed+", Signature="+signature)
|
||||
}
|
||||
|
||||
// canonicalPath is the request path with each segment URI-encoded once.
|
||||
func canonicalPath(u *url.URL) string {
|
||||
p := u.EscapedPath()
|
||||
if p == "" {
|
||||
return "/"
|
||||
}
|
||||
if unescaped, err := url.PathUnescape(p); err == nil {
|
||||
p = unescaped
|
||||
}
|
||||
segments := strings.Split(p, "/")
|
||||
for i, s := range segments {
|
||||
segments[i] = uriEncode(s)
|
||||
}
|
||||
return strings.Join(segments, "/")
|
||||
}
|
||||
|
||||
// canonicalQuery is the query sorted by name, names and values URI-encoded.
|
||||
func canonicalQuery(q url.Values) string {
|
||||
pairs := make([]string, 0, len(q))
|
||||
for name, values := range q {
|
||||
for _, v := range values {
|
||||
pairs = append(pairs, uriEncode(name)+"="+uriEncode(v))
|
||||
}
|
||||
}
|
||||
slices.Sort(pairs)
|
||||
return strings.Join(pairs, "&")
|
||||
}
|
||||
|
||||
// uriEncode percent-encodes everything but the unreserved characters, as
|
||||
// SigV4 requires; url.QueryEscape differs on space and '~'.
|
||||
func uriEncode(s string) string {
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' || c == '_' || c == '.' || c == '~' {
|
||||
b.WriteByte(c)
|
||||
continue
|
||||
}
|
||||
b.WriteString("%" + strings.ToUpper(hex.EncodeToString([]byte{c})))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func hmacSHA256(key []byte, msg string) []byte {
|
||||
m := hmac.New(sha256.New, key)
|
||||
m.Write([]byte(msg))
|
||||
return m.Sum(nil)
|
||||
}
|
||||
|
||||
func hexSHA256(b []byte) string {
|
||||
sum := sha256.Sum256(b)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Contents is a backup's data: the schema version that wrote it, and each
|
||||
// table's rows as JSON objects with their keys sorted, in primary-key order.
|
||||
// An unchanged row is the same bytes in every backup.
|
||||
type Contents struct {
|
||||
Schema int64
|
||||
Tables map[string][][]byte
|
||||
}
|
||||
|
||||
// Snapshot reads every table in Tables as of one moment.
|
||||
func Snapshot(ctx context.Context, pool *pgxpool.Pool) (Contents, error) {
|
||||
c := Contents{Tables: make(map[string][][]byte, len(Tables))}
|
||||
tx, err := pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly})
|
||||
if err != nil {
|
||||
return c, fmt.Errorf("backup: begin: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
if c.Schema, err = schemaVersion(ctx, tx); err != nil {
|
||||
return c, err
|
||||
}
|
||||
for _, t := range Tables {
|
||||
lines, err := readTable(ctx, tx, t)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
c.Tables[t.Name] = lines
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func readTable(ctx context.Context, tx pgx.Tx, t Table) ([][]byte, error) {
|
||||
pk, err := primaryKey(ctx, tx, t.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
order := make([]string, len(pk))
|
||||
for i, col := range pk {
|
||||
order[i] = "t." + pgx.Identifier{col}.Sanitize()
|
||||
}
|
||||
omit := t.Omit
|
||||
if omit == nil {
|
||||
omit = []string{}
|
||||
}
|
||||
rows, err := tx.Query(ctx, fmt.Sprintf("SELECT to_jsonb(t) - $1::text[] FROM %s t ORDER BY %s",
|
||||
pgx.Identifier{t.Name}.Sanitize(), strings.Join(order, ", ")), omit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read %s: %w", t.Name, err)
|
||||
}
|
||||
lines, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) ([]byte, error) {
|
||||
var raw []byte
|
||||
if err := row.Scan(&raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return canonical(raw)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read %s: %w", t.Name, err)
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// canonical re-encodes a row with its keys sorted. Numbers pass through as
|
||||
// written, so a bigint or a float stays exact.
|
||||
func canonical(raw []byte) ([]byte, error) {
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.UseNumber()
|
||||
var row map[string]any
|
||||
if err := dec.Decode(&row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(row)
|
||||
}
|
||||
|
||||
// primaryKey returns a table's primary-key columns in key order.
|
||||
func primaryKey(ctx context.Context, tx pgx.Tx, table string) ([]string, error) {
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT a.attname
|
||||
FROM pg_index i
|
||||
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY (i.indkey)
|
||||
WHERE i.indrelid = to_regclass($1) AND i.indisprimary
|
||||
ORDER BY array_position(i.indkey, a.attnum)`, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read %s's primary key: %w", table, err)
|
||||
}
|
||||
cols, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read %s's primary key: %w", table, err)
|
||||
}
|
||||
if len(cols) == 0 {
|
||||
return nil, fmt.Errorf("backup: %s has no primary key to order its rows by", table)
|
||||
}
|
||||
return cols, nil
|
||||
}
|
||||
|
||||
func schemaVersion(ctx context.Context, tx pgx.Tx) (int64, error) {
|
||||
var v int64
|
||||
if err := tx.QueryRow(ctx, "SELECT version FROM schema_migrations").Scan(&v); err != nil {
|
||||
return 0, fmt.Errorf("backup: read the schema version: %w", err)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
@@ -1,193 +0,0 @@
|
||||
// Package backup reads Minstrel's irreplaceable data out of Postgres as JSON
|
||||
// Lines, one table at a time, and loads it back into an empty instance.
|
||||
//
|
||||
// What goes in is everything a person made or decided: accounts, likes, play
|
||||
// and skip history, playlists, follows, wants and requests, download history,
|
||||
// admin decisions, and settings. What stays out is everything the restored
|
||||
// server rebuilds by itself: what it computes from the music files, what it
|
||||
// fetches again from MusicBrainz, ListenBrainz and the other providers, and
|
||||
// logs of past runs. Excluded says why for each, and the coverage test fails
|
||||
// on a table that is in neither list.
|
||||
package backup
|
||||
|
||||
// Table is one table a backup holds.
|
||||
//
|
||||
// Omit names columns left out of the file. They restore to their defaults:
|
||||
// - a credential Minstrel stores sealed, which only the server holding the
|
||||
// stream secret could open, so the restored server asks for it again;
|
||||
// - a marker saying work on an excluded table is done (a tag source, an art
|
||||
// source), so the restored server does that work again rather than
|
||||
// believing it has the result.
|
||||
type Table struct {
|
||||
Name string
|
||||
Omit []string
|
||||
}
|
||||
|
||||
// Tables is every table a backup holds, in the order a restore loads them:
|
||||
// each after every table it references. There are no self-referencing
|
||||
// tables, so no table needs its rows loaded in depth order.
|
||||
var Tables = []Table{
|
||||
{Name: "users", Omit: []string{"subsonic_password", "listenbrainz_token"}},
|
||||
|
||||
// The library's identity. Every like, play and playlist entry hangs off a
|
||||
// track id, so the rows travel even though a scan could rebuild most of
|
||||
// their columns; the scan after a restore finds each file at its path and
|
||||
// keeps its id.
|
||||
{Name: "artists", Omit: []string{"artist_thumb_path", "artist_fanart_path", "artist_art_source", "artist_art_sources_version"}},
|
||||
{Name: "albums", Omit: []string{"cover_art_path", "cover_art_source", "cover_art_sources_version"}},
|
||||
{Name: "tracks", Omit: []string{"tag_source", "tag_sources_version"}},
|
||||
|
||||
// Settings.
|
||||
{Name: "library_settings"},
|
||||
{Name: "network_settings"},
|
||||
{Name: "registration_settings"},
|
||||
{Name: "smtp_config", Omit: []string{"password"}},
|
||||
{Name: "notification_email_settings"},
|
||||
{Name: "acoustid_settings", Omit: []string{"api_key"}},
|
||||
{Name: "fingerprint_settings"},
|
||||
{Name: "loudness_settings"},
|
||||
{Name: "cover_art_provider_settings", Omit: []string{"api_key"}},
|
||||
{Name: "cover_art_sources_meta"},
|
||||
{Name: "tag_provider_settings", Omit: []string{"api_key"}},
|
||||
{Name: "tag_sources_meta"},
|
||||
{Name: "acquisition_services", Omit: []string{"api_key", "password"}},
|
||||
{Name: "acquisition_settings"},
|
||||
{Name: "reacquisition_settings"},
|
||||
{Name: "discover_tuning"},
|
||||
{Name: "taste_tuning"},
|
||||
{Name: "recommendation_weight_profiles"},
|
||||
{Name: "recommendation_tuning_audit"},
|
||||
|
||||
// Follows, wants, requests and downloads.
|
||||
{Name: "release_type_profiles"},
|
||||
{Name: "follows"},
|
||||
{Name: "follow_seen_groups"},
|
||||
{Name: "wants"},
|
||||
{Name: "grabs"},
|
||||
{Name: "download_interventions"},
|
||||
{Name: "release_blocks"},
|
||||
{Name: "requests"},
|
||||
|
||||
// Listening history and likes.
|
||||
{Name: "play_sessions"},
|
||||
{Name: "play_events"},
|
||||
{Name: "skip_events"},
|
||||
{Name: "scrobble_queue"},
|
||||
{Name: "contextual_likes"},
|
||||
{Name: "general_likes"},
|
||||
{Name: "general_likes_albums"},
|
||||
{Name: "general_likes_artists"},
|
||||
|
||||
// Playlists and what is built from history. The taste profile and the
|
||||
// "you might like" rows are rebuilt nightly from the history above; they
|
||||
// travel so a restored Home is full before that first nightly pass.
|
||||
{Name: "playlists"},
|
||||
{Name: "playlist_tracks"},
|
||||
{Name: "system_playlist_runs"},
|
||||
{Name: "system_playlist_rotation_state"},
|
||||
{Name: "taste_profile_artists"},
|
||||
{Name: "taste_profile_tags"},
|
||||
{Name: "taste_profile_eras"},
|
||||
{Name: "taste_profile_moods"},
|
||||
{Name: "you_might_like_albums"},
|
||||
{Name: "you_might_like_artists"},
|
||||
|
||||
// Per-user preferences and notifications.
|
||||
{Name: "user_normalization_prefs"},
|
||||
{Name: "user_notification_prefs"},
|
||||
{Name: "user_notification_email_state"},
|
||||
{Name: "user_notifications"},
|
||||
{Name: "suggestion_snoozes"},
|
||||
{Name: "quarantine"},
|
||||
{Name: "quarantine_actions"},
|
||||
|
||||
// Admin decisions: an album taken off the wanted listing stays off, and a
|
||||
// dismissed duplicate group stays dismissed when the sweep finds it again.
|
||||
{Name: "library_album_matches"},
|
||||
{Name: "duplicate_sweeps"},
|
||||
{Name: "duplicate_groups"},
|
||||
{Name: "duplicate_group_members"},
|
||||
|
||||
// The library change feed is how the apps sync, including a first sync,
|
||||
// so a restored server carries it on from where the backup left it.
|
||||
{Name: "library_changes"},
|
||||
{Name: "audit_log"},
|
||||
{Name: "playback_errors"},
|
||||
}
|
||||
|
||||
// Excluded is every table a backup leaves out, with the reason.
|
||||
var Excluded = map[string]string{
|
||||
"schema_migrations": "the restored server's own migrations write it; the backup's schema version is in its manifest",
|
||||
"sessions": "sign-ins; everyone signs in again on the restored server",
|
||||
"password_resets": "one-time reset links, which expire within hours",
|
||||
"user_invites": "open sign-up links; an admin sends an invite again from the restored server",
|
||||
|
||||
"track_fingerprints": "computed from the music files",
|
||||
"track_loudness": "computed from the music files",
|
||||
"album_loudness": "computed from the music files",
|
||||
"track_acoustid_lookups": "looked up again from each track's fingerprint",
|
||||
"track_tags": "fetched again from the tag providers",
|
||||
"candidate_artist_tags": "fetched again from the tag providers",
|
||||
"candidate_artist_tag_state": "fetched again from the tag providers",
|
||||
|
||||
"mb_artists": "MusicBrainz cache, fetched again",
|
||||
"mb_release_groups": "MusicBrainz cache, fetched again",
|
||||
"mb_releases": "MusicBrainz cache, fetched again",
|
||||
"mb_release_tracks": "MusicBrainz cache, fetched again",
|
||||
"mb_artist_release_groups": "MusicBrainz cache, fetched again",
|
||||
|
||||
"listenbrainz_similar_recordings": "similarity, fetched again from ListenBrainz",
|
||||
"track_similarity": "similarity, fetched again from the similarity providers",
|
||||
"track_similarity_fetches": "similarity, fetched again from the similarity providers",
|
||||
"artist_similarity": "similarity, fetched again from the similarity providers",
|
||||
"artist_similarity_fetches": "similarity, fetched again from the similarity providers",
|
||||
"artist_similarity_unmatched": "similarity, fetched again from the similarity providers",
|
||||
|
||||
"want_candidates": "the last search's results; the next search finds them again",
|
||||
"scan_runs": "a log of past library scans",
|
||||
"diagnostic_events": "debug-mode diagnostics",
|
||||
}
|
||||
|
||||
// carried reports whether a table is in Tables.
|
||||
func carried(name string) bool {
|
||||
for _, t := range Tables {
|
||||
if t.Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Coverage checks a schema's tables against Tables and Excluded. It returns
|
||||
// one line per problem: a table in neither list, in both, or listed twice,
|
||||
// and a listed table the schema doesn't have.
|
||||
func Coverage(schema []string) []string {
|
||||
var problems []string
|
||||
inSchema := map[string]bool{}
|
||||
for _, name := range schema {
|
||||
inSchema[name] = true
|
||||
_, excluded := Excluded[name]
|
||||
switch {
|
||||
case carried(name) && excluded:
|
||||
problems = append(problems, name+": both backed up and excluded")
|
||||
case !carried(name) && !excluded:
|
||||
problems = append(problems, name+": neither backed up nor excluded; add it to Tables, or to Excluded with the reason")
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, t := range Tables {
|
||||
if seen[t.Name] {
|
||||
problems = append(problems, t.Name+": listed twice in Tables")
|
||||
}
|
||||
seen[t.Name] = true
|
||||
if !inSchema[t.Name] {
|
||||
problems = append(problems, t.Name+": in Tables but not in the schema")
|
||||
}
|
||||
}
|
||||
for name := range Excluded {
|
||||
if !inSchema[name] {
|
||||
problems = append(problems, name+": in Excluded but not in the schema")
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func everyListedTable() []string {
|
||||
var names []string
|
||||
for _, t := range Tables {
|
||||
names = append(names, t.Name)
|
||||
}
|
||||
for name := range Excluded {
|
||||
names = append(names, name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func TestCoverageAcceptsASchemaThatMatchesTheLists(t *testing.T) {
|
||||
if problems := Coverage(everyListedTable()); len(problems) != 0 {
|
||||
t.Fatalf("problems = %v, want none", problems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageNamesATableInNeitherList(t *testing.T) {
|
||||
problems := Coverage(append(everyListedTable(), "new_table"))
|
||||
if len(problems) != 1 || !strings.HasPrefix(problems[0], "new_table: neither") {
|
||||
t.Fatalf("problems = %v, want one naming new_table", problems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageNamesAListedTableTheSchemaLacks(t *testing.T) {
|
||||
schema := slices.DeleteFunc(everyListedTable(), func(n string) bool { return n == "playlists" || n == "scan_runs" })
|
||||
problems := Coverage(schema)
|
||||
if len(problems) != 2 {
|
||||
t.Fatalf("problems = %v, want playlists and scan_runs named", problems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryExclusionHasAReason(t *testing.T) {
|
||||
for name, why := range Excluded {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
t.Errorf("%s is excluded without a reason", name)
|
||||
}
|
||||
if carried(name) {
|
||||
t.Errorf("%s is both backed up and excluded", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// SettingsService reconciles the registry of compiled-in providers
|
||||
@@ -113,7 +112,7 @@ func (s *SettingsService) reconcile(ctx context.Context) error {
|
||||
}
|
||||
key := ""
|
||||
if row.ApiKey != nil {
|
||||
key = secret.Reveal(secret.CoverArtAPIKey.Purpose(), row.ProviderID, *row.ApiKey)
|
||||
key = *row.ApiKey
|
||||
}
|
||||
s.enabledIDs[p.ID()] = row.Enabled
|
||||
s.apiKeys[p.ID()] = key
|
||||
@@ -205,15 +204,11 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
|
||||
// (apiKeyChanged flag) and $4 (api_key value). The actual generated
|
||||
// field for $4 is ApiKey *string.
|
||||
apiKeyChanged := patch.APIKey != nil
|
||||
sealedKey, err := secret.SealPtr(secret.CoverArtAPIKey.Purpose(), providerID, patch.APIKey)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("seal provider api key: %w", err)
|
||||
}
|
||||
if err := q.UpdateProviderSettings(ctx, dbq.UpdateProviderSettingsParams{
|
||||
ProviderID: providerID,
|
||||
Enabled: enabledVal,
|
||||
Column3: apiKeyChanged,
|
||||
ApiKey: sealedKey,
|
||||
ApiKey: patch.APIKey,
|
||||
}); err != nil {
|
||||
return false, fmt.Errorf("update provider settings: %w", err)
|
||||
}
|
||||
@@ -230,7 +225,7 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
|
||||
for _, r := range rows {
|
||||
key := ""
|
||||
if r.ApiKey != nil {
|
||||
key = secret.Reveal(secret.CoverArtAPIKey.Purpose(), r.ProviderID, *r.ApiKey)
|
||||
key = *r.ApiKey
|
||||
}
|
||||
s.enabledIDs[r.ProviderID] = r.Enabled
|
||||
s.apiKeys[r.ProviderID] = key
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
package dbtest
|
||||
|
||||
import "git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
|
||||
// Integration tests run with a master key installed, as the server runs
|
||||
// after startup, so code that seals or opens a stored credential behaves as
|
||||
// it does in production.
|
||||
func init() {
|
||||
secret.Install([]byte("minstrel integration tests: not a deployment key"))
|
||||
}
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// AcoustID lookup settings: a database row, changed without a restart, shared
|
||||
@@ -98,12 +97,8 @@ func (s *AcoustIDSettingsService) Set(ctx context.Context, in AcoustIDSettingsUp
|
||||
}
|
||||
params := dbq.UpdateAcoustIDSettingsParams{Enabled: in.Enabled, MinScore: in.MinScore}
|
||||
if in.APIKey != nil {
|
||||
sealed, err := secret.Seal(secret.AcoustIDAPIKey.Purpose(), "", *in.APIKey)
|
||||
if err != nil {
|
||||
return AcoustIDSettings{}, fmt.Errorf("acoustid settings: seal key: %w", err)
|
||||
}
|
||||
params.SetApiKey = true
|
||||
params.ApiKey = sealed
|
||||
params.ApiKey = *in.APIKey
|
||||
}
|
||||
row, err := dbq.New(s.pool).UpdateAcoustIDSettings(ctx, params)
|
||||
if err != nil {
|
||||
@@ -131,7 +126,7 @@ func (s *AcoustIDSettingsService) Changed() <-chan struct{} {
|
||||
func acoustIDSettingsFromRow(row dbq.AcoustidSetting) AcoustIDSettings {
|
||||
out := AcoustIDSettings{Enabled: row.Enabled, MinScore: row.MinScore, UpdatedAt: row.UpdatedAt.Time}
|
||||
if row.ApiKey != nil {
|
||||
out.APIKey = secret.Reveal(secret.AcoustIDAPIKey.Purpose(), "", *row.ApiKey)
|
||||
out.APIKey = *row.ApiKey
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// ErrNotConfigured is returned when smtp_config.enabled is false or
|
||||
@@ -69,8 +68,7 @@ func (s *SMTPSender) Send(ctx context.Context, to, subject, textBody, htmlBody s
|
||||
|
||||
var auth smtp.Auth
|
||||
if cfg.Username != "" {
|
||||
password := secret.Reveal(secret.SMTPPassword.Purpose(), "", cfg.Password)
|
||||
auth = smtp.PlainAuth("", cfg.Username, password, cfg.Host)
|
||||
auth = smtp.PlainAuth("", cfg.Username, cfg.Password, cfg.Host)
|
||||
}
|
||||
|
||||
// We respect the use_tls flag: if true, we require STARTTLS via
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
func testPool(t *testing.T) (*pgxpool.Pool, *dbq.Queries) {
|
||||
@@ -64,13 +63,8 @@ func seed(t *testing.T, opts seedOpts) setup {
|
||||
t.Fatalf("user: %v", err)
|
||||
}
|
||||
if opts.lbToken != "" {
|
||||
// Stored sealed, as the settings route stores it.
|
||||
sealed, err := secret.Seal(secret.ListenBrainzToken.Purpose(), u.ID.String(), opts.lbToken)
|
||||
if err != nil {
|
||||
t.Fatalf("seal token: %v", err)
|
||||
}
|
||||
if err := q.SetListenBrainzToken(ctx, dbq.SetListenBrainzTokenParams{
|
||||
ID: u.ID, ListenbrainzToken: &sealed,
|
||||
ID: u.ID, ListenbrainzToken: &opts.lbToken,
|
||||
}); err != nil {
|
||||
t.Fatalf("token: %v", err)
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble/listenbrainz"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// backoffSchedule maps the failure count (1-indexed) to the delay before
|
||||
@@ -99,11 +98,7 @@ func (w *Worker) tickOnce(ctx context.Context) error {
|
||||
key := r.UserID.String()
|
||||
ub := batches[key]
|
||||
if ub == nil {
|
||||
token := secret.Reveal(secret.ListenBrainzToken.Purpose(), key, *r.LbToken)
|
||||
if token == "" {
|
||||
continue
|
||||
}
|
||||
ub = &userBatch{token: token, userID: r.UserID}
|
||||
ub = &userBatch{token: *r.LbToken, userID: r.UserID}
|
||||
batches[key] = ub
|
||||
}
|
||||
ub.queueIDs = append(ub.queueIDs, r.QueueID)
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
// Package secret seals the credentials Minstrel keeps for outside systems
|
||||
// (indexer and download-client keys, the SMTP password, provider API keys,
|
||||
// ListenBrainz tokens, Subsonic passwords) so the database holds ciphertext,
|
||||
// not the credential.
|
||||
//
|
||||
// Each column has its own key, derived from the install's master key with
|
||||
// HKDF-SHA256 and the column as the info string, so a value sealed for one
|
||||
// column cannot be opened as another, and the master key never touches a
|
||||
// cipher. Each value is also bound to the row it belongs to (its owner), so a
|
||||
// sealed value copied onto another row does not open there.
|
||||
//
|
||||
// The master key is the stream secret (MINSTREL_STREAM_SECRET, or the
|
||||
// stream_secret file in the data directory). It lives outside the database,
|
||||
// so a copy of the database alone opens nothing. Changing it makes every
|
||||
// sealed value unreadable: each one reads as unset and has to be entered
|
||||
// again in Settings.
|
||||
package secret
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/hkdf"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// ErrNoKey is a seal or open before Install has given the package a key.
|
||||
var ErrNoKey = errors.New("secret: no master key installed")
|
||||
|
||||
// ErrCorrupt is a stored value that does not open: the master key changed,
|
||||
// the value was edited by hand, or it belongs to a different row.
|
||||
var ErrCorrupt = errors.New("secret: the stored value does not open")
|
||||
|
||||
// sealedPrefix versions the format, so a future change of cipher can still
|
||||
// read what this one wrote. Nothing a person types starts with it by chance
|
||||
// often enough to matter, and Open rejects anything without it.
|
||||
const sealedPrefix = "v1."
|
||||
|
||||
var b64 = base64.RawURLEncoding
|
||||
|
||||
var (
|
||||
mu sync.RWMutex
|
||||
master []byte
|
||||
boxes map[string]cipher.AEAD
|
||||
)
|
||||
|
||||
// Install sets the master key every Seal and Open derives from. The server
|
||||
// calls it once at startup, before anything reads a credential.
|
||||
func Install(key []byte) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
master = append([]byte(nil), key...)
|
||||
boxes = map[string]cipher.AEAD{}
|
||||
}
|
||||
|
||||
// aead returns purpose's cipher, deriving it on first use.
|
||||
func aead(purpose string) (cipher.AEAD, error) {
|
||||
mu.RLock()
|
||||
a, ok := boxes[purpose]
|
||||
key := master
|
||||
mu.RUnlock()
|
||||
if ok {
|
||||
return a, nil
|
||||
}
|
||||
if len(key) == 0 {
|
||||
return nil, ErrNoKey
|
||||
}
|
||||
derived, err := hkdf.Key(sha256.New, key, nil, "minstrel/"+purpose, 32)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("secret: derive: %w", err)
|
||||
}
|
||||
block, err := aes.NewCipher(derived)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("secret: %w", err)
|
||||
}
|
||||
a, err = cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("secret: %w", err)
|
||||
}
|
||||
mu.Lock()
|
||||
if boxes != nil {
|
||||
boxes[purpose] = a
|
||||
}
|
||||
mu.Unlock()
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// IsSealed reports whether stored is in the sealed format.
|
||||
func IsSealed(stored string) bool { return strings.HasPrefix(stored, sealedPrefix) }
|
||||
|
||||
// Seal encrypts plain for purpose (the column it is stored in) and owner (the
|
||||
// row it belongs to). The empty string stays empty, so "is one set?" stays a
|
||||
// test for the empty string.
|
||||
func Seal(purpose, owner, plain string) (string, error) {
|
||||
if plain == "" {
|
||||
return "", nil
|
||||
}
|
||||
a, err := aead(purpose)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
nonce := make([]byte, a.NonceSize())
|
||||
if _, err := rand.Read(nonce); err != nil {
|
||||
return "", fmt.Errorf("secret: nonce: %w", err)
|
||||
}
|
||||
out := a.Seal(nonce, nonce, []byte(plain), []byte(owner))
|
||||
return sealedPrefix + b64.EncodeToString(out), nil
|
||||
}
|
||||
|
||||
// Open decrypts what Seal stored for the same purpose and owner. The empty
|
||||
// string opens to itself.
|
||||
func Open(purpose, owner, stored string) (string, error) {
|
||||
if stored == "" {
|
||||
return "", nil
|
||||
}
|
||||
a, err := aead(purpose)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
raw, ok := strings.CutPrefix(stored, sealedPrefix)
|
||||
if !ok {
|
||||
return "", ErrCorrupt
|
||||
}
|
||||
data, err := b64.DecodeString(raw)
|
||||
n := a.NonceSize()
|
||||
if err != nil || len(data) < n {
|
||||
return "", ErrCorrupt
|
||||
}
|
||||
plain, err := a.Open(nil, data[:n], data[n:], []byte(owner))
|
||||
if err != nil {
|
||||
return "", ErrCorrupt
|
||||
}
|
||||
return string(plain), nil
|
||||
}
|
||||
|
||||
// Reveal is Open for a caller about to use the credential: a value that does
|
||||
// not open is logged and read as unset, which is what it has become. The
|
||||
// admin sees "not set" in Settings and enters it again.
|
||||
func Reveal(purpose, owner, stored string) string {
|
||||
plain, err := Open(purpose, owner, stored)
|
||||
if err != nil {
|
||||
slog.Warn("secret: a stored credential does not open; it reads as unset until it is entered again",
|
||||
"column", purpose, "err", err)
|
||||
return ""
|
||||
}
|
||||
return plain
|
||||
}
|
||||
|
||||
// RevealPtr is Reveal for a nullable column: nil stays nil, and a value that
|
||||
// does not open reads as nil.
|
||||
func RevealPtr(purpose, owner string, stored *string) *string {
|
||||
if stored == nil {
|
||||
return nil
|
||||
}
|
||||
plain := Reveal(purpose, owner, *stored)
|
||||
if plain == "" {
|
||||
return nil
|
||||
}
|
||||
return &plain
|
||||
}
|
||||
|
||||
// SealPtr is Seal for a nullable column: nil and the empty string both store
|
||||
// NULL.
|
||||
func SealPtr(purpose, owner string, plain *string) (*string, error) {
|
||||
if plain == nil || *plain == "" {
|
||||
return nil, nil
|
||||
}
|
||||
sealed, err := Seal(purpose, owner, *plain)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &sealed, nil
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
package secret
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func withKey(t *testing.T, key string) {
|
||||
t.Helper()
|
||||
mu.RLock()
|
||||
prev := master
|
||||
mu.RUnlock()
|
||||
Install([]byte(key))
|
||||
t.Cleanup(func() { Install(prev) })
|
||||
}
|
||||
|
||||
func TestSealOpensForTheSamePurposeAndOwner(t *testing.T) {
|
||||
withKey(t, "test master key")
|
||||
sealed, err := Seal("t.col", "row-1", "hunter2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !IsSealed(sealed) || strings.Contains(sealed, "hunter2") {
|
||||
t.Fatalf("sealed = %q, want an opaque v1. value", sealed)
|
||||
}
|
||||
plain, err := Open("t.col", "row-1", sealed)
|
||||
if err != nil || plain != "hunter2" {
|
||||
t.Fatalf("Open = %q, %v; want hunter2", plain, err)
|
||||
}
|
||||
again, _ := Seal("t.col", "row-1", "hunter2")
|
||||
if again == sealed {
|
||||
t.Error("two seals of one value are identical; the nonce is not random")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenRefusesAnythingElse(t *testing.T) {
|
||||
withKey(t, "test master key")
|
||||
sealed, _ := Seal("t.col", "row-1", "hunter2")
|
||||
|
||||
cases := map[string]func() (string, error){
|
||||
"another owner": func() (string, error) { return Open("t.col", "row-2", sealed) },
|
||||
"another purpose": func() (string, error) { return Open("t.other", "row-1", sealed) },
|
||||
"plain text": func() (string, error) { return Open("t.col", "row-1", "hunter2") },
|
||||
"truncated": func() (string, error) { return Open("t.col", "row-1", sealed[:len(sealed)-4]) },
|
||||
"not base64": func() (string, error) { return Open("t.col", "row-1", "v1.!!!") },
|
||||
"too short": func() (string, error) { return Open("t.col", "row-1", "v1.AAAA") },
|
||||
}
|
||||
for name, open := range cases {
|
||||
if _, err := open(); !errors.Is(err, ErrCorrupt) {
|
||||
t.Errorf("%s: err = %v, want ErrCorrupt", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
Install([]byte("a different key"))
|
||||
if _, err := Open("t.col", "row-1", sealed); !errors.Is(err, ErrCorrupt) {
|
||||
t.Errorf("another master key: err = %v, want ErrCorrupt", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyStaysEmpty(t *testing.T) {
|
||||
withKey(t, "test master key")
|
||||
if s, err := Seal("t.col", "o", ""); s != "" || err != nil {
|
||||
t.Errorf("Seal(\"\") = %q, %v; want the empty string", s, err)
|
||||
}
|
||||
if s, err := Open("t.col", "o", ""); s != "" || err != nil {
|
||||
t.Errorf("Open(\"\") = %q, %v; want the empty string", s, err)
|
||||
}
|
||||
empty := ""
|
||||
if p, err := SealPtr("t.col", "o", &empty); p != nil || err != nil {
|
||||
t.Errorf("SealPtr(\"\") = %v, %v; want nil", p, err)
|
||||
}
|
||||
if p := RevealPtr("t.col", "o", nil); p != nil {
|
||||
t.Errorf("RevealPtr(nil) = %v, want nil", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoKeyIsAnErrorNotAPanic(t *testing.T) {
|
||||
withKey(t, "")
|
||||
if _, err := Seal("t.col", "o", "x"); !errors.Is(err, ErrNoKey) {
|
||||
t.Errorf("Seal: err = %v, want ErrNoKey", err)
|
||||
}
|
||||
if _, err := Open("t.col", "o", "v1.AAAA"); !errors.Is(err, ErrNoKey) {
|
||||
t.Errorf("Open: err = %v, want ErrNoKey", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRevealReadsAnUnopenableValueAsUnset(t *testing.T) {
|
||||
withKey(t, "test master key")
|
||||
if got := Reveal("t.col", "o", "not sealed"); got != "" {
|
||||
t.Errorf("Reveal = %q, want the empty string", got)
|
||||
}
|
||||
bad := "not sealed"
|
||||
if got := RevealPtr("t.col", "o", &bad); got != nil {
|
||||
t.Errorf("RevealPtr = %q, want nil", *got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryColumnHasADistinctPurpose(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
for _, c := range Columns {
|
||||
if seen[c.Purpose()] {
|
||||
t.Errorf("%s is listed twice", c.Purpose())
|
||||
}
|
||||
seen[c.Purpose()] = true
|
||||
}
|
||||
if !seen[AcquisitionAPIKey.Purpose()] || !seen[SubsonicPassword.Purpose()] {
|
||||
t.Error("Columns is missing a declared column")
|
||||
}
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package secret
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Column is one database column that holds a sealed credential.
|
||||
//
|
||||
// Its Purpose ("table.column") is the key-derivation label every Seal and
|
||||
// Open of that column passes. The owner a value is bound to is the row's key
|
||||
// as text (KeyColumn::text), or "" for a single-row settings table.
|
||||
type Column struct {
|
||||
Table string
|
||||
Column string
|
||||
KeyColumn string
|
||||
Singleton bool
|
||||
}
|
||||
|
||||
// Purpose is the label Seal and Open take for this column.
|
||||
func (c Column) Purpose() string { return c.Table + "." + c.Column }
|
||||
|
||||
// The sealed columns, and the purpose strings their callers pass. Every
|
||||
// credential Minstrel stores for an outside system, or that must be read back
|
||||
// in the clear, is listed here.
|
||||
var (
|
||||
AcquisitionAPIKey = Column{"acquisition_services", "api_key", "code", false}
|
||||
AcquisitionPassword = Column{"acquisition_services", "password", "code", false}
|
||||
AcoustIDAPIKey = Column{"acoustid_settings", "api_key", "id", true}
|
||||
SMTPPassword = Column{"smtp_config", "password", "id", true}
|
||||
ListenBrainzToken = Column{"users", "listenbrainz_token", "id", false}
|
||||
SubsonicPassword = Column{"users", "subsonic_password", "id", false}
|
||||
CoverArtAPIKey = Column{"cover_art_provider_settings", "api_key", "provider_id", false}
|
||||
TagProviderAPIKey = Column{"tag_provider_settings", "api_key", "provider_id", false}
|
||||
)
|
||||
|
||||
// Columns is every sealed column, for SealStored and for anything that has to
|
||||
// know which columns hold credentials (a backup leaves them out).
|
||||
var Columns = []Column{
|
||||
AcquisitionAPIKey, AcquisitionPassword, AcoustIDAPIKey, SMTPPassword,
|
||||
ListenBrainzToken, SubsonicPassword, CoverArtAPIKey, TagProviderAPIKey,
|
||||
}
|
||||
|
||||
// Owner is the owner string for a row whose key is key. A single-row table's
|
||||
// rows are owned by "".
|
||||
func (c Column) Owner(key string) string {
|
||||
if c.Singleton {
|
||||
return ""
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// SealStored seals every value in Columns that is still stored in the clear,
|
||||
// and returns how many it sealed. The server runs it at startup, after
|
||||
// migrations and before anything reads a credential, so a database written
|
||||
// before sealing existed is brought over in place. Each row is updated only
|
||||
// if it still holds the value that was read, so a concurrent write wins.
|
||||
func SealStored(ctx context.Context, pool *pgxpool.Pool) (int, error) {
|
||||
total := 0
|
||||
for _, c := range Columns {
|
||||
n, err := sealColumn(ctx, pool, c)
|
||||
total += n
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("seal %s: %w", c.Purpose(), err)
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func sealColumn(ctx context.Context, pool *pgxpool.Pool, c Column) (int, error) {
|
||||
table := pgx.Identifier{c.Table}.Sanitize()
|
||||
col := pgx.Identifier{c.Column}.Sanitize()
|
||||
key := pgx.Identifier{c.KeyColumn}.Sanitize()
|
||||
|
||||
rows, err := pool.Query(ctx, fmt.Sprintf(
|
||||
`SELECT %s::text, %s FROM %s WHERE %s IS NOT NULL AND %s <> '' AND %s NOT LIKE 'v1.%%'`,
|
||||
key, col, table, col, col, col))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
type pending struct{ key, plain string }
|
||||
var todo []pending
|
||||
for rows.Next() {
|
||||
var p pending
|
||||
if err := rows.Scan(&p.key, &p.plain); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
todo = append(todo, p)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
update := fmt.Sprintf(`UPDATE %s SET %s = $1 WHERE %s::text = $2 AND %s = $3`, table, col, key, col)
|
||||
sealed := 0
|
||||
for _, p := range todo {
|
||||
v, err := Seal(c.Purpose(), c.Owner(p.key), p.plain)
|
||||
if err != nil {
|
||||
return sealed, err
|
||||
}
|
||||
if _, err := pool.Exec(ctx, update, v, p.key, p.plain); err != nil {
|
||||
return sealed, err
|
||||
}
|
||||
sealed++
|
||||
}
|
||||
return sealed, nil
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
package secret_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
func newPool(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in -short mode")
|
||||
}
|
||||
dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
|
||||
}
|
||||
if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("pool: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
dbtest.ResetDB(t, pool)
|
||||
return pool
|
||||
}
|
||||
|
||||
// Every listed column exists, so a renamed or dropped column fails here
|
||||
// rather than leaving a credential unsealed.
|
||||
func TestColumnsExistInTheSchema_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
for _, c := range secret.Columns {
|
||||
var n int
|
||||
err := pool.QueryRow(context.Background(),
|
||||
`SELECT count(*) FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = $1 AND column_name IN ($2, $3)`,
|
||||
c.Table, c.Column, c.KeyColumn).Scan(&n)
|
||||
if err != nil || n != 2 {
|
||||
t.Errorf("%s (key %s): found %d of 2 columns, err %v", c.Purpose(), c.KeyColumn, n, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A database written before sealing existed is sealed in place at startup,
|
||||
// every value opens to what it held, and a second pass has nothing to do.
|
||||
func TestSealStoredSealsWhatIsInTheClear_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
ctx := context.Background()
|
||||
q := dbq.New(pool)
|
||||
|
||||
u, err := q.CreateUser(ctx, dbq.CreateUserParams{
|
||||
Username: dbtest.TestUserPrefix + "sealer", PasswordHash: "x", ApiTokenHash: "sealer", IsAdmin: false,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("user: %v", err)
|
||||
}
|
||||
if _, err := q.CreateAcquisitionService(ctx, dbq.CreateAcquisitionServiceParams{
|
||||
Code: "seal-test", Kind: "search", Protocol: "prowlarr", Label: "Seal test",
|
||||
Category: "", TimeoutMs: 1000, SearchesPerHour: 0,
|
||||
}); err != nil {
|
||||
t.Fatalf("acquisition row: %v", err)
|
||||
}
|
||||
seed := []string{
|
||||
`UPDATE acquisition_services SET api_key = 'acq-key', password = 'acq-pw' WHERE code = 'seal-test'`,
|
||||
`UPDATE acoustid_settings SET api_key = 'acoustid-key' WHERE id = true`,
|
||||
`UPDATE smtp_config SET password = 'smtp-pw' WHERE id = true`,
|
||||
fmt.Sprintf(`UPDATE users SET listenbrainz_token = 'lb-token', subsonic_password = 'sub-pw' WHERE id = '%s'`, u.ID.String()),
|
||||
`INSERT INTO cover_art_provider_settings (provider_id, enabled, display_order, api_key)
|
||||
VALUES ('seal-test', false, 99, 'cover-key')
|
||||
ON CONFLICT (provider_id) DO UPDATE SET api_key = EXCLUDED.api_key`,
|
||||
`INSERT INTO tag_provider_settings (provider_id, enabled, display_order, api_key)
|
||||
VALUES ('seal-test', false, 99, 'tag-key')
|
||||
ON CONFLICT (provider_id) DO UPDATE SET api_key = EXCLUDED.api_key`,
|
||||
}
|
||||
for _, s := range seed {
|
||||
if _, err := pool.Exec(ctx, s); err != nil {
|
||||
t.Fatalf("seed %q: %v", s, err)
|
||||
}
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
for _, s := range []string{
|
||||
`DELETE FROM cover_art_provider_settings WHERE provider_id = 'seal-test'`,
|
||||
`DELETE FROM tag_provider_settings WHERE provider_id = 'seal-test'`,
|
||||
`DELETE FROM acquisition_services WHERE code = 'seal-test'`,
|
||||
`UPDATE acoustid_settings SET api_key = NULL WHERE id = true`,
|
||||
`UPDATE smtp_config SET password = '' WHERE id = true`,
|
||||
} {
|
||||
_, _ = pool.Exec(context.Background(), s)
|
||||
}
|
||||
})
|
||||
|
||||
want := []struct {
|
||||
col secret.Column
|
||||
key string
|
||||
plain string
|
||||
}{
|
||||
{secret.AcquisitionAPIKey, "seal-test", "acq-key"},
|
||||
{secret.AcquisitionPassword, "seal-test", "acq-pw"},
|
||||
{secret.AcoustIDAPIKey, "true", "acoustid-key"},
|
||||
{secret.SMTPPassword, "true", "smtp-pw"},
|
||||
{secret.ListenBrainzToken, u.ID.String(), "lb-token"},
|
||||
{secret.SubsonicPassword, u.ID.String(), "sub-pw"},
|
||||
{secret.CoverArtAPIKey, "seal-test", "cover-key"},
|
||||
{secret.TagProviderAPIKey, "seal-test", "tag-key"},
|
||||
}
|
||||
if len(want) != len(secret.Columns) {
|
||||
t.Fatalf("the test covers %d columns, secret.Columns lists %d", len(want), len(secret.Columns))
|
||||
}
|
||||
|
||||
n, err := secret.SealStored(ctx, pool)
|
||||
if err != nil {
|
||||
t.Fatalf("SealStored: %v", err)
|
||||
}
|
||||
if n < len(want) {
|
||||
t.Errorf("sealed %d values, want at least %d", n, len(want))
|
||||
}
|
||||
|
||||
for _, w := range want {
|
||||
var stored string
|
||||
query := fmt.Sprintf(`SELECT %s FROM %s WHERE %s::text = $1`,
|
||||
pgx.Identifier{w.col.Column}.Sanitize(), pgx.Identifier{w.col.Table}.Sanitize(),
|
||||
pgx.Identifier{w.col.KeyColumn}.Sanitize())
|
||||
if err := pool.QueryRow(ctx, query, w.key).Scan(&stored); err != nil {
|
||||
t.Errorf("%s: read: %v", w.col.Purpose(), err)
|
||||
continue
|
||||
}
|
||||
if !secret.IsSealed(stored) {
|
||||
t.Errorf("%s = %q, want it sealed", w.col.Purpose(), stored)
|
||||
continue
|
||||
}
|
||||
got, err := secret.Open(w.col.Purpose(), w.col.Owner(w.key), stored)
|
||||
if err != nil || got != w.plain {
|
||||
t.Errorf("%s opens to %q (err %v), want %q", w.col.Purpose(), got, err, w.plain)
|
||||
}
|
||||
}
|
||||
|
||||
again, err := secret.SealStored(ctx, pool)
|
||||
if err != nil || again != 0 {
|
||||
t.Errorf("second pass sealed %d (err %v), want 0", again, err)
|
||||
}
|
||||
}
|
||||
@@ -272,41 +272,73 @@ func TestDevChannel_PublishesDevAloneAndNeverLatest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A merge to main is enough to put a new app in front of stable phones: main
|
||||
// builds the signed APK from its own commit and bakes it into :latest, as dev
|
||||
// does for :dev. The regression to guard is main falling back to a tag-only
|
||||
// APK build, which leaves every merged app change waiting on a release while
|
||||
// each build stays green.
|
||||
func TestEveryBranchShipsTheAppBuiltFromItsOwnCommit(t *testing.T) {
|
||||
jobs := releaseJobs(t)
|
||||
|
||||
build := strings.Join(strings.Fields(jobs["android-release"].If), " ")
|
||||
for _, ref := range []string{"refs/heads/main", "refs/heads/dev", "refs/tags/v"} {
|
||||
if !strings.Contains(build, ref) {
|
||||
t.Errorf("android-release does not build on %s, so that ref's image carries no app of its own\n if: %s", ref, build)
|
||||
}
|
||||
// The two bundling paths must stay mutually exclusive. If the rebundle step's
|
||||
// condition were relaxed back to "not a tag", a dev push would run BOTH: stage
|
||||
// its freshly-built APK, then overwrite it with the previous release's. The
|
||||
// image would still build and the sidecar would still parse — it would just
|
||||
// quietly serve stale art to the channel whose whole job is being current.
|
||||
func TestDevChannel_RebundlePathIsMainOnly(t *testing.T) {
|
||||
body, err := os.ReadFile(filepath.Join(repoRoot(t), ".gitea", "workflows", "release.yml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
yaml := string(body)
|
||||
|
||||
image := strings.Join(strings.Fields(jobs["image-release"].If), " ")
|
||||
if !strings.Contains(image, "needs.android-release.result == 'success'") || strings.Contains(image, "'skipped'") {
|
||||
t.Errorf("image-release can publish without a freshly-built APK\n if: %s", image)
|
||||
}
|
||||
|
||||
yaml := releaseYAML(t)
|
||||
if strings.Contains(yaml, "releases/latest") {
|
||||
t.Error("release.yml fetches the latest release; an image bundling a release's APK ships an app older than its own commit")
|
||||
}
|
||||
|
||||
// Both bundling steps run on every ref. Sliced from image-release on,
|
||||
// because release-assets has a step of the same name.
|
||||
i := strings.Index(yaml, "\n image-release:")
|
||||
i := strings.Index(yaml, "- name: Bundle latest release APK")
|
||||
if i < 0 {
|
||||
t.Fatal("no image-release job")
|
||||
t.Fatal("no 'Bundle latest release APK' step")
|
||||
}
|
||||
for _, name := range []string{"Download signed APK artifact", "Stage bundled APK + version sidecar"} {
|
||||
step := stepBody(t, yaml[i:], name)
|
||||
if strings.Contains(step, "github.ref") {
|
||||
t.Errorf("%q is gated by ref, so some ref's image ships without its APK\n%s", name, step)
|
||||
step := yaml[i:]
|
||||
if j := strings.Index(step, "\n - name:"); j >= 0 {
|
||||
step = step[:j]
|
||||
}
|
||||
if !strings.Contains(step, "github.ref == 'refs/heads/main'") {
|
||||
t.Errorf("the rebundle step is not gated to main; a dev push would overwrite its own APK\n%s", step)
|
||||
}
|
||||
}
|
||||
|
||||
// The runner invokes `shell: bash` as `bash -e -o pipefail`. That makes any
|
||||
// command substitution whose grep matches NOTHING fatal at the assignment —
|
||||
// pipefail reports the grep's non-zero status even though `head` succeeded —
|
||||
// so the step dies before reaching the `if` written to handle the empty case.
|
||||
//
|
||||
// This is not hypothetical. It took down the first `main` build after the
|
||||
// version rework: the newest release predated sidecar assets, its
|
||||
// `.apk.version` grep matched nothing, and the step aborted instead of
|
||||
// falling through to the name-only branch that exists precisely for it. The
|
||||
// same latent hazard sat on the other two assignments and had simply never
|
||||
// fired, because their greps always matched.
|
||||
//
|
||||
// Pins the property that makes every "degrades gracefully" branch in that
|
||||
// step reachable at all.
|
||||
func TestBundleStep_GrepsCannotKillTheStep(t *testing.T) {
|
||||
body, err := os.ReadFile(filepath.Join(repoRoot(t), ".gitea", "workflows", "release.yml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
yaml := string(body)
|
||||
|
||||
i := strings.Index(yaml, "- name: Bundle latest release APK")
|
||||
if i < 0 {
|
||||
t.Fatal("no 'Bundle latest release APK' step")
|
||||
}
|
||||
step := yaml[i:]
|
||||
if j := strings.Index(step, "\n - name:"); j >= 0 {
|
||||
step = step[:j]
|
||||
}
|
||||
|
||||
for _, line := range strings.Split(step, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "#") || !strings.Contains(trimmed, "grep") {
|
||||
continue
|
||||
}
|
||||
// Only assignments from a command substitution can abort the step.
|
||||
if !strings.Contains(trimmed, `="$(`) {
|
||||
continue
|
||||
}
|
||||
if !strings.HasSuffix(trimmed, "|| true") {
|
||||
t.Errorf("this assignment dies under pipefail when its grep matches nothing, "+
|
||||
"skipping the empty-case branch below it:\n %s", trimmed)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -344,7 +376,7 @@ func TestReleaseBuild_RefreshesTheChannelAndMintsNothingElse(t *testing.T) {
|
||||
arm, _, _ := imageTagArms(t, releaseYAML(t))
|
||||
|
||||
if !strings.Contains(arm, "${IMAGE}:latest") {
|
||||
t.Errorf("tag arm does not refresh :latest — the server would hand out a different APK build from the one attached to the Release\n%s", arm)
|
||||
t.Errorf("tag arm does not refresh :latest — the channel would keep serving the PREVIOUS release's APK until someone pushed to main\n%s", arm)
|
||||
}
|
||||
if strings.Contains(arm, "GITHUB_SHA") {
|
||||
t.Errorf("tag arm mints a :<sha> image; that would re-push main's immutable rollback target with different bundled contents\n%s", arm)
|
||||
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// Config controls wire-format concessions the server makes for Subsonic
|
||||
@@ -102,15 +101,11 @@ func authenticate(r *http.Request, pool *pgxpool.Pool, cfg Config) (dbq.User, in
|
||||
return dbq.User{}, ErrGeneric, "Auth lookup failed"
|
||||
}
|
||||
|
||||
// The stored Subsonic password is sealed; both t+s and p= compare against
|
||||
// it opened. One that does not open reads as not configured.
|
||||
subsonicPassword := secret.RevealPtr(secret.SubsonicPassword.Purpose(), user.ID.String(), user.SubsonicPassword)
|
||||
|
||||
if token, salt := params.Get("t"), params.Get("s"); token != "" && salt != "" {
|
||||
if subsonicPassword == nil {
|
||||
if user.SubsonicPassword == nil {
|
||||
return dbq.User{}, ErrTokenNotSupported, "Subsonic token auth not configured; use apiKey or set a Subsonic password"
|
||||
}
|
||||
sum := md5.Sum([]byte(*subsonicPassword + salt))
|
||||
sum := md5.Sum([]byte(*user.SubsonicPassword + salt))
|
||||
expected := hex.EncodeToString(sum[:])
|
||||
if subtle.ConstantTimeCompare([]byte(expected), []byte(token)) != 1 {
|
||||
return dbq.User{}, ErrWrongCredentials, "Wrong username or password"
|
||||
@@ -122,14 +117,14 @@ func authenticate(r *http.Request, pool *pgxpool.Pool, cfg Config) (dbq.User, in
|
||||
if !cfg.AllowPlaintextPassword {
|
||||
return dbq.User{}, ErrTokenNotSupported, "Plaintext password auth is disabled; use apiKey or t+s"
|
||||
}
|
||||
if subsonicPassword == nil {
|
||||
if user.SubsonicPassword == nil {
|
||||
return dbq.User{}, ErrTokenNotSupported, "Subsonic password not configured for this user"
|
||||
}
|
||||
decoded, err := decodePassword(password)
|
||||
if err != nil {
|
||||
return dbq.User{}, ErrWrongCredentials, "Malformed password"
|
||||
}
|
||||
if subtle.ConstantTimeCompare([]byte(*subsonicPassword), []byte(decoded)) != 1 {
|
||||
if subtle.ConstantTimeCompare([]byte(*user.SubsonicPassword), []byte(decoded)) != 1 {
|
||||
return dbq.User{}, ErrWrongCredentials, "Wrong username or password"
|
||||
}
|
||||
return user, 0, ""
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||
)
|
||||
|
||||
// SettingsService reconciles the compiled-in provider registry with the
|
||||
@@ -102,7 +101,7 @@ func (s *SettingsService) reconcile(ctx context.Context) error {
|
||||
}
|
||||
key := ""
|
||||
if row.ApiKey != nil {
|
||||
key = secret.Reveal(secret.TagProviderAPIKey.Purpose(), row.ProviderID, *row.ApiKey)
|
||||
key = *row.ApiKey
|
||||
}
|
||||
s.enabledIDs[p.ID()] = row.Enabled
|
||||
s.apiKeys[p.ID()] = key
|
||||
@@ -182,15 +181,11 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
|
||||
if patch.Enabled != nil {
|
||||
enabledVal = *patch.Enabled
|
||||
}
|
||||
sealedKey, err := secret.SealPtr(secret.TagProviderAPIKey.Purpose(), providerID, patch.APIKey)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("seal provider api key: %w", err)
|
||||
}
|
||||
if err := q.UpdateTagProviderSettings(ctx, dbq.UpdateTagProviderSettingsParams{
|
||||
ProviderID: providerID,
|
||||
Enabled: enabledVal,
|
||||
Column3: patch.APIKey != nil,
|
||||
ApiKey: sealedKey,
|
||||
ApiKey: patch.APIKey,
|
||||
}); err != nil {
|
||||
return false, fmt.Errorf("update provider settings: %w", err)
|
||||
}
|
||||
@@ -205,7 +200,7 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
|
||||
for _, r := range rows {
|
||||
key := ""
|
||||
if r.ApiKey != nil {
|
||||
key = secret.Reveal(secret.TagProviderAPIKey.Purpose(), r.ProviderID, *r.ApiKey)
|
||||
key = *r.ApiKey
|
||||
}
|
||||
s.enabledIDs[r.ProviderID] = r.Enabled
|
||||
s.apiKeys[r.ProviderID] = key
|
||||
|
||||
Reference in New Issue
Block a user