Compare commits
2
Commits
main
...
a67897a6cd
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a67897a6cd | ||
|
|
a8a2d20307 |
@@ -35,6 +35,7 @@ import (
|
|||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/requests"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/requests"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble/listenbrainz"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble/listenbrainz"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/server"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/server"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/similarity"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/similarity"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/subsonic"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/subsonic"
|
||||||
@@ -128,6 +129,18 @@ func run() error {
|
|||||||
}
|
}
|
||||||
defer pool.Close()
|
defer pool.Close()
|
||||||
|
|
||||||
|
// Stored credentials are sealed under a key derived from the stream
|
||||||
|
// secret. Install it before anything reads one, and seal any value still
|
||||||
|
// stored in the clear.
|
||||||
|
secret.Install(cfg.StreamSecret)
|
||||||
|
sealed, err := secret.SealStored(ctx, pool)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("seal stored credentials: %w", err)
|
||||||
|
}
|
||||||
|
if sealed > 0 {
|
||||||
|
logger.Info("sealed stored credentials", "count", sealed)
|
||||||
|
}
|
||||||
|
|
||||||
// Fingerprinting settings: one instance, shared by the scanner,
|
// Fingerprinting settings: one instance, shared by the scanner,
|
||||||
// the fingerprint backfill, the duplicate sweep and the admin API, so a save
|
// the fingerprint backfill, the duplicate sweep and the admin API, so a save
|
||||||
// reaches all of them without a restart. A load failure is logged, not fatal:
|
// reaches all of them without a restart. A load failure is logged, not fatal:
|
||||||
|
|||||||
+26
-6
@@ -49,22 +49,42 @@ cross-origin preflight. Requests authenticated with a bearer token or the
|
|||||||
Subsonic query parameters carry nothing a browser attaches on its own, so
|
Subsonic query parameters carry nothing a browser attaches on its own, so
|
||||||
they aren't checked.
|
they aren't checked.
|
||||||
|
|
||||||
## Subsonic sign-in, and the one password stored in plain text
|
## Credentials Minstrel has to read back
|
||||||
|
|
||||||
|
Some credentials can't be hashed, because Minstrel has to send or compare the
|
||||||
|
value itself:
|
||||||
|
- the API keys and passwords for indexers, Prowlarr and download clients;
|
||||||
|
- the AcoustID key, and the cover-art and tag provider keys;
|
||||||
|
- the SMTP password;
|
||||||
|
- each user's ListenBrainz token and Subsonic password.
|
||||||
|
|
||||||
|
These are encrypted in the database (AES-256-GCM). Each column has its own key,
|
||||||
|
derived from the stream secret (`MINSTREL_STREAM_SECRET`, or the
|
||||||
|
`stream_secret` file in the data directory), and each value is bound to its
|
||||||
|
row. The stream secret is never stored in the database, so a copy of the
|
||||||
|
database alone doesn't reveal any of them. None of them is ever sent back to a
|
||||||
|
browser: Settings only shows whether one is set.
|
||||||
|
|
||||||
|
**Keep the stream secret with the data directory, and don't change it
|
||||||
|
casually.** If it changes, every stored credential reads as not set, and each
|
||||||
|
one has to be entered again in Settings (Subsonic passwords generated again).
|
||||||
|
Minstrel logs a warning for each one that no longer opens.
|
||||||
|
|
||||||
|
## Subsonic sign-in
|
||||||
|
|
||||||
Classic Subsonic clients sign in with `t` and `s`: the MD5 of the password
|
Classic Subsonic clients sign in with `t` and `s`: the MD5 of the password
|
||||||
followed by a random salt. To check that, the server has to know the password
|
followed by a random salt. To check that, the server has to know the password
|
||||||
itself, so supporting this sign-in method means storing a password Minstrel
|
itself, so supporting this sign-in method means storing a password Minstrel
|
||||||
can read. That is the Subsonic password, and it is the only credential
|
can read: the Subsonic password, encrypted as described above.
|
||||||
Minstrel keeps unhashed.
|
|
||||||
|
|
||||||
- **The recommended way in is the API key.** Clients that support the
|
- **The recommended way in is the API key.** Clients that support the
|
||||||
OpenSubsonic `apiKey` should use it. The key is stored hashed and can be
|
OpenSubsonic `apiKey` should use it. The key is stored hashed and can be
|
||||||
replaced at any time in Settings.
|
replaced at any time in Settings.
|
||||||
- **The Subsonic password is never your login password.** Minstrel generates
|
- **The Subsonic password is never your login password.** Minstrel generates
|
||||||
it (**Settings → Subsonic password**), shows it once, and lets you replace
|
it (**Settings → Subsonic password**), shows it once, and lets you replace
|
||||||
or turn it off. Because it is random, a copy of the database exposes access
|
or turn it off. Because it is random, even someone holding both the database
|
||||||
to this server's Subsonic API and nothing else: it can't be a password you
|
and the stream secret gains access to this server's Subsonic API and nothing
|
||||||
also use somewhere else.
|
else: it can't be a password you also use somewhere else.
|
||||||
- **`t`/`s` and `p=` sign-in are off for an account until it has a Subsonic
|
- **`t`/`s` and `p=` sign-in are off for an account until it has a Subsonic
|
||||||
password.** Plain `p=` sign-in is additionally off server-wide unless
|
password.** Plain `p=` sign-in is additionally off server-wide unless
|
||||||
`subsonic.allow_plaintext_password` is enabled.
|
`subsonic.allow_plaintext_password` is enabled.
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ func Build(ctx context.Context, pool *pgxpool.Pool) (*Service, dbq.AcquisitionSe
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, settings, fmt.Errorf("acquire: read search sources: %w", err)
|
return nil, settings, fmt.Errorf("acquire: read search sources: %w", err)
|
||||||
}
|
}
|
||||||
clients, err := q.ListUsableAcquisitionServices(ctx, JobDownload)
|
clients, err := usableServices(ctx, q, JobDownload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, settings, fmt.Errorf("acquire: read download clients: %w", err)
|
return nil, settings, fmt.Errorf("acquire: read download clients: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import (
|
|||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/exchange"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/exchange"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/prowlarr"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/prowlarr"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Kinds: which list a row appears in.
|
// Kinds: which list a row appears in.
|
||||||
@@ -159,7 +160,7 @@ func DownloadClient(row dbq.AcquisitionService) (downloader.Driver, error) {
|
|||||||
// DownloadClientFor builds the download client a grab went to, by the code
|
// DownloadClientFor builds the download client a grab went to, by the code
|
||||||
// of its acquisition_services row.
|
// of its acquisition_services row.
|
||||||
func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (downloader.Driver, error) {
|
func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (downloader.Driver, error) {
|
||||||
row, err := dbq.New(pool).GetAcquisitionService(ctx, code)
|
row, err := getService(ctx, dbq.New(pool), code)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, pgx.ErrNoRows) {
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
return nil, fmt.Errorf("the download client %q is no longer set up", code)
|
return nil, fmt.Errorf("the download client %q is no longer set up", code)
|
||||||
@@ -171,7 +172,7 @@ func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (do
|
|||||||
|
|
||||||
// Sources builds every enabled search source.
|
// Sources builds every enabled search source.
|
||||||
func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) {
|
func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) {
|
||||||
rows, err := dbq.New(pool).ListUsableAcquisitionServices(ctx, JobSearch)
|
rows, err := usableServices(ctx, dbq.New(pool), JobSearch)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -182,4 +183,33 @@ func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getService and usableServices are the only reads of a full
|
||||||
|
// acquisition_services row. They hand back the credentials opened, so the
|
||||||
|
// clients built from a row are given the key and password themselves.
|
||||||
|
func getService(ctx context.Context, q *dbq.Queries, code string) (dbq.AcquisitionService, error) {
|
||||||
|
row, err := q.GetAcquisitionService(ctx, code)
|
||||||
|
if err != nil {
|
||||||
|
return row, err
|
||||||
|
}
|
||||||
|
return revealed(row), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func usableServices(ctx context.Context, q *dbq.Queries, job string) ([]dbq.AcquisitionService, error) {
|
||||||
|
rows, err := q.ListUsableAcquisitionServices(ctx, job)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for i := range rows {
|
||||||
|
rows[i] = revealed(rows[i])
|
||||||
|
}
|
||||||
|
return rows, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// revealed is row with its sealed credentials opened.
|
||||||
|
func revealed(row dbq.AcquisitionService) dbq.AcquisitionService {
|
||||||
|
row.ApiKey = secret.Reveal(secret.AcquisitionAPIKey.Purpose(), row.Code, row.ApiKey)
|
||||||
|
row.Password = secret.Reveal(secret.AcquisitionPassword.Purpose(), row.Code, row.Password)
|
||||||
|
return row
|
||||||
|
}
|
||||||
|
|
||||||
func msDuration(ms int32) time.Duration { return time.Duration(ms) * time.Millisecond }
|
func msDuration(ms int32) time.Duration { return time.Duration(ms) * time.Millisecond }
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrNotFound is a code with no row.
|
// ErrNotFound is a code with no row.
|
||||||
@@ -49,7 +50,7 @@ func (s *Store) List(ctx context.Context) ([]dbq.ListAcquisitionServicesRow, err
|
|||||||
|
|
||||||
// Get returns one row, secrets included: for building a client only.
|
// Get returns one row, secrets included: for building a client only.
|
||||||
func (s *Store) Get(ctx context.Context, code string) (dbq.AcquisitionService, error) {
|
func (s *Store) Get(ctx context.Context, code string) (dbq.AcquisitionService, error) {
|
||||||
row, err := dbq.New(s.pool).GetAcquisitionService(ctx, code)
|
row, err := getService(ctx, dbq.New(s.pool), code)
|
||||||
if errors.Is(err, pgx.ErrNoRows) {
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
return row, ErrNotFound
|
return row, ErrNotFound
|
||||||
}
|
}
|
||||||
@@ -163,19 +164,27 @@ func (s *Store) Delete(ctx context.Context, code string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetAPIKey stores a key, trimmed; empty clears it.
|
// SetAPIKey stores a key, trimmed and sealed; empty clears it.
|
||||||
func (s *Store) SetAPIKey(ctx context.Context, code, key string) error {
|
func (s *Store) SetAPIKey(ctx context.Context, code, key string) error {
|
||||||
|
sealed, err := secret.Seal(secret.AcquisitionAPIKey.Purpose(), code, strings.TrimSpace(key))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
n, err := dbq.New(s.pool).SetAcquisitionServiceAPIKey(ctx, dbq.SetAcquisitionServiceAPIKeyParams{
|
n, err := dbq.New(s.pool).SetAcquisitionServiceAPIKey(ctx, dbq.SetAcquisitionServiceAPIKeyParams{
|
||||||
Code: code, ApiKey: strings.TrimSpace(key),
|
Code: code, ApiKey: sealed,
|
||||||
})
|
})
|
||||||
return affected(n, err)
|
return affected(n, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetPassword stores a password as typed (spaces are part of a password);
|
// SetPassword stores a password as typed (spaces are part of a password),
|
||||||
// empty clears it.
|
// sealed; empty clears it.
|
||||||
func (s *Store) SetPassword(ctx context.Context, code, password string) error {
|
func (s *Store) SetPassword(ctx context.Context, code, password string) error {
|
||||||
|
sealed, err := secret.Seal(secret.AcquisitionPassword.Purpose(), code, password)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
n, err := dbq.New(s.pool).SetAcquisitionServicePassword(ctx, dbq.SetAcquisitionServicePasswordParams{
|
n, err := dbq.New(s.pool).SetAcquisitionServicePassword(ctx, dbq.SetAcquisitionServicePasswordParams{
|
||||||
Code: code, Password: password,
|
Code: code, Password: sealed,
|
||||||
})
|
})
|
||||||
return affected(n, err)
|
return affected(n, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
func newPool(t *testing.T) *pgxpool.Pool {
|
func newPool(t *testing.T) *pgxpool.Pool {
|
||||||
@@ -83,6 +84,15 @@ func TestStore_AddsConfiguresAndRemoves_Integration(t *testing.T) {
|
|||||||
if err != nil || full.ApiKey != "key" {
|
if err != nil || full.ApiKey != "key" {
|
||||||
t.Errorf("stored key = %q (err %v), want it trimmed", full.ApiKey, err)
|
t.Errorf("stored key = %q (err %v), want it trimmed", full.ApiKey, err)
|
||||||
}
|
}
|
||||||
|
// The column holds the key sealed, never the key.
|
||||||
|
var raw string
|
||||||
|
if err := s.pool.QueryRow(ctx,
|
||||||
|
"SELECT api_key FROM acquisition_services WHERE code = $1", first.Code).Scan(&raw); err != nil {
|
||||||
|
t.Fatalf("read api_key: %v", err)
|
||||||
|
}
|
||||||
|
if raw == "key" || !secret.IsSealed(raw) {
|
||||||
|
t.Errorf("api_key column = %q, want it sealed", raw)
|
||||||
|
}
|
||||||
rows, err := s.List(ctx)
|
rows, err := s.List(ctx)
|
||||||
if err != nil || len(rows) != 2 {
|
if err != nil || len(rows) != 2 {
|
||||||
t.Fatalf("list: %d rows, err %v", len(rows), err)
|
t.Fatalf("list: %d rows, err %v", len(rows), err)
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
type smtpConfigResp struct {
|
type smtpConfigResp struct {
|
||||||
@@ -76,11 +77,15 @@ func (h *handlers) handleUpdateSMTPConfig(w http.ResponseWriter, r *http.Request
|
|||||||
writeErrWithLog(w, h.logger, "admin smtp: get failed", apierror.Internal(err))
|
writeErrWithLog(w, h.logger, "admin smtp: get failed", apierror.Internal(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// Preserve existing password when the request sends "" (so the
|
// Preserve the stored (sealed) password when the request sends "", so
|
||||||
// admin doesn't have to re-enter it on every save).
|
// the admin doesn't have to re-enter it on every save.
|
||||||
password := req.Password
|
password := current.Password
|
||||||
if password == "" {
|
if req.Password != "" {
|
||||||
password = current.Password
|
password, err = secret.Seal(secret.SMTPPassword.Purpose(), "", req.Password)
|
||||||
|
if err != nil {
|
||||||
|
writeErrWithLog(w, h.logger, "admin smtp: seal password", apierror.Internal(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err := q.UpdateSMTPConfig(r.Context(), dbq.UpdateSMTPConfigParams{
|
if err := q.UpdateSMTPConfig(r.Context(), dbq.UpdateSMTPConfigParams{
|
||||||
Enabled: req.Enabled,
|
Enabled: req.Enabled,
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
// LBStatus is the GET response body and the PUT response body. Token is
|
// LBStatus is the GET response body and the PUT response body. Token is
|
||||||
@@ -53,10 +54,11 @@ func (h *handlers) handlePutListenBrainz(w http.ResponseWriter, r *http.Request)
|
|||||||
|
|
||||||
q := dbq.New(h.pool)
|
q := dbq.New(h.pool)
|
||||||
if body.Token != nil {
|
if body.Token != nil {
|
||||||
t := *body.Token
|
tokenPtr, err := secret.SealPtr(secret.ListenBrainzToken.Purpose(), user.ID.String(), body.Token)
|
||||||
var tokenPtr *string
|
if err != nil {
|
||||||
if t != "" {
|
h.logger.Error("api: seal listenbrainz token", "err", err)
|
||||||
tokenPtr = &t
|
writeErr(w, apierror.InternalMsg("update failed", err))
|
||||||
|
return
|
||||||
}
|
}
|
||||||
if err := q.SetListenBrainzToken(r.Context(), dbq.SetListenBrainzTokenParams{
|
if err := q.SetListenBrainzToken(r.Context(), dbq.SetListenBrainzTokenParams{
|
||||||
ID: user.ID, ListenbrainzToken: tokenPtr,
|
ID: user.ID, ListenbrainzToken: tokenPtr,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The Subsonic password is for clients that sign in with Subsonic's t/s
|
// The Subsonic password is for clients that sign in with Subsonic's t/s
|
||||||
@@ -51,9 +52,15 @@ func (h *handlers) handleGenerateMySubsonicPassword(w http.ResponseWriter, r *ht
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
pw := base64.RawURLEncoding.EncodeToString(b)
|
pw := base64.RawURLEncoding.EncodeToString(b)
|
||||||
|
sealed, err := secret.Seal(secret.SubsonicPassword.Purpose(), user.ID.String(), pw)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.Error("generate subsonic password: seal failed", "err", err)
|
||||||
|
writeErr(w, apierror.Internal(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
if err := dbq.New(h.pool).SetSubsonicPassword(r.Context(), dbq.SetSubsonicPasswordParams{
|
if err := dbq.New(h.pool).SetSubsonicPassword(r.Context(), dbq.SetSubsonicPasswordParams{
|
||||||
ID: user.ID,
|
ID: user.ID,
|
||||||
SubsonicPassword: &pw,
|
SubsonicPassword: &sealed,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
h.logger.Error("generate subsonic password: update failed", "err", err)
|
h.logger.Error("generate subsonic password: update failed", "err", err)
|
||||||
writeErr(w, apierror.Internal(err))
|
writeErr(w, apierror.Internal(err))
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
func newMeSubsonicPasswordRouter(h *handlers) chi.Router {
|
func newMeSubsonicPasswordRouter(h *handlers) chi.Router {
|
||||||
@@ -21,6 +22,8 @@ func newMeSubsonicPasswordRouter(h *handlers) chi.Router {
|
|||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// readSubsonicPassword returns the stored Subsonic password opened. It fails
|
||||||
|
// the test if the column holds the password in the clear.
|
||||||
func readSubsonicPassword(t *testing.T, h *handlers, user dbq.User) *string {
|
func readSubsonicPassword(t *testing.T, h *handlers, user dbq.User) *string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
var pw *string
|
var pw *string
|
||||||
@@ -28,7 +31,17 @@ func readSubsonicPassword(t *testing.T, h *handlers, user dbq.User) *string {
|
|||||||
"SELECT subsonic_password FROM users WHERE id = $1", user.ID).Scan(&pw); err != nil {
|
"SELECT subsonic_password FROM users WHERE id = $1", user.ID).Scan(&pw); err != nil {
|
||||||
t.Fatalf("read subsonic_password: %v", err)
|
t.Fatalf("read subsonic_password: %v", err)
|
||||||
}
|
}
|
||||||
return pw
|
if pw == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !secret.IsSealed(*pw) {
|
||||||
|
t.Fatalf("subsonic_password is stored in the clear")
|
||||||
|
}
|
||||||
|
plain, err := secret.Open(secret.SubsonicPassword.Purpose(), user.ID.String(), *pw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open subsonic_password: %v", err)
|
||||||
|
}
|
||||||
|
return &plain
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubsonicPassword_GenerateIsRandomAndNotTheLoginPassword(t *testing.T) {
|
func TestSubsonicPassword_GenerateIsRandomAndNotTheLoginPassword(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,238 @@
|
|||||||
|
package backup_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newPool(t *testing.T) *pgxpool.Pool {
|
||||||
|
t.Helper()
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping integration test in -short mode")
|
||||||
|
}
|
||||||
|
dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
|
||||||
|
if dsn == "" {
|
||||||
|
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
|
||||||
|
}
|
||||||
|
if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil {
|
||||||
|
t.Fatalf("migrate: %v", err)
|
||||||
|
}
|
||||||
|
pool, err := pgxpool.New(context.Background(), dsn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("pool: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(pool.Close)
|
||||||
|
dbtest.ResetDB(t, pool)
|
||||||
|
return pool
|
||||||
|
}
|
||||||
|
|
||||||
|
func index(name string) int {
|
||||||
|
return slices.IndexFunc(backup.Tables, func(t backup.Table) bool { return t.Name == name })
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every table in the schema is backed up or excluded with a reason, so a new
|
||||||
|
// table fails here until someone decides which.
|
||||||
|
func TestEveryTableIsAccountedFor_Integration(t *testing.T) {
|
||||||
|
pool := newPool(t)
|
||||||
|
rows, err := pool.Query(context.Background(), "SELECT tablename FROM pg_tables WHERE schemaname = 'public'")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
schema, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, p := range backup.Coverage(schema) {
|
||||||
|
t.Error(p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A restore loads each table after the ones it references, and never needs a
|
||||||
|
// table the backup leaves out.
|
||||||
|
func TestTablesComeAfterWhatTheyReference_Integration(t *testing.T) {
|
||||||
|
pool := newPool(t)
|
||||||
|
rows, err := pool.Query(context.Background(), `
|
||||||
|
SELECT cl.relname, pc.relname
|
||||||
|
FROM pg_constraint co
|
||||||
|
JOIN pg_class cl ON cl.oid = co.conrelid
|
||||||
|
JOIN pg_class pc ON pc.oid = co.confrelid
|
||||||
|
JOIN pg_namespace n ON n.oid = cl.relnamespace
|
||||||
|
WHERE co.contype = 'f' AND n.nspname = 'public'`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
type fk struct{ child, parent string }
|
||||||
|
fks, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (fk, error) {
|
||||||
|
var f fk
|
||||||
|
err := r.Scan(&f.child, &f.parent)
|
||||||
|
return f, err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(fks) == 0 {
|
||||||
|
t.Fatal("found no foreign keys; the query is wrong")
|
||||||
|
}
|
||||||
|
for _, f := range fks {
|
||||||
|
child, parent := index(f.child), index(f.parent)
|
||||||
|
if child < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case parent < 0:
|
||||||
|
t.Errorf("%s is backed up but references %s, which is excluded", f.child, f.parent)
|
||||||
|
case parent >= child && f.parent != f.child:
|
||||||
|
t.Errorf("%s references %s, so it must come after it in Tables", f.child, f.parent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every omitted column exists, so a renamed column fails here rather than a
|
||||||
|
// sealed credential slipping into the file under its new name.
|
||||||
|
func TestOmittedColumnsExist_Integration(t *testing.T) {
|
||||||
|
pool := newPool(t)
|
||||||
|
for _, tbl := range backup.Tables {
|
||||||
|
for _, col := range tbl.Omit {
|
||||||
|
var n int
|
||||||
|
if err := pool.QueryRow(context.Background(), `
|
||||||
|
SELECT count(*) FROM information_schema.columns
|
||||||
|
WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2`,
|
||||||
|
tbl.Name, col).Scan(&n); err != nil || n != 1 {
|
||||||
|
t.Errorf("%s.%s: found %d columns, err %v", tbl.Name, col, n, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustExec(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := pool.Exec(context.Background(), sql, args...); err != nil {
|
||||||
|
t.Fatalf("%s: %v", sql, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustID(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) string {
|
||||||
|
t.Helper()
|
||||||
|
var id string
|
||||||
|
if err := pool.QueryRow(context.Background(), sql, args...).Scan(&id); err != nil {
|
||||||
|
t.Fatalf("%s: %v", sql, err)
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
// A backup restored into an emptied instance gives back the same rows, byte
|
||||||
|
// for byte, without the credentials and markers it leaves out.
|
||||||
|
func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) {
|
||||||
|
pool := newPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
user := mustID(t, pool, `INSERT INTO users (username, password_hash, api_token_hash, listenbrainz_token)
|
||||||
|
VALUES ($1, 'hash', 'backup-token', 'lb-token') RETURNING id::text`, dbtest.TestUserPrefix+"backup")
|
||||||
|
artist := mustID(t, pool, `INSERT INTO artists (name, sort_name, artist_art_source) VALUES ('Artist', 'artist', 'none') RETURNING id::text`)
|
||||||
|
album := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id, cover_art_source, cover_art_path)
|
||||||
|
VALUES ('Album', 'album', $1, 'mbcaa', '/data/covers/a.jpg') RETURNING id::text`, artist)
|
||||||
|
track := mustID(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format, tag_source)
|
||||||
|
VALUES ('Track', $1, $2, 180000, '/music/a.flac', 1000, 'flac', 'musicbrainz') RETURNING id::text`, album, artist)
|
||||||
|
session := mustID(t, pool, `INSERT INTO play_sessions (user_id, started_at, last_event_at) VALUES ($1, now(), now()) RETURNING id::text`, user)
|
||||||
|
mustExec(t, pool, `INSERT INTO play_events (user_id, track_id, session_id, started_at) VALUES ($1, $2, $3, now())`, user, track, session)
|
||||||
|
mustExec(t, pool, `INSERT INTO general_likes (user_id, track_id) VALUES ($1, $2)`, user, track)
|
||||||
|
playlist := mustID(t, pool, `INSERT INTO playlists (user_id, name) VALUES ($1, 'Mine') RETURNING id::text`, user)
|
||||||
|
mustExec(t, pool, `INSERT INTO playlist_tracks (playlist_id, position, track_id, title, artist_name, album_title, duration_sec)
|
||||||
|
VALUES ($1, 0, $2, 'Track', 'Artist', 'Album', 180)`, playlist, track)
|
||||||
|
mustExec(t, pool, `INSERT INTO follows (artist_mbid, artist_name, profile_id, followed_by)
|
||||||
|
SELECT 'mbid-1', 'Artist', id, $1 FROM release_type_profiles WHERE is_default`, user)
|
||||||
|
mustExec(t, pool, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert')`, track)
|
||||||
|
|
||||||
|
first, err := backup.Snapshot(ctx, pool)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("snapshot: %v", err)
|
||||||
|
}
|
||||||
|
for _, tbl := range backup.Tables {
|
||||||
|
for _, line := range first.Tables[tbl.Name] {
|
||||||
|
var row map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(line, &row); err != nil {
|
||||||
|
t.Fatalf("%s: %v", tbl.Name, err)
|
||||||
|
}
|
||||||
|
for _, col := range tbl.Omit {
|
||||||
|
if _, ok := row[col]; ok {
|
||||||
|
t.Errorf("%s.%s is in the backup", tbl.Name, col)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, tbl := range []string{"users", "tracks", "play_events", "general_likes", "playlist_tracks", "follows", "library_changes"} {
|
||||||
|
if len(first.Tables[tbl]) == 0 {
|
||||||
|
t.Errorf("the backup holds no %s", tbl)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := backup.Restore(ctx, pool, first); !errors.Is(err, backup.ErrNotEmpty) {
|
||||||
|
t.Fatalf("restore over data: err = %v, want ErrNotEmpty", err)
|
||||||
|
}
|
||||||
|
newer := first
|
||||||
|
newer.Schema++
|
||||||
|
if err := backup.Restore(ctx, pool, newer); !errors.Is(err, backup.ErrNewerSchema) {
|
||||||
|
t.Fatalf("restore from a newer schema: err = %v, want ErrNewerSchema", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mustExec(t, pool, "TRUNCATE users, artists, library_changes CASCADE")
|
||||||
|
if err := backup.Restore(ctx, pool, first); err != nil {
|
||||||
|
t.Fatalf("restore: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
second, err := backup.Snapshot(ctx, pool)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("snapshot after restore: %v", err)
|
||||||
|
}
|
||||||
|
for _, tbl := range backup.Tables {
|
||||||
|
a, b := first.Tables[tbl.Name], second.Tables[tbl.Name]
|
||||||
|
if len(a) != len(b) {
|
||||||
|
t.Errorf("%s: %d rows restored, the backup held %d", tbl.Name, len(b), len(a))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for i := range a {
|
||||||
|
if !bytes.Equal(a[i], b[i]) {
|
||||||
|
t.Errorf("%s row %d:\nbacked up %s\nrestored %s", tbl.Name, i+1, a[i], b[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the backup leaves out comes back unset, so the server asks for the
|
||||||
|
// credential again and redoes the art and tags.
|
||||||
|
var lbToken, coverSource, tagSource *string
|
||||||
|
if err := pool.QueryRow(ctx, `SELECT u.listenbrainz_token, al.cover_art_source, tr.tag_source
|
||||||
|
FROM users u, albums al, tracks tr WHERE u.id = $1 AND al.id = $2 AND tr.id = $3`,
|
||||||
|
user, album, track).Scan(&lbToken, &coverSource, &tagSource); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if lbToken != nil || coverSource != nil || tagSource != nil {
|
||||||
|
t.Errorf("token %v, cover source %v, tag source %v; want all unset", lbToken, coverSource, tagSource)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The change feed carries on after the restored ids.
|
||||||
|
var restoredMax, next int64
|
||||||
|
if err := pool.QueryRow(ctx, "SELECT max(id) FROM library_changes").Scan(&restoredMax); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op)
|
||||||
|
VALUES ('track', $1, 'upsert') RETURNING id`, track).Scan(&next); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if next <= restoredMax {
|
||||||
|
t.Errorf("next change id %d, want it after the restored %d", next, restoredMax)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
package backup
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// ErrNotEmpty is a restore into an instance that already has accounts or
|
||||||
|
// a library.
|
||||||
|
ErrNotEmpty = errors.New("backup: this Minstrel already has accounts or a library; a backup restores into an empty one")
|
||||||
|
|
||||||
|
// ErrNewerSchema is a backup written by a newer Minstrel than this one.
|
||||||
|
ErrNewerSchema = errors.New("backup: the backup was written by a newer version of Minstrel; upgrade this one first")
|
||||||
|
|
||||||
|
// ErrNotABackup is contents that aren't rows a backup writes.
|
||||||
|
ErrNotABackup = errors.New("backup: not a Minstrel backup")
|
||||||
|
)
|
||||||
|
|
||||||
|
// batchRows is how many rows one INSERT carries.
|
||||||
|
const batchRows = 500
|
||||||
|
|
||||||
|
// Restore loads a backup into an empty instance, in one transaction: either
|
||||||
|
// all of it lands or none of it does.
|
||||||
|
//
|
||||||
|
// The instance must have no accounts and no tracks. Its other rows in the
|
||||||
|
// tables the backup holds are replaced, which is what a fresh instance's
|
||||||
|
// seeded settings need. A table the backup doesn't hold, because it was
|
||||||
|
// written before that table existed, keeps what the instance has.
|
||||||
|
//
|
||||||
|
// A row's columns are those the backup and the table both have: an older
|
||||||
|
// backup's rows take the defaults of columns added since, and a column dropped
|
||||||
|
// since is left out.
|
||||||
|
func Restore(ctx context.Context, pool *pgxpool.Pool, c Contents) error {
|
||||||
|
tx, err := pool.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("backup: begin: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback(ctx) }()
|
||||||
|
|
||||||
|
schema, err := schemaVersion(ctx, tx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if c.Schema > schema {
|
||||||
|
return ErrNewerSchema
|
||||||
|
}
|
||||||
|
// Taken before the emptiness check, so a first account or a scan starting
|
||||||
|
// at the same moment waits for the restore, then finds the data.
|
||||||
|
if _, err := tx.Exec(ctx, "LOCK TABLE users, tracks IN EXCLUSIVE MODE"); err != nil {
|
||||||
|
return fmt.Errorf("backup: lock: %w", err)
|
||||||
|
}
|
||||||
|
var occupied bool
|
||||||
|
if err := tx.QueryRow(ctx,
|
||||||
|
"SELECT EXISTS (SELECT 1 FROM users) OR EXISTS (SELECT 1 FROM tracks)").Scan(&occupied); err != nil {
|
||||||
|
return fmt.Errorf("backup: check the instance is empty: %w", err)
|
||||||
|
}
|
||||||
|
if occupied {
|
||||||
|
return ErrNotEmpty
|
||||||
|
}
|
||||||
|
|
||||||
|
// Children first, so no row is left pointing at one already gone.
|
||||||
|
for _, t := range slices.Backward(Tables) {
|
||||||
|
if _, ok := c.Tables[t.Name]; !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, "DELETE FROM "+pgx.Identifier{t.Name}.Sanitize()); err != nil {
|
||||||
|
return fmt.Errorf("backup: clear %s: %w", t.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, t := range Tables {
|
||||||
|
rows, ok := c.Tables[t.Name]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := restoreTable(ctx, tx, t.Name, rows); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return fmt.Errorf("backup: commit: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type column struct {
|
||||||
|
name string
|
||||||
|
identity bool
|
||||||
|
sequence bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func restoreTable(ctx context.Context, tx pgx.Tx, table string, rows [][]byte) error {
|
||||||
|
cols, err := writableColumns(ctx, tx, table)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ident := pgx.Identifier{table}.Sanitize()
|
||||||
|
overriding := ""
|
||||||
|
if slices.ContainsFunc(cols, func(c column) bool { return c.identity }) {
|
||||||
|
overriding = " OVERRIDING SYSTEM VALUE"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rows go in batches that share one key set; a backup's rows for a table
|
||||||
|
// all do, since one read wrote them.
|
||||||
|
var (
|
||||||
|
batch []json.RawMessage
|
||||||
|
list string
|
||||||
|
)
|
||||||
|
flush := func() error {
|
||||||
|
if len(batch) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(batch)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sql := fmt.Sprintf("INSERT INTO %s (%s)%s SELECT %s FROM jsonb_populate_recordset(NULL::%s, $1::jsonb)",
|
||||||
|
ident, list, overriding, list, ident)
|
||||||
|
if _, err := tx.Exec(ctx, sql, string(payload)); err != nil {
|
||||||
|
return fmt.Errorf("backup: restore %s: %w", table, err)
|
||||||
|
}
|
||||||
|
batch = batch[:0]
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for i, raw := range rows {
|
||||||
|
var keys map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(raw, &keys); err != nil {
|
||||||
|
return fmt.Errorf("%w: row %d of %s is not an object", ErrNotABackup, i+1, table)
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, c := range cols {
|
||||||
|
if _, ok := keys[c.name]; ok {
|
||||||
|
names = append(names, pgx.Identifier{c.name}.Sanitize())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(names) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
slices.Sort(names)
|
||||||
|
rowList := strings.Join(names, ", ")
|
||||||
|
if rowList != list || len(batch) == batchRows {
|
||||||
|
if err := flush(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
list = rowList
|
||||||
|
}
|
||||||
|
batch = append(batch, raw)
|
||||||
|
}
|
||||||
|
if err := flush(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// A sequence carries on after the highest restored value, so the next row
|
||||||
|
// doesn't collide with one from the backup.
|
||||||
|
for _, c := range cols {
|
||||||
|
if !c.sequence {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
col := pgx.Identifier{c.name}.Sanitize()
|
||||||
|
if _, err := tx.Exec(ctx, fmt.Sprintf(
|
||||||
|
"SELECT setval(pg_get_serial_sequence($1, $2), COALESCE((SELECT max(%s) FROM %s), 0) + 1, false)", col, ident),
|
||||||
|
table, c.name); err != nil {
|
||||||
|
return fmt.Errorf("backup: carry on %s's sequence: %w", table, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// writableColumns returns the columns of a table an INSERT can set: all but
|
||||||
|
// generated ones, which Postgres computes.
|
||||||
|
func writableColumns(ctx context.Context, tx pgx.Tx, table string) ([]column, error) {
|
||||||
|
rows, err := tx.Query(ctx, `
|
||||||
|
SELECT column_name, is_identity = 'YES', pg_get_serial_sequence($1::text, column_name) IS NOT NULL
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = current_schema() AND table_name = $1::text AND is_generated = 'NEVER'
|
||||||
|
ORDER BY ordinal_position`, table)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("backup: read %s's columns: %w", table, err)
|
||||||
|
}
|
||||||
|
cols, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (column, error) {
|
||||||
|
var c column
|
||||||
|
err := row.Scan(&c.name, &c.identity, &c.sequence)
|
||||||
|
return c, err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("backup: read %s's columns: %w", table, err)
|
||||||
|
}
|
||||||
|
return cols, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
package backup
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Contents is a backup's data: the schema version that wrote it, and each
|
||||||
|
// table's rows as JSON objects with their keys sorted, in primary-key order.
|
||||||
|
// An unchanged row is the same bytes in every backup.
|
||||||
|
type Contents struct {
|
||||||
|
Schema int64
|
||||||
|
Tables map[string][][]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot reads every table in Tables as of one moment.
|
||||||
|
func Snapshot(ctx context.Context, pool *pgxpool.Pool) (Contents, error) {
|
||||||
|
c := Contents{Tables: make(map[string][][]byte, len(Tables))}
|
||||||
|
tx, err := pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly})
|
||||||
|
if err != nil {
|
||||||
|
return c, fmt.Errorf("backup: begin: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback(ctx) }()
|
||||||
|
|
||||||
|
if c.Schema, err = schemaVersion(ctx, tx); err != nil {
|
||||||
|
return c, err
|
||||||
|
}
|
||||||
|
for _, t := range Tables {
|
||||||
|
lines, err := readTable(ctx, tx, t)
|
||||||
|
if err != nil {
|
||||||
|
return c, err
|
||||||
|
}
|
||||||
|
c.Tables[t.Name] = lines
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readTable(ctx context.Context, tx pgx.Tx, t Table) ([][]byte, error) {
|
||||||
|
pk, err := primaryKey(ctx, tx, t.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
order := make([]string, len(pk))
|
||||||
|
for i, col := range pk {
|
||||||
|
order[i] = "t." + pgx.Identifier{col}.Sanitize()
|
||||||
|
}
|
||||||
|
omit := t.Omit
|
||||||
|
if omit == nil {
|
||||||
|
omit = []string{}
|
||||||
|
}
|
||||||
|
rows, err := tx.Query(ctx, fmt.Sprintf("SELECT to_jsonb(t) - $1::text[] FROM %s t ORDER BY %s",
|
||||||
|
pgx.Identifier{t.Name}.Sanitize(), strings.Join(order, ", ")), omit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("backup: read %s: %w", t.Name, err)
|
||||||
|
}
|
||||||
|
lines, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) ([]byte, error) {
|
||||||
|
var raw []byte
|
||||||
|
if err := row.Scan(&raw); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return canonical(raw)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("backup: read %s: %w", t.Name, err)
|
||||||
|
}
|
||||||
|
return lines, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// canonical re-encodes a row with its keys sorted. Numbers pass through as
|
||||||
|
// written, so a bigint or a float stays exact.
|
||||||
|
func canonical(raw []byte) ([]byte, error) {
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
dec.UseNumber()
|
||||||
|
var row map[string]any
|
||||||
|
if err := dec.Decode(&row); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return json.Marshal(row)
|
||||||
|
}
|
||||||
|
|
||||||
|
// primaryKey returns a table's primary-key columns in key order.
|
||||||
|
func primaryKey(ctx context.Context, tx pgx.Tx, table string) ([]string, error) {
|
||||||
|
rows, err := tx.Query(ctx, `
|
||||||
|
SELECT a.attname
|
||||||
|
FROM pg_index i
|
||||||
|
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY (i.indkey)
|
||||||
|
WHERE i.indrelid = to_regclass($1) AND i.indisprimary
|
||||||
|
ORDER BY array_position(i.indkey, a.attnum)`, table)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("backup: read %s's primary key: %w", table, err)
|
||||||
|
}
|
||||||
|
cols, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("backup: read %s's primary key: %w", table, err)
|
||||||
|
}
|
||||||
|
if len(cols) == 0 {
|
||||||
|
return nil, fmt.Errorf("backup: %s has no primary key to order its rows by", table)
|
||||||
|
}
|
||||||
|
return cols, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func schemaVersion(ctx context.Context, tx pgx.Tx) (int64, error) {
|
||||||
|
var v int64
|
||||||
|
if err := tx.QueryRow(ctx, "SELECT version FROM schema_migrations").Scan(&v); err != nil {
|
||||||
|
return 0, fmt.Errorf("backup: read the schema version: %w", err)
|
||||||
|
}
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,193 @@
|
|||||||
|
// Package backup reads Minstrel's irreplaceable data out of Postgres as JSON
|
||||||
|
// Lines, one table at a time, and loads it back into an empty instance.
|
||||||
|
//
|
||||||
|
// What goes in is everything a person made or decided: accounts, likes, play
|
||||||
|
// and skip history, playlists, follows, wants and requests, download history,
|
||||||
|
// admin decisions, and settings. What stays out is everything the restored
|
||||||
|
// server rebuilds by itself: what it computes from the music files, what it
|
||||||
|
// fetches again from MusicBrainz, ListenBrainz and the other providers, and
|
||||||
|
// logs of past runs. Excluded says why for each, and the coverage test fails
|
||||||
|
// on a table that is in neither list.
|
||||||
|
package backup
|
||||||
|
|
||||||
|
// Table is one table a backup holds.
|
||||||
|
//
|
||||||
|
// Omit names columns left out of the file. They restore to their defaults:
|
||||||
|
// - a credential Minstrel stores sealed, which only the server holding the
|
||||||
|
// stream secret could open, so the restored server asks for it again;
|
||||||
|
// - a marker saying work on an excluded table is done (a tag source, an art
|
||||||
|
// source), so the restored server does that work again rather than
|
||||||
|
// believing it has the result.
|
||||||
|
type Table struct {
|
||||||
|
Name string
|
||||||
|
Omit []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tables is every table a backup holds, in the order a restore loads them:
|
||||||
|
// each after every table it references. There are no self-referencing
|
||||||
|
// tables, so no table needs its rows loaded in depth order.
|
||||||
|
var Tables = []Table{
|
||||||
|
{Name: "users", Omit: []string{"subsonic_password", "listenbrainz_token"}},
|
||||||
|
|
||||||
|
// The library's identity. Every like, play and playlist entry hangs off a
|
||||||
|
// track id, so the rows travel even though a scan could rebuild most of
|
||||||
|
// their columns; the scan after a restore finds each file at its path and
|
||||||
|
// keeps its id.
|
||||||
|
{Name: "artists", Omit: []string{"artist_thumb_path", "artist_fanart_path", "artist_art_source", "artist_art_sources_version"}},
|
||||||
|
{Name: "albums", Omit: []string{"cover_art_path", "cover_art_source", "cover_art_sources_version"}},
|
||||||
|
{Name: "tracks", Omit: []string{"tag_source", "tag_sources_version"}},
|
||||||
|
|
||||||
|
// Settings.
|
||||||
|
{Name: "library_settings"},
|
||||||
|
{Name: "network_settings"},
|
||||||
|
{Name: "registration_settings"},
|
||||||
|
{Name: "smtp_config", Omit: []string{"password"}},
|
||||||
|
{Name: "notification_email_settings"},
|
||||||
|
{Name: "acoustid_settings", Omit: []string{"api_key"}},
|
||||||
|
{Name: "fingerprint_settings"},
|
||||||
|
{Name: "loudness_settings"},
|
||||||
|
{Name: "cover_art_provider_settings", Omit: []string{"api_key"}},
|
||||||
|
{Name: "cover_art_sources_meta"},
|
||||||
|
{Name: "tag_provider_settings", Omit: []string{"api_key"}},
|
||||||
|
{Name: "tag_sources_meta"},
|
||||||
|
{Name: "acquisition_services", Omit: []string{"api_key", "password"}},
|
||||||
|
{Name: "acquisition_settings"},
|
||||||
|
{Name: "reacquisition_settings"},
|
||||||
|
{Name: "discover_tuning"},
|
||||||
|
{Name: "taste_tuning"},
|
||||||
|
{Name: "recommendation_weight_profiles"},
|
||||||
|
{Name: "recommendation_tuning_audit"},
|
||||||
|
|
||||||
|
// Follows, wants, requests and downloads.
|
||||||
|
{Name: "release_type_profiles"},
|
||||||
|
{Name: "follows"},
|
||||||
|
{Name: "follow_seen_groups"},
|
||||||
|
{Name: "wants"},
|
||||||
|
{Name: "grabs"},
|
||||||
|
{Name: "download_interventions"},
|
||||||
|
{Name: "release_blocks"},
|
||||||
|
{Name: "requests"},
|
||||||
|
|
||||||
|
// Listening history and likes.
|
||||||
|
{Name: "play_sessions"},
|
||||||
|
{Name: "play_events"},
|
||||||
|
{Name: "skip_events"},
|
||||||
|
{Name: "scrobble_queue"},
|
||||||
|
{Name: "contextual_likes"},
|
||||||
|
{Name: "general_likes"},
|
||||||
|
{Name: "general_likes_albums"},
|
||||||
|
{Name: "general_likes_artists"},
|
||||||
|
|
||||||
|
// Playlists and what is built from history. The taste profile and the
|
||||||
|
// "you might like" rows are rebuilt nightly from the history above; they
|
||||||
|
// travel so a restored Home is full before that first nightly pass.
|
||||||
|
{Name: "playlists"},
|
||||||
|
{Name: "playlist_tracks"},
|
||||||
|
{Name: "system_playlist_runs"},
|
||||||
|
{Name: "system_playlist_rotation_state"},
|
||||||
|
{Name: "taste_profile_artists"},
|
||||||
|
{Name: "taste_profile_tags"},
|
||||||
|
{Name: "taste_profile_eras"},
|
||||||
|
{Name: "taste_profile_moods"},
|
||||||
|
{Name: "you_might_like_albums"},
|
||||||
|
{Name: "you_might_like_artists"},
|
||||||
|
|
||||||
|
// Per-user preferences and notifications.
|
||||||
|
{Name: "user_normalization_prefs"},
|
||||||
|
{Name: "user_notification_prefs"},
|
||||||
|
{Name: "user_notification_email_state"},
|
||||||
|
{Name: "user_notifications"},
|
||||||
|
{Name: "suggestion_snoozes"},
|
||||||
|
{Name: "quarantine"},
|
||||||
|
{Name: "quarantine_actions"},
|
||||||
|
|
||||||
|
// Admin decisions: an album taken off the wanted listing stays off, and a
|
||||||
|
// dismissed duplicate group stays dismissed when the sweep finds it again.
|
||||||
|
{Name: "library_album_matches"},
|
||||||
|
{Name: "duplicate_sweeps"},
|
||||||
|
{Name: "duplicate_groups"},
|
||||||
|
{Name: "duplicate_group_members"},
|
||||||
|
|
||||||
|
// The library change feed is how the apps sync, including a first sync,
|
||||||
|
// so a restored server carries it on from where the backup left it.
|
||||||
|
{Name: "library_changes"},
|
||||||
|
{Name: "audit_log"},
|
||||||
|
{Name: "playback_errors"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Excluded is every table a backup leaves out, with the reason.
|
||||||
|
var Excluded = map[string]string{
|
||||||
|
"schema_migrations": "the restored server's own migrations write it; the backup's schema version is in its manifest",
|
||||||
|
"sessions": "sign-ins; everyone signs in again on the restored server",
|
||||||
|
"password_resets": "one-time reset links, which expire within hours",
|
||||||
|
"user_invites": "open sign-up links; an admin sends an invite again from the restored server",
|
||||||
|
|
||||||
|
"track_fingerprints": "computed from the music files",
|
||||||
|
"track_loudness": "computed from the music files",
|
||||||
|
"album_loudness": "computed from the music files",
|
||||||
|
"track_acoustid_lookups": "looked up again from each track's fingerprint",
|
||||||
|
"track_tags": "fetched again from the tag providers",
|
||||||
|
"candidate_artist_tags": "fetched again from the tag providers",
|
||||||
|
"candidate_artist_tag_state": "fetched again from the tag providers",
|
||||||
|
|
||||||
|
"mb_artists": "MusicBrainz cache, fetched again",
|
||||||
|
"mb_release_groups": "MusicBrainz cache, fetched again",
|
||||||
|
"mb_releases": "MusicBrainz cache, fetched again",
|
||||||
|
"mb_release_tracks": "MusicBrainz cache, fetched again",
|
||||||
|
"mb_artist_release_groups": "MusicBrainz cache, fetched again",
|
||||||
|
|
||||||
|
"listenbrainz_similar_recordings": "similarity, fetched again from ListenBrainz",
|
||||||
|
"track_similarity": "similarity, fetched again from the similarity providers",
|
||||||
|
"track_similarity_fetches": "similarity, fetched again from the similarity providers",
|
||||||
|
"artist_similarity": "similarity, fetched again from the similarity providers",
|
||||||
|
"artist_similarity_fetches": "similarity, fetched again from the similarity providers",
|
||||||
|
"artist_similarity_unmatched": "similarity, fetched again from the similarity providers",
|
||||||
|
|
||||||
|
"want_candidates": "the last search's results; the next search finds them again",
|
||||||
|
"scan_runs": "a log of past library scans",
|
||||||
|
"diagnostic_events": "debug-mode diagnostics",
|
||||||
|
}
|
||||||
|
|
||||||
|
// carried reports whether a table is in Tables.
|
||||||
|
func carried(name string) bool {
|
||||||
|
for _, t := range Tables {
|
||||||
|
if t.Name == name {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Coverage checks a schema's tables against Tables and Excluded. It returns
|
||||||
|
// one line per problem: a table in neither list, in both, or listed twice,
|
||||||
|
// and a listed table the schema doesn't have.
|
||||||
|
func Coverage(schema []string) []string {
|
||||||
|
var problems []string
|
||||||
|
inSchema := map[string]bool{}
|
||||||
|
for _, name := range schema {
|
||||||
|
inSchema[name] = true
|
||||||
|
_, excluded := Excluded[name]
|
||||||
|
switch {
|
||||||
|
case carried(name) && excluded:
|
||||||
|
problems = append(problems, name+": both backed up and excluded")
|
||||||
|
case !carried(name) && !excluded:
|
||||||
|
problems = append(problems, name+": neither backed up nor excluded; add it to Tables, or to Excluded with the reason")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, t := range Tables {
|
||||||
|
if seen[t.Name] {
|
||||||
|
problems = append(problems, t.Name+": listed twice in Tables")
|
||||||
|
}
|
||||||
|
seen[t.Name] = true
|
||||||
|
if !inSchema[t.Name] {
|
||||||
|
problems = append(problems, t.Name+": in Tables but not in the schema")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for name := range Excluded {
|
||||||
|
if !inSchema[name] {
|
||||||
|
problems = append(problems, name+": in Excluded but not in the schema")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return problems
|
||||||
|
}
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
package backup
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func everyListedTable() []string {
|
||||||
|
var names []string
|
||||||
|
for _, t := range Tables {
|
||||||
|
names = append(names, t.Name)
|
||||||
|
}
|
||||||
|
for name := range Excluded {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoverageAcceptsASchemaThatMatchesTheLists(t *testing.T) {
|
||||||
|
if problems := Coverage(everyListedTable()); len(problems) != 0 {
|
||||||
|
t.Fatalf("problems = %v, want none", problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoverageNamesATableInNeitherList(t *testing.T) {
|
||||||
|
problems := Coverage(append(everyListedTable(), "new_table"))
|
||||||
|
if len(problems) != 1 || !strings.HasPrefix(problems[0], "new_table: neither") {
|
||||||
|
t.Fatalf("problems = %v, want one naming new_table", problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoverageNamesAListedTableTheSchemaLacks(t *testing.T) {
|
||||||
|
schema := slices.DeleteFunc(everyListedTable(), func(n string) bool { return n == "playlists" || n == "scan_runs" })
|
||||||
|
problems := Coverage(schema)
|
||||||
|
if len(problems) != 2 {
|
||||||
|
t.Fatalf("problems = %v, want playlists and scan_runs named", problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEveryExclusionHasAReason(t *testing.T) {
|
||||||
|
for name, why := range Excluded {
|
||||||
|
if strings.TrimSpace(why) == "" {
|
||||||
|
t.Errorf("%s is excluded without a reason", name)
|
||||||
|
}
|
||||||
|
if carried(name) {
|
||||||
|
t.Errorf("%s is both backed up and excluded", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SettingsService reconciles the registry of compiled-in providers
|
// SettingsService reconciles the registry of compiled-in providers
|
||||||
@@ -112,7 +113,7 @@ func (s *SettingsService) reconcile(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
key := ""
|
key := ""
|
||||||
if row.ApiKey != nil {
|
if row.ApiKey != nil {
|
||||||
key = *row.ApiKey
|
key = secret.Reveal(secret.CoverArtAPIKey.Purpose(), row.ProviderID, *row.ApiKey)
|
||||||
}
|
}
|
||||||
s.enabledIDs[p.ID()] = row.Enabled
|
s.enabledIDs[p.ID()] = row.Enabled
|
||||||
s.apiKeys[p.ID()] = key
|
s.apiKeys[p.ID()] = key
|
||||||
@@ -204,11 +205,15 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
|
|||||||
// (apiKeyChanged flag) and $4 (api_key value). The actual generated
|
// (apiKeyChanged flag) and $4 (api_key value). The actual generated
|
||||||
// field for $4 is ApiKey *string.
|
// field for $4 is ApiKey *string.
|
||||||
apiKeyChanged := patch.APIKey != nil
|
apiKeyChanged := patch.APIKey != nil
|
||||||
|
sealedKey, err := secret.SealPtr(secret.CoverArtAPIKey.Purpose(), providerID, patch.APIKey)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("seal provider api key: %w", err)
|
||||||
|
}
|
||||||
if err := q.UpdateProviderSettings(ctx, dbq.UpdateProviderSettingsParams{
|
if err := q.UpdateProviderSettings(ctx, dbq.UpdateProviderSettingsParams{
|
||||||
ProviderID: providerID,
|
ProviderID: providerID,
|
||||||
Enabled: enabledVal,
|
Enabled: enabledVal,
|
||||||
Column3: apiKeyChanged,
|
Column3: apiKeyChanged,
|
||||||
ApiKey: patch.APIKey,
|
ApiKey: sealedKey,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return false, fmt.Errorf("update provider settings: %w", err)
|
return false, fmt.Errorf("update provider settings: %w", err)
|
||||||
}
|
}
|
||||||
@@ -225,7 +230,7 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
|
|||||||
for _, r := range rows {
|
for _, r := range rows {
|
||||||
key := ""
|
key := ""
|
||||||
if r.ApiKey != nil {
|
if r.ApiKey != nil {
|
||||||
key = *r.ApiKey
|
key = secret.Reveal(secret.CoverArtAPIKey.Purpose(), r.ProviderID, *r.ApiKey)
|
||||||
}
|
}
|
||||||
s.enabledIDs[r.ProviderID] = r.Enabled
|
s.enabledIDs[r.ProviderID] = r.Enabled
|
||||||
s.apiKeys[r.ProviderID] = key
|
s.apiKeys[r.ProviderID] = key
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
package dbtest
|
||||||
|
|
||||||
|
import "git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
|
|
||||||
|
// Integration tests run with a master key installed, as the server runs
|
||||||
|
// after startup, so code that seals or opens a stored credential behaves as
|
||||||
|
// it does in production.
|
||||||
|
func init() {
|
||||||
|
secret.Install([]byte("minstrel integration tests: not a deployment key"))
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
// AcoustID lookup settings: a database row, changed without a restart, shared
|
// AcoustID lookup settings: a database row, changed without a restart, shared
|
||||||
@@ -97,8 +98,12 @@ func (s *AcoustIDSettingsService) Set(ctx context.Context, in AcoustIDSettingsUp
|
|||||||
}
|
}
|
||||||
params := dbq.UpdateAcoustIDSettingsParams{Enabled: in.Enabled, MinScore: in.MinScore}
|
params := dbq.UpdateAcoustIDSettingsParams{Enabled: in.Enabled, MinScore: in.MinScore}
|
||||||
if in.APIKey != nil {
|
if in.APIKey != nil {
|
||||||
|
sealed, err := secret.Seal(secret.AcoustIDAPIKey.Purpose(), "", *in.APIKey)
|
||||||
|
if err != nil {
|
||||||
|
return AcoustIDSettings{}, fmt.Errorf("acoustid settings: seal key: %w", err)
|
||||||
|
}
|
||||||
params.SetApiKey = true
|
params.SetApiKey = true
|
||||||
params.ApiKey = *in.APIKey
|
params.ApiKey = sealed
|
||||||
}
|
}
|
||||||
row, err := dbq.New(s.pool).UpdateAcoustIDSettings(ctx, params)
|
row, err := dbq.New(s.pool).UpdateAcoustIDSettings(ctx, params)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -126,7 +131,7 @@ func (s *AcoustIDSettingsService) Changed() <-chan struct{} {
|
|||||||
func acoustIDSettingsFromRow(row dbq.AcoustidSetting) AcoustIDSettings {
|
func acoustIDSettingsFromRow(row dbq.AcoustidSetting) AcoustIDSettings {
|
||||||
out := AcoustIDSettings{Enabled: row.Enabled, MinScore: row.MinScore, UpdatedAt: row.UpdatedAt.Time}
|
out := AcoustIDSettings{Enabled: row.Enabled, MinScore: row.MinScore, UpdatedAt: row.UpdatedAt.Time}
|
||||||
if row.ApiKey != nil {
|
if row.ApiKey != nil {
|
||||||
out.APIKey = *row.ApiKey
|
out.APIKey = secret.Reveal(secret.AcoustIDAPIKey.Purpose(), "", *row.ApiKey)
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import (
|
|||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrNotConfigured is returned when smtp_config.enabled is false or
|
// ErrNotConfigured is returned when smtp_config.enabled is false or
|
||||||
@@ -68,7 +69,8 @@ func (s *SMTPSender) Send(ctx context.Context, to, subject, textBody, htmlBody s
|
|||||||
|
|
||||||
var auth smtp.Auth
|
var auth smtp.Auth
|
||||||
if cfg.Username != "" {
|
if cfg.Username != "" {
|
||||||
auth = smtp.PlainAuth("", cfg.Username, cfg.Password, cfg.Host)
|
password := secret.Reveal(secret.SMTPPassword.Purpose(), "", cfg.Password)
|
||||||
|
auth = smtp.PlainAuth("", cfg.Username, password, cfg.Host)
|
||||||
}
|
}
|
||||||
|
|
||||||
// We respect the use_tls flag: if true, we require STARTTLS via
|
// We respect the use_tls flag: if true, we require STARTTLS via
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
func testPool(t *testing.T) (*pgxpool.Pool, *dbq.Queries) {
|
func testPool(t *testing.T) (*pgxpool.Pool, *dbq.Queries) {
|
||||||
@@ -63,8 +64,13 @@ func seed(t *testing.T, opts seedOpts) setup {
|
|||||||
t.Fatalf("user: %v", err)
|
t.Fatalf("user: %v", err)
|
||||||
}
|
}
|
||||||
if opts.lbToken != "" {
|
if opts.lbToken != "" {
|
||||||
|
// Stored sealed, as the settings route stores it.
|
||||||
|
sealed, err := secret.Seal(secret.ListenBrainzToken.Purpose(), u.ID.String(), opts.lbToken)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("seal token: %v", err)
|
||||||
|
}
|
||||||
if err := q.SetListenBrainzToken(ctx, dbq.SetListenBrainzTokenParams{
|
if err := q.SetListenBrainzToken(ctx, dbq.SetListenBrainzTokenParams{
|
||||||
ID: u.ID, ListenbrainzToken: &opts.lbToken,
|
ID: u.ID, ListenbrainzToken: &sealed,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatalf("token: %v", err)
|
t.Fatalf("token: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble/listenbrainz"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble/listenbrainz"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
// backoffSchedule maps the failure count (1-indexed) to the delay before
|
// backoffSchedule maps the failure count (1-indexed) to the delay before
|
||||||
@@ -98,7 +99,11 @@ func (w *Worker) tickOnce(ctx context.Context) error {
|
|||||||
key := r.UserID.String()
|
key := r.UserID.String()
|
||||||
ub := batches[key]
|
ub := batches[key]
|
||||||
if ub == nil {
|
if ub == nil {
|
||||||
ub = &userBatch{token: *r.LbToken, userID: r.UserID}
|
token := secret.Reveal(secret.ListenBrainzToken.Purpose(), key, *r.LbToken)
|
||||||
|
if token == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ub = &userBatch{token: token, userID: r.UserID}
|
||||||
batches[key] = ub
|
batches[key] = ub
|
||||||
}
|
}
|
||||||
ub.queueIDs = append(ub.queueIDs, r.QueueID)
|
ub.queueIDs = append(ub.queueIDs, r.QueueID)
|
||||||
|
|||||||
@@ -0,0 +1,179 @@
|
|||||||
|
// Package secret seals the credentials Minstrel keeps for outside systems
|
||||||
|
// (indexer and download-client keys, the SMTP password, provider API keys,
|
||||||
|
// ListenBrainz tokens, Subsonic passwords) so the database holds ciphertext,
|
||||||
|
// not the credential.
|
||||||
|
//
|
||||||
|
// Each column has its own key, derived from the install's master key with
|
||||||
|
// HKDF-SHA256 and the column as the info string, so a value sealed for one
|
||||||
|
// column cannot be opened as another, and the master key never touches a
|
||||||
|
// cipher. Each value is also bound to the row it belongs to (its owner), so a
|
||||||
|
// sealed value copied onto another row does not open there.
|
||||||
|
//
|
||||||
|
// The master key is the stream secret (MINSTREL_STREAM_SECRET, or the
|
||||||
|
// stream_secret file in the data directory). It lives outside the database,
|
||||||
|
// so a copy of the database alone opens nothing. Changing it makes every
|
||||||
|
// sealed value unreadable: each one reads as unset and has to be entered
|
||||||
|
// again in Settings.
|
||||||
|
package secret
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/aes"
|
||||||
|
"crypto/cipher"
|
||||||
|
"crypto/hkdf"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrNoKey is a seal or open before Install has given the package a key.
|
||||||
|
var ErrNoKey = errors.New("secret: no master key installed")
|
||||||
|
|
||||||
|
// ErrCorrupt is a stored value that does not open: the master key changed,
|
||||||
|
// the value was edited by hand, or it belongs to a different row.
|
||||||
|
var ErrCorrupt = errors.New("secret: the stored value does not open")
|
||||||
|
|
||||||
|
// sealedPrefix versions the format, so a future change of cipher can still
|
||||||
|
// read what this one wrote. Nothing a person types starts with it by chance
|
||||||
|
// often enough to matter, and Open rejects anything without it.
|
||||||
|
const sealedPrefix = "v1."
|
||||||
|
|
||||||
|
var b64 = base64.RawURLEncoding
|
||||||
|
|
||||||
|
var (
|
||||||
|
mu sync.RWMutex
|
||||||
|
master []byte
|
||||||
|
boxes map[string]cipher.AEAD
|
||||||
|
)
|
||||||
|
|
||||||
|
// Install sets the master key every Seal and Open derives from. The server
|
||||||
|
// calls it once at startup, before anything reads a credential.
|
||||||
|
func Install(key []byte) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
master = append([]byte(nil), key...)
|
||||||
|
boxes = map[string]cipher.AEAD{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// aead returns purpose's cipher, deriving it on first use.
|
||||||
|
func aead(purpose string) (cipher.AEAD, error) {
|
||||||
|
mu.RLock()
|
||||||
|
a, ok := boxes[purpose]
|
||||||
|
key := master
|
||||||
|
mu.RUnlock()
|
||||||
|
if ok {
|
||||||
|
return a, nil
|
||||||
|
}
|
||||||
|
if len(key) == 0 {
|
||||||
|
return nil, ErrNoKey
|
||||||
|
}
|
||||||
|
derived, err := hkdf.Key(sha256.New, key, nil, "minstrel/"+purpose, 32)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("secret: derive: %w", err)
|
||||||
|
}
|
||||||
|
block, err := aes.NewCipher(derived)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("secret: %w", err)
|
||||||
|
}
|
||||||
|
a, err = cipher.NewGCM(block)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("secret: %w", err)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
if boxes != nil {
|
||||||
|
boxes[purpose] = a
|
||||||
|
}
|
||||||
|
mu.Unlock()
|
||||||
|
return a, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsSealed reports whether stored is in the sealed format.
|
||||||
|
func IsSealed(stored string) bool { return strings.HasPrefix(stored, sealedPrefix) }
|
||||||
|
|
||||||
|
// Seal encrypts plain for purpose (the column it is stored in) and owner (the
|
||||||
|
// row it belongs to). The empty string stays empty, so "is one set?" stays a
|
||||||
|
// test for the empty string.
|
||||||
|
func Seal(purpose, owner, plain string) (string, error) {
|
||||||
|
if plain == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
a, err := aead(purpose)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
nonce := make([]byte, a.NonceSize())
|
||||||
|
if _, err := rand.Read(nonce); err != nil {
|
||||||
|
return "", fmt.Errorf("secret: nonce: %w", err)
|
||||||
|
}
|
||||||
|
out := a.Seal(nonce, nonce, []byte(plain), []byte(owner))
|
||||||
|
return sealedPrefix + b64.EncodeToString(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open decrypts what Seal stored for the same purpose and owner. The empty
|
||||||
|
// string opens to itself.
|
||||||
|
func Open(purpose, owner, stored string) (string, error) {
|
||||||
|
if stored == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
a, err := aead(purpose)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
raw, ok := strings.CutPrefix(stored, sealedPrefix)
|
||||||
|
if !ok {
|
||||||
|
return "", ErrCorrupt
|
||||||
|
}
|
||||||
|
data, err := b64.DecodeString(raw)
|
||||||
|
n := a.NonceSize()
|
||||||
|
if err != nil || len(data) < n {
|
||||||
|
return "", ErrCorrupt
|
||||||
|
}
|
||||||
|
plain, err := a.Open(nil, data[:n], data[n:], []byte(owner))
|
||||||
|
if err != nil {
|
||||||
|
return "", ErrCorrupt
|
||||||
|
}
|
||||||
|
return string(plain), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reveal is Open for a caller about to use the credential: a value that does
|
||||||
|
// not open is logged and read as unset, which is what it has become. The
|
||||||
|
// admin sees "not set" in Settings and enters it again.
|
||||||
|
func Reveal(purpose, owner, stored string) string {
|
||||||
|
plain, err := Open(purpose, owner, stored)
|
||||||
|
if err != nil {
|
||||||
|
slog.Warn("secret: a stored credential does not open; it reads as unset until it is entered again",
|
||||||
|
"column", purpose, "err", err)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return plain
|
||||||
|
}
|
||||||
|
|
||||||
|
// RevealPtr is Reveal for a nullable column: nil stays nil, and a value that
|
||||||
|
// does not open reads as nil.
|
||||||
|
func RevealPtr(purpose, owner string, stored *string) *string {
|
||||||
|
if stored == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
plain := Reveal(purpose, owner, *stored)
|
||||||
|
if plain == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &plain
|
||||||
|
}
|
||||||
|
|
||||||
|
// SealPtr is Seal for a nullable column: nil and the empty string both store
|
||||||
|
// NULL.
|
||||||
|
func SealPtr(purpose, owner string, plain *string) (*string, error) {
|
||||||
|
if plain == nil || *plain == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
sealed, err := Seal(purpose, owner, *plain)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &sealed, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
package secret
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func withKey(t *testing.T, key string) {
|
||||||
|
t.Helper()
|
||||||
|
mu.RLock()
|
||||||
|
prev := master
|
||||||
|
mu.RUnlock()
|
||||||
|
Install([]byte(key))
|
||||||
|
t.Cleanup(func() { Install(prev) })
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSealOpensForTheSamePurposeAndOwner(t *testing.T) {
|
||||||
|
withKey(t, "test master key")
|
||||||
|
sealed, err := Seal("t.col", "row-1", "hunter2")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !IsSealed(sealed) || strings.Contains(sealed, "hunter2") {
|
||||||
|
t.Fatalf("sealed = %q, want an opaque v1. value", sealed)
|
||||||
|
}
|
||||||
|
plain, err := Open("t.col", "row-1", sealed)
|
||||||
|
if err != nil || plain != "hunter2" {
|
||||||
|
t.Fatalf("Open = %q, %v; want hunter2", plain, err)
|
||||||
|
}
|
||||||
|
again, _ := Seal("t.col", "row-1", "hunter2")
|
||||||
|
if again == sealed {
|
||||||
|
t.Error("two seals of one value are identical; the nonce is not random")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenRefusesAnythingElse(t *testing.T) {
|
||||||
|
withKey(t, "test master key")
|
||||||
|
sealed, _ := Seal("t.col", "row-1", "hunter2")
|
||||||
|
|
||||||
|
cases := map[string]func() (string, error){
|
||||||
|
"another owner": func() (string, error) { return Open("t.col", "row-2", sealed) },
|
||||||
|
"another purpose": func() (string, error) { return Open("t.other", "row-1", sealed) },
|
||||||
|
"plain text": func() (string, error) { return Open("t.col", "row-1", "hunter2") },
|
||||||
|
"truncated": func() (string, error) { return Open("t.col", "row-1", sealed[:len(sealed)-4]) },
|
||||||
|
"not base64": func() (string, error) { return Open("t.col", "row-1", "v1.!!!") },
|
||||||
|
"too short": func() (string, error) { return Open("t.col", "row-1", "v1.AAAA") },
|
||||||
|
}
|
||||||
|
for name, open := range cases {
|
||||||
|
if _, err := open(); !errors.Is(err, ErrCorrupt) {
|
||||||
|
t.Errorf("%s: err = %v, want ErrCorrupt", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Install([]byte("a different key"))
|
||||||
|
if _, err := Open("t.col", "row-1", sealed); !errors.Is(err, ErrCorrupt) {
|
||||||
|
t.Errorf("another master key: err = %v, want ErrCorrupt", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmptyStaysEmpty(t *testing.T) {
|
||||||
|
withKey(t, "test master key")
|
||||||
|
if s, err := Seal("t.col", "o", ""); s != "" || err != nil {
|
||||||
|
t.Errorf("Seal(\"\") = %q, %v; want the empty string", s, err)
|
||||||
|
}
|
||||||
|
if s, err := Open("t.col", "o", ""); s != "" || err != nil {
|
||||||
|
t.Errorf("Open(\"\") = %q, %v; want the empty string", s, err)
|
||||||
|
}
|
||||||
|
empty := ""
|
||||||
|
if p, err := SealPtr("t.col", "o", &empty); p != nil || err != nil {
|
||||||
|
t.Errorf("SealPtr(\"\") = %v, %v; want nil", p, err)
|
||||||
|
}
|
||||||
|
if p := RevealPtr("t.col", "o", nil); p != nil {
|
||||||
|
t.Errorf("RevealPtr(nil) = %v, want nil", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoKeyIsAnErrorNotAPanic(t *testing.T) {
|
||||||
|
withKey(t, "")
|
||||||
|
if _, err := Seal("t.col", "o", "x"); !errors.Is(err, ErrNoKey) {
|
||||||
|
t.Errorf("Seal: err = %v, want ErrNoKey", err)
|
||||||
|
}
|
||||||
|
if _, err := Open("t.col", "o", "v1.AAAA"); !errors.Is(err, ErrNoKey) {
|
||||||
|
t.Errorf("Open: err = %v, want ErrNoKey", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRevealReadsAnUnopenableValueAsUnset(t *testing.T) {
|
||||||
|
withKey(t, "test master key")
|
||||||
|
if got := Reveal("t.col", "o", "not sealed"); got != "" {
|
||||||
|
t.Errorf("Reveal = %q, want the empty string", got)
|
||||||
|
}
|
||||||
|
bad := "not sealed"
|
||||||
|
if got := RevealPtr("t.col", "o", &bad); got != nil {
|
||||||
|
t.Errorf("RevealPtr = %q, want nil", *got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEveryColumnHasADistinctPurpose(t *testing.T) {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, c := range Columns {
|
||||||
|
if seen[c.Purpose()] {
|
||||||
|
t.Errorf("%s is listed twice", c.Purpose())
|
||||||
|
}
|
||||||
|
seen[c.Purpose()] = true
|
||||||
|
}
|
||||||
|
if !seen[AcquisitionAPIKey.Purpose()] || !seen[SubsonicPassword.Purpose()] {
|
||||||
|
t.Error("Columns is missing a declared column")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
package secret
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Column is one database column that holds a sealed credential.
|
||||||
|
//
|
||||||
|
// Its Purpose ("table.column") is the key-derivation label every Seal and
|
||||||
|
// Open of that column passes. The owner a value is bound to is the row's key
|
||||||
|
// as text (KeyColumn::text), or "" for a single-row settings table.
|
||||||
|
type Column struct {
|
||||||
|
Table string
|
||||||
|
Column string
|
||||||
|
KeyColumn string
|
||||||
|
Singleton bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Purpose is the label Seal and Open take for this column.
|
||||||
|
func (c Column) Purpose() string { return c.Table + "." + c.Column }
|
||||||
|
|
||||||
|
// The sealed columns, and the purpose strings their callers pass. Every
|
||||||
|
// credential Minstrel stores for an outside system, or that must be read back
|
||||||
|
// in the clear, is listed here.
|
||||||
|
var (
|
||||||
|
AcquisitionAPIKey = Column{"acquisition_services", "api_key", "code", false}
|
||||||
|
AcquisitionPassword = Column{"acquisition_services", "password", "code", false}
|
||||||
|
AcoustIDAPIKey = Column{"acoustid_settings", "api_key", "id", true}
|
||||||
|
SMTPPassword = Column{"smtp_config", "password", "id", true}
|
||||||
|
ListenBrainzToken = Column{"users", "listenbrainz_token", "id", false}
|
||||||
|
SubsonicPassword = Column{"users", "subsonic_password", "id", false}
|
||||||
|
CoverArtAPIKey = Column{"cover_art_provider_settings", "api_key", "provider_id", false}
|
||||||
|
TagProviderAPIKey = Column{"tag_provider_settings", "api_key", "provider_id", false}
|
||||||
|
)
|
||||||
|
|
||||||
|
// Columns is every sealed column, for SealStored and for anything that has to
|
||||||
|
// know which columns hold credentials (a backup leaves them out).
|
||||||
|
var Columns = []Column{
|
||||||
|
AcquisitionAPIKey, AcquisitionPassword, AcoustIDAPIKey, SMTPPassword,
|
||||||
|
ListenBrainzToken, SubsonicPassword, CoverArtAPIKey, TagProviderAPIKey,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Owner is the owner string for a row whose key is key. A single-row table's
|
||||||
|
// rows are owned by "".
|
||||||
|
func (c Column) Owner(key string) string {
|
||||||
|
if c.Singleton {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
// SealStored seals every value in Columns that is still stored in the clear,
|
||||||
|
// and returns how many it sealed. The server runs it at startup, after
|
||||||
|
// migrations and before anything reads a credential, so a database written
|
||||||
|
// before sealing existed is brought over in place. Each row is updated only
|
||||||
|
// if it still holds the value that was read, so a concurrent write wins.
|
||||||
|
func SealStored(ctx context.Context, pool *pgxpool.Pool) (int, error) {
|
||||||
|
total := 0
|
||||||
|
for _, c := range Columns {
|
||||||
|
n, err := sealColumn(ctx, pool, c)
|
||||||
|
total += n
|
||||||
|
if err != nil {
|
||||||
|
return total, fmt.Errorf("seal %s: %w", c.Purpose(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return total, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sealColumn(ctx context.Context, pool *pgxpool.Pool, c Column) (int, error) {
|
||||||
|
table := pgx.Identifier{c.Table}.Sanitize()
|
||||||
|
col := pgx.Identifier{c.Column}.Sanitize()
|
||||||
|
key := pgx.Identifier{c.KeyColumn}.Sanitize()
|
||||||
|
|
||||||
|
rows, err := pool.Query(ctx, fmt.Sprintf(
|
||||||
|
`SELECT %s::text, %s FROM %s WHERE %s IS NOT NULL AND %s <> '' AND %s NOT LIKE 'v1.%%'`,
|
||||||
|
key, col, table, col, col, col))
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
type pending struct{ key, plain string }
|
||||||
|
var todo []pending
|
||||||
|
for rows.Next() {
|
||||||
|
var p pending
|
||||||
|
if err := rows.Scan(&p.key, &p.plain); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
todo = append(todo, p)
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
update := fmt.Sprintf(`UPDATE %s SET %s = $1 WHERE %s::text = $2 AND %s = $3`, table, col, key, col)
|
||||||
|
sealed := 0
|
||||||
|
for _, p := range todo {
|
||||||
|
v, err := Seal(c.Purpose(), c.Owner(p.key), p.plain)
|
||||||
|
if err != nil {
|
||||||
|
return sealed, err
|
||||||
|
}
|
||||||
|
if _, err := pool.Exec(ctx, update, v, p.key, p.plain); err != nil {
|
||||||
|
return sealed, err
|
||||||
|
}
|
||||||
|
sealed++
|
||||||
|
}
|
||||||
|
return sealed, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
package secret_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newPool(t *testing.T) *pgxpool.Pool {
|
||||||
|
t.Helper()
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping integration test in -short mode")
|
||||||
|
}
|
||||||
|
dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
|
||||||
|
if dsn == "" {
|
||||||
|
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
|
||||||
|
}
|
||||||
|
if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil {
|
||||||
|
t.Fatalf("migrate: %v", err)
|
||||||
|
}
|
||||||
|
pool, err := pgxpool.New(context.Background(), dsn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("pool: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(pool.Close)
|
||||||
|
dbtest.ResetDB(t, pool)
|
||||||
|
return pool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every listed column exists, so a renamed or dropped column fails here
|
||||||
|
// rather than leaving a credential unsealed.
|
||||||
|
func TestColumnsExistInTheSchema_Integration(t *testing.T) {
|
||||||
|
pool := newPool(t)
|
||||||
|
for _, c := range secret.Columns {
|
||||||
|
var n int
|
||||||
|
err := pool.QueryRow(context.Background(),
|
||||||
|
`SELECT count(*) FROM information_schema.columns
|
||||||
|
WHERE table_schema = 'public' AND table_name = $1 AND column_name IN ($2, $3)`,
|
||||||
|
c.Table, c.Column, c.KeyColumn).Scan(&n)
|
||||||
|
if err != nil || n != 2 {
|
||||||
|
t.Errorf("%s (key %s): found %d of 2 columns, err %v", c.Purpose(), c.KeyColumn, n, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A database written before sealing existed is sealed in place at startup,
|
||||||
|
// every value opens to what it held, and a second pass has nothing to do.
|
||||||
|
func TestSealStoredSealsWhatIsInTheClear_Integration(t *testing.T) {
|
||||||
|
pool := newPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
q := dbq.New(pool)
|
||||||
|
|
||||||
|
u, err := q.CreateUser(ctx, dbq.CreateUserParams{
|
||||||
|
Username: dbtest.TestUserPrefix + "sealer", PasswordHash: "x", ApiTokenHash: "sealer", IsAdmin: false,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("user: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := q.CreateAcquisitionService(ctx, dbq.CreateAcquisitionServiceParams{
|
||||||
|
Code: "seal-test", Kind: "search", Protocol: "prowlarr", Label: "Seal test",
|
||||||
|
Category: "", TimeoutMs: 1000, SearchesPerHour: 0,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("acquisition row: %v", err)
|
||||||
|
}
|
||||||
|
seed := []string{
|
||||||
|
`UPDATE acquisition_services SET api_key = 'acq-key', password = 'acq-pw' WHERE code = 'seal-test'`,
|
||||||
|
`UPDATE acoustid_settings SET api_key = 'acoustid-key' WHERE id = true`,
|
||||||
|
`UPDATE smtp_config SET password = 'smtp-pw' WHERE id = true`,
|
||||||
|
fmt.Sprintf(`UPDATE users SET listenbrainz_token = 'lb-token', subsonic_password = 'sub-pw' WHERE id = '%s'`, u.ID.String()),
|
||||||
|
`INSERT INTO cover_art_provider_settings (provider_id, enabled, display_order, api_key)
|
||||||
|
VALUES ('seal-test', false, 99, 'cover-key')
|
||||||
|
ON CONFLICT (provider_id) DO UPDATE SET api_key = EXCLUDED.api_key`,
|
||||||
|
`INSERT INTO tag_provider_settings (provider_id, enabled, display_order, api_key)
|
||||||
|
VALUES ('seal-test', false, 99, 'tag-key')
|
||||||
|
ON CONFLICT (provider_id) DO UPDATE SET api_key = EXCLUDED.api_key`,
|
||||||
|
}
|
||||||
|
for _, s := range seed {
|
||||||
|
if _, err := pool.Exec(ctx, s); err != nil {
|
||||||
|
t.Fatalf("seed %q: %v", s, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
for _, s := range []string{
|
||||||
|
`DELETE FROM cover_art_provider_settings WHERE provider_id = 'seal-test'`,
|
||||||
|
`DELETE FROM tag_provider_settings WHERE provider_id = 'seal-test'`,
|
||||||
|
`DELETE FROM acquisition_services WHERE code = 'seal-test'`,
|
||||||
|
`UPDATE acoustid_settings SET api_key = NULL WHERE id = true`,
|
||||||
|
`UPDATE smtp_config SET password = '' WHERE id = true`,
|
||||||
|
} {
|
||||||
|
_, _ = pool.Exec(context.Background(), s)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
want := []struct {
|
||||||
|
col secret.Column
|
||||||
|
key string
|
||||||
|
plain string
|
||||||
|
}{
|
||||||
|
{secret.AcquisitionAPIKey, "seal-test", "acq-key"},
|
||||||
|
{secret.AcquisitionPassword, "seal-test", "acq-pw"},
|
||||||
|
{secret.AcoustIDAPIKey, "true", "acoustid-key"},
|
||||||
|
{secret.SMTPPassword, "true", "smtp-pw"},
|
||||||
|
{secret.ListenBrainzToken, u.ID.String(), "lb-token"},
|
||||||
|
{secret.SubsonicPassword, u.ID.String(), "sub-pw"},
|
||||||
|
{secret.CoverArtAPIKey, "seal-test", "cover-key"},
|
||||||
|
{secret.TagProviderAPIKey, "seal-test", "tag-key"},
|
||||||
|
}
|
||||||
|
if len(want) != len(secret.Columns) {
|
||||||
|
t.Fatalf("the test covers %d columns, secret.Columns lists %d", len(want), len(secret.Columns))
|
||||||
|
}
|
||||||
|
|
||||||
|
n, err := secret.SealStored(ctx, pool)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("SealStored: %v", err)
|
||||||
|
}
|
||||||
|
if n < len(want) {
|
||||||
|
t.Errorf("sealed %d values, want at least %d", n, len(want))
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, w := range want {
|
||||||
|
var stored string
|
||||||
|
query := fmt.Sprintf(`SELECT %s FROM %s WHERE %s::text = $1`,
|
||||||
|
pgx.Identifier{w.col.Column}.Sanitize(), pgx.Identifier{w.col.Table}.Sanitize(),
|
||||||
|
pgx.Identifier{w.col.KeyColumn}.Sanitize())
|
||||||
|
if err := pool.QueryRow(ctx, query, w.key).Scan(&stored); err != nil {
|
||||||
|
t.Errorf("%s: read: %v", w.col.Purpose(), err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !secret.IsSealed(stored) {
|
||||||
|
t.Errorf("%s = %q, want it sealed", w.col.Purpose(), stored)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
got, err := secret.Open(w.col.Purpose(), w.col.Owner(w.key), stored)
|
||||||
|
if err != nil || got != w.plain {
|
||||||
|
t.Errorf("%s opens to %q (err %v), want %q", w.col.Purpose(), got, err, w.plain)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
again, err := secret.SealStored(ctx, pool)
|
||||||
|
if err != nil || again != 0 {
|
||||||
|
t.Errorf("second pass sealed %d (err %v), want 0", again, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config controls wire-format concessions the server makes for Subsonic
|
// Config controls wire-format concessions the server makes for Subsonic
|
||||||
@@ -101,11 +102,15 @@ func authenticate(r *http.Request, pool *pgxpool.Pool, cfg Config) (dbq.User, in
|
|||||||
return dbq.User{}, ErrGeneric, "Auth lookup failed"
|
return dbq.User{}, ErrGeneric, "Auth lookup failed"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The stored Subsonic password is sealed; both t+s and p= compare against
|
||||||
|
// it opened. One that does not open reads as not configured.
|
||||||
|
subsonicPassword := secret.RevealPtr(secret.SubsonicPassword.Purpose(), user.ID.String(), user.SubsonicPassword)
|
||||||
|
|
||||||
if token, salt := params.Get("t"), params.Get("s"); token != "" && salt != "" {
|
if token, salt := params.Get("t"), params.Get("s"); token != "" && salt != "" {
|
||||||
if user.SubsonicPassword == nil {
|
if subsonicPassword == nil {
|
||||||
return dbq.User{}, ErrTokenNotSupported, "Subsonic token auth not configured; use apiKey or set a Subsonic password"
|
return dbq.User{}, ErrTokenNotSupported, "Subsonic token auth not configured; use apiKey or set a Subsonic password"
|
||||||
}
|
}
|
||||||
sum := md5.Sum([]byte(*user.SubsonicPassword + salt))
|
sum := md5.Sum([]byte(*subsonicPassword + salt))
|
||||||
expected := hex.EncodeToString(sum[:])
|
expected := hex.EncodeToString(sum[:])
|
||||||
if subtle.ConstantTimeCompare([]byte(expected), []byte(token)) != 1 {
|
if subtle.ConstantTimeCompare([]byte(expected), []byte(token)) != 1 {
|
||||||
return dbq.User{}, ErrWrongCredentials, "Wrong username or password"
|
return dbq.User{}, ErrWrongCredentials, "Wrong username or password"
|
||||||
@@ -117,14 +122,14 @@ func authenticate(r *http.Request, pool *pgxpool.Pool, cfg Config) (dbq.User, in
|
|||||||
if !cfg.AllowPlaintextPassword {
|
if !cfg.AllowPlaintextPassword {
|
||||||
return dbq.User{}, ErrTokenNotSupported, "Plaintext password auth is disabled; use apiKey or t+s"
|
return dbq.User{}, ErrTokenNotSupported, "Plaintext password auth is disabled; use apiKey or t+s"
|
||||||
}
|
}
|
||||||
if user.SubsonicPassword == nil {
|
if subsonicPassword == nil {
|
||||||
return dbq.User{}, ErrTokenNotSupported, "Subsonic password not configured for this user"
|
return dbq.User{}, ErrTokenNotSupported, "Subsonic password not configured for this user"
|
||||||
}
|
}
|
||||||
decoded, err := decodePassword(password)
|
decoded, err := decodePassword(password)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return dbq.User{}, ErrWrongCredentials, "Malformed password"
|
return dbq.User{}, ErrWrongCredentials, "Malformed password"
|
||||||
}
|
}
|
||||||
if subtle.ConstantTimeCompare([]byte(*user.SubsonicPassword), []byte(decoded)) != 1 {
|
if subtle.ConstantTimeCompare([]byte(*subsonicPassword), []byte(decoded)) != 1 {
|
||||||
return dbq.User{}, ErrWrongCredentials, "Wrong username or password"
|
return dbq.User{}, ErrWrongCredentials, "Wrong username or password"
|
||||||
}
|
}
|
||||||
return user, 0, ""
|
return user, 0, ""
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SettingsService reconciles the compiled-in provider registry with the
|
// SettingsService reconciles the compiled-in provider registry with the
|
||||||
@@ -101,7 +102,7 @@ func (s *SettingsService) reconcile(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
key := ""
|
key := ""
|
||||||
if row.ApiKey != nil {
|
if row.ApiKey != nil {
|
||||||
key = *row.ApiKey
|
key = secret.Reveal(secret.TagProviderAPIKey.Purpose(), row.ProviderID, *row.ApiKey)
|
||||||
}
|
}
|
||||||
s.enabledIDs[p.ID()] = row.Enabled
|
s.enabledIDs[p.ID()] = row.Enabled
|
||||||
s.apiKeys[p.ID()] = key
|
s.apiKeys[p.ID()] = key
|
||||||
@@ -181,11 +182,15 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
|
|||||||
if patch.Enabled != nil {
|
if patch.Enabled != nil {
|
||||||
enabledVal = *patch.Enabled
|
enabledVal = *patch.Enabled
|
||||||
}
|
}
|
||||||
|
sealedKey, err := secret.SealPtr(secret.TagProviderAPIKey.Purpose(), providerID, patch.APIKey)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("seal provider api key: %w", err)
|
||||||
|
}
|
||||||
if err := q.UpdateTagProviderSettings(ctx, dbq.UpdateTagProviderSettingsParams{
|
if err := q.UpdateTagProviderSettings(ctx, dbq.UpdateTagProviderSettingsParams{
|
||||||
ProviderID: providerID,
|
ProviderID: providerID,
|
||||||
Enabled: enabledVal,
|
Enabled: enabledVal,
|
||||||
Column3: patch.APIKey != nil,
|
Column3: patch.APIKey != nil,
|
||||||
ApiKey: patch.APIKey,
|
ApiKey: sealedKey,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return false, fmt.Errorf("update provider settings: %w", err)
|
return false, fmt.Errorf("update provider settings: %w", err)
|
||||||
}
|
}
|
||||||
@@ -200,7 +205,7 @@ func (s *SettingsService) UpdateProvider(ctx context.Context, providerID string,
|
|||||||
for _, r := range rows {
|
for _, r := range rows {
|
||||||
key := ""
|
key := ""
|
||||||
if r.ApiKey != nil {
|
if r.ApiKey != nil {
|
||||||
key = *r.ApiKey
|
key = secret.Reveal(secret.TagProviderAPIKey.Purpose(), r.ProviderID, *r.ApiKey)
|
||||||
}
|
}
|
||||||
s.enabledIDs[r.ProviderID] = r.Enabled
|
s.enabledIDs[r.ProviderID] = r.Enabled
|
||||||
s.apiKeys[r.ProviderID] = key
|
s.apiKeys[r.ProviderID] = key
|
||||||
|
|||||||
Reference in New Issue
Block a user