Compare commits

..
Author SHA1 Message Date
bvandeusen f70df9f827 Recommendation relevance, the rollback unit, and a version that names what shipped (#131)
test-web / test (push) Successful in 1m5s
test-go / test (push) Successful in 1m33s
test-go / integration (push) Successful in 4m25s
android / Build + lint + test (push) Successful in 5m33s
release / Build signed APK (releases and dev) (push) Successful in 4m38s
release / Build + push container image (push) Successful in 16s
release / Verify release artifacts (tag releases only) (push) Successful in 2s
2026-09-10 23:37:57 -04:00
bvandeusen 439c8625d5 Fix the rebundle step: a non-matching grep must not kill it (#130)
test-go / test (push) Successful in 59s
test-go / integration (push) Successful in 3m1s
release / Build signed APK (releases and dev) (push) Successful in 4m41s
release / Build + push container image (push) Successful in 14s
release / Verify release artifacts (tag releases only) (push) Successful in 2s
2026-09-10 10:49:40 -04:00
bvandeusen 1d67c160b2 Versioning rework, a dev channel, and the miniplayer gap (#129)
release / Build signed APK (releases and dev) (push) Skipped
android / Build + lint + test (push) Failing after 4s
release / Build + push container image (push) Failing after 5s
release / Verify release artifacts (tag releases only) (push) Skipped
test-go / test (push) Successful in 1m24s
test-go / integration (push) Successful in 6m25s
2026-09-10 08:33:26 -04:00
bvandeusen 237380b122 New brand mark, and both clients stop fetching their fonts at runtime (#128)
test-web / test (push) Successful in 54s
android / Build + lint + test (push) Successful in 5m36s
release / Build signed APK (tag releases only) (push) Successful in 4m36s
release / Build + push container image (push) Successful in 24s
release / Verify release artifacts (tag releases only) (push) Successful in 2s
2026-09-09 14:36:15 -04:00
bvandeusen 4f077736b6 Merge PR #127: Sonos queue verification, cast double-download fix, stutter instrumentation
android / Build + lint + test (push) Successful in 4m16s
release / Build signed APK (tag releases only) (push) Successful in 5m25s
release / Build + push container image (push) Successful in 1m16s
release / Verify release artifacts (tag releases only) (push) Successful in 1s
2026-08-18 10:50:10 -04:00
bvandeusen 727f68950e Merge PR #126: missing-file lifecycle, UPnP stall recovery, Android browse parity, Flutter client removed
release / Build signed APK (tag releases only) (push) Skipped
test-web / test (push) Successful in 1m5s
test-go / test (push) Successful in 1m26s
release / Build + push container image (push) Successful in 1m38s
release / Verify release artifacts (tag releases only) (push) Skipped
android / Build + lint + test (push) Successful in 4m54s
test-go / integration (push) Successful in 5m19s
2026-08-17 16:28:13 -04:00
bvandeusen aa9f534f3c Merge pull request 'Genre index: sort A–Z, and repair casing damage at scan time' (#124) from dev into main
test-web / test (push) Successful in 49s
test-go / test (push) Successful in 1m9s
test-go / integration (push) Successful in 5m1s
release / Build signed APK (tag releases only) (push) Successful in 4m2s
release / Build + push container image (push) Successful in 14s
release / Verify release artifacts (tag releases only) (push) Successful in 2s
2026-08-07 21:40:33 -04:00
bvandeusen 011b4d9a9c Merge pull request 'ci(release): verify a tag release actually shipped its artifacts' (#123) from dev into main
release / Build signed APK (tag releases only) (push) Successful in 3m38s
release / Build + push container image (push) Successful in 1m31s
release / Verify release artifacts (tag releases only) (push) Successful in 2s
2026-08-07 08:32:48 -04:00
bvandeusen d5aa081157 Merge pull request 'Recommendation metrics: publish the margin of error on every delta' (#122) from dev into main
test-web / test (push) Successful in 52s
test-go / test (push) Successful in 1m10s
test-go / integration (push) Successful in 4m53s
release / Build signed APK (tag releases only) (push) Successful in 3m57s
release / Build + push container image (push) Successful in 1m37s
2026-08-06 21:50:41 -04:00
bvandeusen a99f855e98 Merge pull request 'Missing files: detect them, stop offering them, and follow them when they move' (#121) from dev into main
test-go / test (push) Successful in 56s
test-go / integration (push) Successful in 5m0s
release / Build signed APK (tag releases only) (push) Successful in 4m14s
release / Build + push container image (push) Successful in 15s
2026-08-06 20:40:39 -04:00
bvandeusen 7e4727fc49 Merge pull request 'Genre tags: read multi-value frames correctly, and repair existing rows' (#120) from dev into main
test-go / test (push) Successful in 57s
test-go / integration (push) Successful in 4m57s
release / Build signed APK (tag releases only) (push) Successful in 4m23s
release / Build + push container image (push) Successful in 1m39s
2026-08-05 22:10:41 -04:00
bvandeusen 1b7fa635d8 Merge pull request 'Silent self-update, active sessions with real client IPs, genre/year browsing, handoff fix' (#119) from dev into main
test-web / test (push) Successful in 1m3s
test-go / test (push) Successful in 1m13s
test-go / integration (push) Successful in 5m29s
android / Build + lint + test (push) Successful in 5m34s
release / Build signed APK (tag releases only) (push) Successful in 5m5s
release / Build + push container image (push) Successful in 16s
2026-08-05 15:14:48 -04:00
bvandeusen 57d2299180 Merge pull request 'Queue row gestures: album art as grab surface + swipe-to-remove' (#118) from dev into main
test-web / test (push) Successful in 48s
android / Build + lint + test (push) Successful in 5m15s
release / Build signed APK (tag releases only) (push) Successful in 4m38s
release / Build + push container image (push) Successful in 1m48s
2026-08-04 11:37:30 -04:00
bvandeusen fa7ea41ccf Merge pull request 'Minstrel gets a mark — favicon, header lockup, Android adaptive icon' (#117) from dev into main
test-web / test (push) Successful in 47s
android / Build + lint + test (push) Successful in 4m37s
release / Build signed APK (tag releases only) (push) Successful in 8m36s
release / Build + push container image (push) Successful in 1m37s
2026-08-03 20:52:27 -04:00
bvandeusen 324059b2bd Merge pull request 'Discover request surface — taste-aware, rotating, snoozable, tag-targeted (milestone #268)' (#116) from dev into main
test-web / test (push) Successful in 1m5s
test-go / test (push) Successful in 1m30s
android / Build + lint + test (push) Successful in 5m1s
test-go / integration (push) Successful in 5m29s
release / Build signed APK (tag releases only) (push) Successful in 4m21s
release / Build + push container image (push) Successful in 17s
2026-08-03 08:38:24 -04:00
bvandeusen 1138d75a45 Merge pull request 'Playlist-track atomic replace + ci-requirements true-up' (#115) from dev into main
release / Build signed APK (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 1m33s
android / Build + lint + test (push) Successful in 4m30s
2026-08-01 12:23:37 -04:00
177 changed files with 1159 additions and 11161 deletions
+92
View File
@@ -0,0 +1,92 @@
name: android
# Native Android (Kotlin/Compose/Media3) — M8 rewrite, now the only client.
# This workflow is testing only — lint + detekt + unit tests on every push
# to dev/main, plus a debug APK artifact for main. The signed-release
# build + asset attach + image-bundling lives in release.yml under a
# `needs:` chain so the docker image cannot ship without the APK.
on:
push:
branches: [main, dev]
paths:
- 'android/**'
- '.gitea/workflows/android.yml'
# pull_request trigger intentionally omitted — see test-web.yml for
# the rationale (single-author repo, push covers PR-merge equivalent).
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
# Silences the JDK 22+ "restricted method in java.lang.System has been
# called" warning that Gradle 9.1's bundled native-platform jar trips
# at launch (System.load for native primitives). Affects the LAUNCHER
# JVM, not the daemon — that's why org.gradle.jvmargs in
# gradle.properties isn't enough. Future-compat: required opt-in once
# JDK 25 promotes the warning to an error.
JAVA_TOOL_OPTIONS: "--enable-native-access=ALL-UNNAMED"
jobs:
build:
name: Build + lint + test
# Using flutter-ci runner label because it's the only proven-working
# label with docker that can pull our container.image. Switch to
# android-ci once the operator registers that runner label.
runs-on: flutter-ci
container:
image: git.fabledsword.com/bvandeusen/ci-android:36
defaults:
run:
working-directory: android
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Cache Gradle dirs
# Resolved deps + Gradle distribution + Kotlin daemon caches.
# Saves ~3 min per CI run after the first warm-up.
uses: actions/cache@v4
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
~/.kotlin
key: gradle-${{ runner.os }}-${{ hashFiles('android/gradle/wrapper/gradle-wrapper.properties', 'android/gradle/libs.versions.toml', 'android/**/*.gradle.kts') }}
restore-keys: |
gradle-${{ runner.os }}-
- name: Make gradlew executable
run: chmod +x ./gradlew
- name: Gradle wrapper validation
run: ./gradlew --version
- name: ktlint
run: ./gradlew ktlintCheck
- name: detekt
run: ./gradlew detekt
- name: Unit tests
run: ./gradlew testDebugUnitTest
- name: Assemble debug
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
run: ./gradlew assembleDebug
- name: Upload debug APK
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
# Stock action: it works on this forge since the runner moved to
# gitea/runner 3.x, which edits upload-artifact's client-side GHES refusal
# out of the action bundle (Scribe snippet #2271). Never @v3 — it reports
# success while Gitea serves artifacts back only through the v4 API, and
# it is what left 72 unreachable artifacts on this repo (Scribe 2270).
uses: actions/upload-artifact@v7
with:
name: minstrel-android-debug-${{ github.sha }}
path: android/app/build/outputs/apk/debug/app-debug.apk
if-no-files-found: error
+18 -345
View File
@@ -72,8 +72,9 @@ name: release
# Android APK and uploads it as a workflow artifact. The image-release
# job declares `needs: android-release`, so the docker image cannot
# start building until the APK is guaranteed-ready — no polling, no
# race, no silent-failure mode. Attaching the APK to the gitea Release is
# its own job (release-assets), behind the test gate below.
# race, no silent-failure mode. Asset attachment to the gitea Release
# happens in the same android-release job, so the Release-page download
# link and the in-image bundled APK are both populated atomically.
#
# :latest always carries an APK. Because every main push also moves
# :latest (not just tags), a main build with no APK would silently strip
@@ -83,33 +84,8 @@ name: release
# recomputed ones — so no rebuild is needed, just a rebundle. Tag builds
# keep bundling their own freshly-built APK.
#
# THE GATE (rule 177, M462 #4984). Every verifying lane lives in this file —
# Go (vet, lint, short tests), the Postgres integration suite, the web app
# (npm audit, svelte-check, vitest), Android (ktlint, detekt, unit tests) and
# govulncheck — and every job that publishes something names each of them in
# `needs:` and requires `success` from each, by name. Nothing publishes on red.
#
# They used to be three separate workflows (test-go, test-web, android) on the
# same push trigger as this one. Separate workflows cannot see each other's
# verdict, so :dev meant "it built", never "it passed": a red test run and a
# fresh :dev could carry the same timestamp. One graph is the only place the
# edge can be written.
#
# A skipped lane is NOT a pass. The publishing conditions check
# `result == 'success'` per lane rather than `!failure()`, so a lane that
# never started blocks the publish exactly as a red one does. Lanes carry no
# path filters for the same reason: a web-only push still runs the Go suite,
# because "not run" must never read as "passed".
#
# What publishes, and is therefore gated: the image tags (image-release) and
# the APK + version sidecar attached to a tag's Release (release-assets).
# android-release only BUILDS the signed APK into a workflow artifact, which
# nobody outside this run can pull, so it runs in parallel with the lanes
# instead of after them; attaching it to the Release is the publishing half,
# and that half waits for the gate.
#
# To watch the gate refuse: dispatch this workflow with force_red=true. The go
# lane fails on purpose, and both publishing jobs must report skipped.
# Android testing (lint + detekt + unit tests, debug APK upload on main)
# lives in android.yml and runs independently on every push.
on:
push:
@@ -119,11 +95,6 @@ on:
- 'docs/**'
- '**/*.md'
workflow_dispatch:
inputs:
force_red:
description: Fail the go lane on purpose, to check that nothing publishes on red
type: boolean
default: false
# A rapid re-push to main should supersede the in-flight build — the
# operator explicitly wants the later commit to win. Tags no longer enter
@@ -134,265 +105,6 @@ concurrency:
cancel-in-progress: true
jobs:
# ---------------------------------------------------------------- lanes --
# Verifying jobs. Each one is named in the `needs:` of every publishing job
# below; add a lane here and it must be added there in the same commit.
go:
runs-on: go-ci
container:
image: git.fabledsword.com/bvandeusen/ci-go:1.26
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Forced failure (gate check)
if: github.event.inputs.force_red == 'true'
run: |
echo "::error::force_red dispatch: failing on purpose so the publishing jobs must skip"
exit 1
- name: Toolchain versions
run: |
go version
golangci-lint --version
- name: Generated code matches queries (sqlc)
run: make verify-generate
- name: go vet
run: go vet ./...
- name: golangci-lint
run: golangci-lint run ./...
- name: go test (short, race)
run: go test -short -race ./...
# Full `go test -race` against an ephemeral Postgres.
#
# DB wiring follows the act_runner shared-daemon pattern: the runner's Docker
# daemon also runs the operator's dev compose stack, so service containers
# get NO published ports (collision) and no service-name DNS. We discover the
# service container through the mounted docker socket and reach it by bridge
# IP. The exactly-one assertion is a hard guard — pointing tests at the dev
# Postgres would truncate it (the disaster Fable #339 exists to prevent).
#
# The key stays `integration` with no `name:` (rule 80): act_runner derives
# the service container's name from the job's display name.
#
# `web/build/` has a committed placeholder index.html so go:embed succeeds
# without the SPA being built first.
integration:
runs-on: go-ci
container:
image: git.fabledsword.com/bvandeusen/ci-go:1.26
services:
postgres:
image: postgres:16-alpine
env:
POSTGRES_USER: minstrel
POSTGRES_PASSWORD: minstrel
POSTGRES_DB: minstrel_test
# No `ports:` — the runner shares the operator's dev compose
# Docker daemon; publishing a fixed host port collides.
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Integration suite (discover service by bridge IP, migrate, test)
run: |
set -eux
# Discover THIS job's Postgres service container via the
# mounted docker socket. act_runner attaches the job
# container and its service container(s) to a shared per-job
# network, so scope discovery to a postgres that sits on a
# network THIS job container is also on. The old
# `--filter name=integration` matched EVERY concurrent
# integration run's postgres (a dev push + the main-merge run
# overlap → 2 candidates → false "expected exactly 1" abort).
# The operator's dev compose `minstrel-postgres-*` is never on
# this job's network; skip it explicitly as belt-and-suspenders
# (a wrong target would truncate real data).
SELF=$(cat /etc/hostname)
SELF_NETS=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' "$SELF")
test -n "$SELF_NETS"
echo "self ($SELF) networks: $SELF_NETS"
PG_ID=""
PG_NAME=""
for cid in $(docker ps --filter "ancestor=postgres:16-alpine" -q); do
nm=$(docker inspect -f '{{.Name}}' "$cid" | sed 's#^/##')
case "$nm" in *minstrel-postgres*|*_postgres_*) continue ;; esac
for net in $(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' "$cid"); do
case " $SELF_NETS " in *" $net "*) PG_ID="$cid"; PG_NAME="$nm"; break 2 ;; esac
done
done
test -n "$PG_ID" || { echo "FATAL: no postgres service container on this job's network (self nets: $SELF_NETS)"; exit 1; }
echo "selected postgres: $PG_ID $PG_NAME"
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG_ID")
test -n "$PG_IP"
export MINSTREL_TEST_DATABASE_URL="postgres://minstrel:minstrel@${PG_IP}:5432/minstrel_test?sslmode=disable"
# Wait for Postgres to accept connections. Asked of the service
# container itself: the run: shell is dash (rule 81), where the
# old `/dev/tcp` probe never connects and the loop silently
# burned its full two minutes on every run.
ready=""
for i in $(seq 1 60); do
if docker exec "$PG_ID" pg_isready -U minstrel -d minstrel_test -q; then ready=1; break; fi
sleep 2
done
test -n "$ready" || { echo "FATAL: postgres never became ready"; exit 1; }
# Relax durability on the throwaway CI Postgres. Our test pattern
# is dbtest.ResetDB → TRUNCATE … RESTART IDENTITY CASCADE before
# every test, and the per-TRUNCATE commit fsync is the dominant
# cost of the integration suite. The CI DB is rebuilt every run so
# fsync / full_page_writes / synchronous_commit buy nothing. Apply
# via docker exec because:
# - The act_runner `services:` block can't override the container
# command, so `postgres -c fsync=off` at boot isn't an option.
# - ALTER SYSTEM cannot run inside a transaction; psql -c
# auto-commits each statement, which is what we need.
# - fsync / full_page_writes are sighup GUCs and
# synchronous_commit is user-context, so pg_reload_conf() picks
# all three up with no restart.
# Non-fatal: a perms surprise degrades to "slower", never red CI.
docker exec "$PG_ID" psql -U minstrel -d minstrel_test \
-c "ALTER SYSTEM SET fsync = off" \
-c "ALTER SYSTEM SET synchronous_commit = off" \
-c "ALTER SYSTEM SET full_page_writes = off" \
-c "SELECT pg_reload_conf()" \
|| echo "WARN: durability relax failed; continuing"
# Apply embedded migrations to the fresh test DB, then run the
# full suite (no -short → integration tests execute). -p 1:
# every integration package TRUNCATEs the one shared test DB;
# concurrent package binaries → TRUNCATE deadlocks. Serialize
# package execution (the documented local invocation too).
MINSTREL_DATABASE_URL="$MINSTREL_TEST_DATABASE_URL" go run ./cmd/minstrel migrate
go test -p 1 -race ./...
web:
runs-on: go-ci
container:
image: git.fabledsword.com/bvandeusen/ci-go:1.26
defaults:
run:
working-directory: web
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install deps
run: npm ci
# What ships to browsers: `dependencies` and the runtime they pull in
# (svelte, devalue). Build and test tooling (vite, vitest, tailwind,
# kit's dev server) is left out because none of it reaches a user, and
# its open advisories need major-version upgrades tracked separately.
- name: npm audit (shipped dependencies)
run: npm audit --omit=dev --audit-level=moderate
- name: Type-check + svelte-check
run: npm run check
- name: Vitest
run: npm test
android:
# Using flutter-ci runner label because it's the only proven-working
# label with docker that can pull our container.image.
runs-on: flutter-ci
container:
image: git.fabledsword.com/bvandeusen/ci-android:36
defaults:
run:
working-directory: android
env:
# Silences the JDK 22+ "restricted method in java.lang.System has been
# called" warning that Gradle's bundled native-platform jar trips at
# launch. Affects the LAUNCHER JVM, not the daemon — that's why
# org.gradle.jvmargs in gradle.properties isn't enough.
JAVA_TOOL_OPTIONS: "--enable-native-access=ALL-UNNAMED"
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Cache Gradle dirs
uses: actions/cache@v4
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
~/.kotlin
key: gradle-${{ runner.os }}-${{ hashFiles('android/gradle/wrapper/gradle-wrapper.properties', 'android/gradle/libs.versions.toml', 'android/**/*.gradle.kts') }}
restore-keys: |
gradle-${{ runner.os }}-
- name: Make gradlew executable
run: chmod +x ./gradlew
- name: Gradle wrapper validation
run: ./gradlew --version
- name: ktlint
run: ./gradlew ktlintCheck
- name: detekt
run: ./gradlew detekt
- name: Unit tests
run: ./gradlew testDebugUnitTest
- name: Assemble debug
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
run: ./gradlew assembleDebug
- name: Upload debug APK
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
# Stock action: it works on this forge since the runner moved to
# gitea/runner 3.x, which edits upload-artifact's client-side GHES refusal
# out of the action bundle (Scribe snippet #2271). Never @v3 — it reports
# success while Gitea serves artifacts back only through the v4 API, and
# it is what left 72 unreachable artifacts on this repo (Scribe 2270).
uses: actions/upload-artifact@v7
with:
name: minstrel-android-debug-${{ github.sha }}
path: android/app/build/outputs/apk/debug/app-debug.apk
if-no-files-found: error
# Known vulnerabilities in the Go code and the standard library it is built
# with. Runs in the SAME image the Dockerfile's builder stage uses, so the
# standard library it checks is the one that ends up in the shipped binary;
# the ci-go image carries its own Go and would be checking a different
# toolchain. Keep this image and the Dockerfile's builder in step.
#
# govulncheck is fetched at CI time, unpinned (rule 154): the vulnerability
# database and the tool that reads it should both be current.
govulncheck:
runs-on: go-ci
container:
image: golang:1.26-bookworm
steps:
- name: Checkout
uses: actions/checkout@v4
- name: govulncheck
run: |
go version
go run golang.org/x/vuln/cmd/govulncheck@latest ./...
# ------------------------------------------------------------ artifacts --
android-release:
name: Build signed APK (releases and dev)
# Also builds on `dev`, which is what makes a test channel possible at
@@ -533,47 +245,21 @@ jobs:
# artifact existing, so an empty upload must fail here, not there.
if-no-files-found: error
# Publishes the signed APK and its version sidecar on the tag's Release.
# Split out of android-release so the APK can be BUILT in parallel with the
# lanes while being PUBLISHED only once they have all passed.
release-assets:
name: Attach APK to the Release (tag releases only)
needs: [go, integration, web, android, govulncheck, android-release]
if: >-
${{
!cancelled()
&& needs.go.result == 'success'
&& needs.integration.result == 'success'
&& needs.web.result == 'success'
&& needs.android.result == 'success'
&& needs.govulncheck.result == 'success'
&& needs.android-release.result == 'success'
&& startsWith(github.ref, 'refs/tags/v') }}
runs-on: go-ci
container:
image: git.fabledsword.com/bvandeusen/ci-go:1.26
steps:
- name: Download signed APK artifact
uses: actions/download-artifact@v8
with:
name: minstrel-apk
path: release-apk/
- name: Attach APK to gitea Release
# Tag releases only. A dev build has no Release to hang assets on and
# does not need one — the :dev image bundles the APK, and the server
# serves it from /api/client/apk like any other.
if: startsWith(github.ref, 'refs/tags/v')
shell: bash
env:
CI_TOKEN: ${{ secrets.CI_TOKEN }}
VERSION_NAME: ${{ needs.android-release.outputs.version_name }}
VERSION_CODE: ${{ needs.android-release.outputs.version_code }}
VERSION_NAME: ${{ steps.ver.outputs.name }}
VERSION_CODE: ${{ steps.ver.outputs.code }}
run: |
set -euxo pipefail
TAG="${GITHUB_REF#refs/tags/}"
REPO="${GITHUB_REPOSITORY}"
APK_PATH="release-apk/app-release.apk"
APK_PATH="app/build/outputs/apk/release/app-release.apk"
ls -lh "${APK_PATH}"
# Publish the version sidecar as a release asset next to the APK.
@@ -627,21 +313,13 @@ jobs:
image-release:
name: Build + push container image
# Every lane must have SUCCEEDED, each named here (rule 177). Then the
# APK: tag and dev pushes build one and it must have succeeded; main
# pushes skip android-release and bundle the latest release's APK
# instead, so for main alone a skipped android-release is expected.
needs: [go, integration, web, android, govulncheck, android-release]
if: >-
${{
!cancelled()
&& needs.go.result == 'success'
&& needs.integration.result == 'success'
&& needs.web.result == 'success'
&& needs.android.result == 'success'
&& needs.govulncheck.result == 'success'
&& (needs.android-release.result == 'success'
|| (needs.android-release.result == 'skipped' && github.ref == 'refs/heads/main')) }}
# `needs:` waits for android-release. For tag pushes android-release
# runs and must succeed before this job starts — guaranteeing the
# APK artifact is present. For main pushes android-release is
# skipped; the `if: ...` below lets this job run anyway and the
# download/copy steps gate themselves on the tag context.
needs: [android-release]
if: ${{ !failure() && !cancelled() }}
runs-on: go-ci
container:
image: git.fabledsword.com/bvandeusen/ci-go:1.26
@@ -850,13 +528,8 @@ jobs:
- name: Build and push
if: steps.guard.outputs.ready == 'true'
# --pull: the Dockerfile's base images are floating tags (golang:1.26,
# debian:bookworm-slim). Without it the runner's daemon reuses
# whatever it cached, and the shipped binary can sit on a Go patch
# release govulncheck already flagged while the lane, which pulls
# fresh, reports clean.
run: |
docker buildx build --pull \
docker buildx build \
--build-arg MINSTREL_VERSION="${{ steps.tags.outputs.version }}" \
--build-arg MINSTREL_CHANNEL="${{ steps.tags.outputs.channel }}" \
--push ${{ steps.tags.outputs.args }} .
@@ -881,7 +554,7 @@ jobs:
# above did NOT succeed.
verify-release:
name: Verify release artifacts (tag releases only)
needs: [android-release, release-assets, image-release]
needs: [android-release, image-release]
if: ${{ always() && startsWith(github.ref, 'refs/tags/v') }}
runs-on: go-ci
container:
+156
View File
@@ -0,0 +1,156 @@
name: test-go
# Go server: vet + golangci-lint + short race tests. Runs on push to
# dev/main and PRs to main, scoped to Go-side files only — web-only or
# Flutter-only diffs don't trigger this workflow.
#
# Two jobs: `test` (fast — vet + lint + `go test -short -race`, no DB) and
# `integration` (full `go test -race` against an ephemeral Postgres).
#
# Integration-job DB wiring follows the act_runner shared-daemon pattern:
# the runner's Docker daemon also runs the operator's dev compose stack,
# so service containers get NO published ports (collision) and no
# service-name DNS. We discover the service container by the job-scoped
# name filter via the mounted docker socket and reach it by bridge IP.
# The exactly-one assertion is a hard guard — pointing tests at the dev
# Postgres would truncate it (the disaster Fable #339 exists to prevent).
#
# `web/build/` has a committed placeholder index.html so go:embed succeeds
# without needing the SPA to be freshly built. Real builds happen in
# release.yml (container) and locally during dev.
on:
push:
branches: [dev, main]
paths:
- '**/*.go'
- 'go.mod'
- 'go.sum'
- 'sqlc.yaml'
- 'Makefile'
- 'internal/**'
- 'cmd/**'
- '.golangci.yml'
- '.gitea/workflows/test-go.yml'
# The release lane's own trigger is `main` + tags, so nothing it
# contains is exercised until a release is already running. These two
# entries are what let internal/server/release_version_test.go guard
# the version derivation on ordinary dev pushes instead.
- 'ci/**'
- '.gitea/workflows/release.yml'
# pull_request trigger intentionally omitted — see test-web.yml for
# the rationale (single-author repo, push covers PR-merge equivalent).
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
runs-on: go-ci
container:
image: git.fabledsword.com/bvandeusen/ci-go:1.26
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Toolchain versions
run: |
go version
golangci-lint --version
- name: Generated code matches queries (sqlc)
run: make verify-generate
- name: go vet
run: go vet ./...
- name: golangci-lint
run: golangci-lint run ./...
- name: go test (short, race)
run: go test -short -race ./...
integration:
runs-on: go-ci
container:
image: git.fabledsword.com/bvandeusen/ci-go:1.26
services:
postgres:
image: postgres:16-alpine
env:
POSTGRES_USER: minstrel
POSTGRES_PASSWORD: minstrel
POSTGRES_DB: minstrel_test
# No `ports:` — the runner shares the operator's dev compose
# Docker daemon; publishing a fixed host port collides.
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Integration suite (discover service by bridge IP, migrate, test)
run: |
set -eux
# Discover THIS job's Postgres service container via the
# mounted docker socket. act_runner attaches the job
# container and its service container(s) to a shared per-job
# network, so scope discovery to a postgres that sits on a
# network THIS job container is also on. The old
# `--filter name=integration` matched EVERY concurrent
# integration run's postgres (a dev push + the main-merge run
# overlap → 2 candidates → false "expected exactly 1" abort).
# The operator's dev compose `minstrel-postgres-*` is never on
# this job's network; skip it explicitly as belt-and-suspenders
# (a wrong target would truncate real data).
SELF=$(cat /etc/hostname)
SELF_NETS=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' "$SELF")
test -n "$SELF_NETS"
echo "self ($SELF) networks: $SELF_NETS"
PG_ID=""
PG_NAME=""
for cid in $(docker ps --filter "ancestor=postgres:16-alpine" -q); do
nm=$(docker inspect -f '{{.Name}}' "$cid" | sed 's#^/##')
case "$nm" in *minstrel-postgres*|*_postgres_*) continue ;; esac
for net in $(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' "$cid"); do
case " $SELF_NETS " in *" $net "*) PG_ID="$cid"; PG_NAME="$nm"; break 2 ;; esac
done
done
test -n "$PG_ID" || { echo "FATAL: no postgres service container on this job's network (self nets: $SELF_NETS)"; exit 1; }
echo "selected postgres: $PG_ID $PG_NAME"
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG_ID")
test -n "$PG_IP"
export MINSTREL_TEST_DATABASE_URL="postgres://minstrel:minstrel@${PG_IP}:5432/minstrel_test?sslmode=disable"
# Wait for Postgres to accept TCP (no health-check dependency).
for i in $(seq 1 60); do (echo > "/dev/tcp/${PG_IP}/5432") 2>/dev/null && break; sleep 2; done
# Relax durability on the throwaway CI Postgres. Our test pattern
# is dbtest.ResetDB → TRUNCATE … RESTART IDENTITY CASCADE before
# every test, and the per-TRUNCATE commit fsync is the dominant
# cost of the integration suite. The CI DB is rebuilt every run so
# fsync / full_page_writes / synchronous_commit buy nothing. Apply
# via docker exec because:
# - The act_runner `services:` block can't override the container
# command, so `postgres -c fsync=off` at boot isn't an option.
# - ALTER SYSTEM cannot run inside a transaction; psql -c
# auto-commits each statement, which is what we need.
# - fsync / full_page_writes are sighup GUCs and
# synchronous_commit is user-context, so pg_reload_conf() picks
# all three up with no restart.
# Non-fatal: a perms surprise degrades to "slower", never red CI.
docker exec "$PG_ID" psql -U minstrel -d minstrel_test \
-c "ALTER SYSTEM SET fsync = off" \
-c "ALTER SYSTEM SET synchronous_commit = off" \
-c "ALTER SYSTEM SET full_page_writes = off" \
-c "SELECT pg_reload_conf()" \
|| echo "WARN: durability relax failed; continuing"
# Apply embedded migrations to the fresh test DB, then run the
# full suite (no -short → integration tests execute). -p 1:
# every integration package TRUNCATEs the one shared test DB;
# concurrent package binaries → TRUNCATE deadlocks. Serialize
# package execution (the documented local invocation too).
MINSTREL_DATABASE_URL="$MINSTREL_TEST_DATABASE_URL" go run ./cmd/minstrel migrate
go test -p 1 -race ./...
+44
View File
@@ -0,0 +1,44 @@
name: test-web
# Web SPA: vitest + svelte-check. Runs on push to dev/main only —
# the `pull_request` trigger is intentionally omitted because every
# branch on this repo is local-only (no fork PRs), so the dev push
# fully covers what a PR run would re-execute. Keeping both events
# doubled CI cost on every commit.
on:
push:
branches: [dev, main]
paths:
- 'web/**'
- '.gitea/workflows/test-web.yml'
# Cancel an earlier in-flight run for the same ref when a newer
# commit arrives. With cancel-in-progress, rapid re-pushes don't
# pile up zombie runs.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
runs-on: go-ci
container:
image: git.fabledsword.com/bvandeusen/ci-go:1.26
defaults:
run:
working-directory: web
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install deps
run: npm ci
- name: Type-check + svelte-check
run: npm run check
- name: Vitest
run: npm test
+2 -8
View File
@@ -7,7 +7,7 @@ RUN npm ci
COPY web/ ./
RUN npm run build
FROM golang:1.26-bookworm AS builder
FROM golang:1.25-bookworm AS builder
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
@@ -33,14 +33,8 @@ RUN go build -trimpath \
-o /out/minstrel ./cmd/minstrel
FROM debian:bookworm-slim
# ffmpeg: duration probes and the exact-tier audio hash (a SHA-256 of the
# encoded audio packets, so no decode). libchromaprint-tools: fpcalc, the
# acoustic fingerprint that tells the same recording at two bitrates apart
# from two different recordings (M400). Both are baked in at build time so a
# deployed instance never fetches either (rule 164); fpcalc is shelled out
# rather than bound because CGO_ENABLED=0 above rules out cgo.
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates ffmpeg libchromaprint-tools \
&& apt-get install -y --no-install-recommends ca-certificates ffmpeg \
&& rm -rf /var/lib/apt/lists/*
RUN groupadd --system --gid 1000 minstrel \
+4 -9
View File
@@ -37,12 +37,8 @@ services:
ports: ['4533:4533']
volumes:
# Your music library. Point ./music at wherever your audio files
# live. Writable, because Minstrel deletes a file when an admin asks
# it to (for example, quarantine's "Delete file"). It never moves,
# renames or retags anything. The container runs as uid 1000, so that
# user needs write access to the folders. Mount it :ro to forbid even
# deletes: those actions then refuse, say why, and delete nothing.
- ./music:/music
# live. Mounted read-only — Minstrel never writes to your library.
- ./music:/music:ro
# Generated data: playlist cover collages, artist art, caches.
# The path must match MINSTREL_STORAGE_DATA_DIR, which the image
# sets to /app/data — keep this mount on /app/data or your cache
@@ -51,7 +47,7 @@ services:
environment:
MINSTREL_DATABASE_URL: postgres://minstrel:minstrel@db:5432/minstrel?sslmode=disable
# Colon-separated library roots to scan; must match the container
# path of the music mount above (/music here).
# path of the read-only music mount above (/music here).
MINSTREL_LIBRARY_SCAN_PATHS: /music
depends_on: [db]
@@ -154,8 +150,7 @@ Two concurrent dev processes:
truncates your dev `minstrel` data (admin user, library, likes). It
brings up the compose Postgres and creates the test DB if missing.
- CI runs both: a fast `go test -short -race` gate plus an integration
job with its own ephemeral Postgres (the `integration` lane in
`.gitea/workflows/release.yml`, which also gates every image publish).
job with its own ephemeral Postgres (`.gitea/workflows/test-go.yml`).
### Production build
@@ -15,14 +15,10 @@ import androidx.compose.material3.HorizontalDivider
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedButton
import androidx.compose.material3.Scaffold
import androidx.compose.material3.SnackbarHost
import androidx.compose.material3.SnackbarHostState
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.text.style.TextOverflow
@@ -46,12 +42,6 @@ fun AdminQuarantineScreen(
viewModel: AdminQuarantineViewModel = hiltViewModel(),
) {
val state by viewModel.uiState.collectAsStateWithLifecycle()
val snackbarHostState = remember { SnackbarHostState() }
LaunchedEffect(Unit) {
viewModel.transientMessages.collect { msg ->
snackbarHostState.showSnackbar(msg)
}
}
Scaffold(
contentWindowInsets = ShellContentWindowInsets,
modifier = Modifier.fillMaxSize(),
@@ -63,7 +53,6 @@ fun AdminQuarantineScreen(
onBack = { navController.popBackStack() },
)
},
snackbarHost = { SnackbarHost(snackbarHostState) },
) { inner ->
PullToRefreshScaffold(
onRefresh = { viewModel.refresh().join() },
@@ -10,13 +10,10 @@ import com.fabledsword.minstrel.events.EventsStream
import com.fabledsword.minstrel.models.AdminQuarantineItemRef
import dagger.hilt.android.lifecycle.HiltViewModel
import kotlinx.coroutines.Job
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.filter
import kotlinx.coroutines.flow.receiveAsFlow
import kotlinx.coroutines.launch
import javax.inject.Inject
@@ -37,15 +34,6 @@ class AdminQuarantineViewModel @Inject constructor(
private val internal = MutableStateFlow<AdminQuarantineUiState>(AdminQuarantineUiState.Loading)
val uiState: StateFlow<AdminQuarantineUiState> = internal.asStateFlow()
/**
* One-shot messages for the screen's snackbar. A failed action has to say
* why: the row quietly reappearing reads as a glitch, and for a Delete
* file refused by a read-only library it hides the one thing the
* operator can fix (#3918).
*/
private val transientMessagesChannel = Channel<String>(Channel.BUFFERED)
val transientMessages: Flow<String> = transientMessagesChannel.receiveAsFlow()
init {
refresh()
viewModelScope.launch {
@@ -98,9 +86,8 @@ class AdminQuarantineViewModel @Inject constructor(
try {
action(trackId)
} catch (
@Suppress("TooGenericExceptionCaught") e: Throwable,
@Suppress("TooGenericExceptionCaught", "SwallowedException") e: Throwable,
) {
transientMessagesChannel.trySend(ErrorCopy.fromThrowable(e))
refresh()
}
}
@@ -37,35 +37,18 @@ object ErrorCopy {
* as connection failures.
*/
fun fromThrowable(t: Throwable): String = when (t) {
is HttpException -> fromHttp(t)
is HttpException -> messageFor(codeFromHttp(t))
is IOException -> messageFor("connection_refused")
else -> TABLE.getValue("unknown")
}
/**
* Codes whose server message carries specifics the operator needs in
* order to act — which directory, which uid — that fixed copy cannot say.
* For these the message follows the copy (#3918). Kept to a named set on
* purpose: most server messages are internal detail. Mirrors web's
* errors.ts.
*/
private val DETAIL_CODES = setOf("library_not_writable", "file_delete_failed")
private fun fromHttp(e: HttpException): String {
val body = bodyFromHttp(e)
val copy = messageFor(body.code.ifEmpty { "unknown" })
return if (body.code in DETAIL_CODES && body.message.isNotBlank()) {
"$copy ${body.message}"
} else {
copy
}
}
private fun bodyFromHttp(e: HttpException): Body {
private fun codeFromHttp(e: HttpException): String {
val raw = runCatching { e.response()?.errorBody()?.string() }.getOrNull()
?: return Body()
return runCatching { json.decodeFromString<Envelope>(raw).error }
.getOrNull() ?: Body()
?: return "unknown"
val code = runCatching { json.decodeFromString<Envelope>(raw).error?.code }
.getOrNull()
.orEmpty()
return code.ifEmpty { "unknown" }
}
private val TABLE: Map<String, String> = mapOf(
@@ -75,7 +58,6 @@ object ErrorCopy {
"forbidden" to "You don't have permission to do that.",
"not_authorized" to "You don't have permission to do that.",
"invalid_credentials" to "Wrong username or password.",
"rate_limited" to "Too many attempts. Wait a few minutes and try again.",
"wrong_password" to "Current password is incorrect.",
"password_too_short" to "Password must be at least 8 characters.",
"username_invalid" to "That username isn't valid.",
@@ -117,8 +99,6 @@ object ErrorCopy {
"request_not_pending" to "This request is no longer pending.",
"request_not_found" to "That request no longer exists.",
"track_not_found" to "That track no longer exists.",
"library_not_writable" to "The music library isn't writable by the server.",
"file_delete_failed" to "The file couldn't be deleted.",
"album_not_found" to "That album no longer exists.",
"artist_not_found" to "That artist no longer exists.",
"playlist_not_found" to "That playlist no longer exists.",
@@ -57,7 +57,7 @@ class PasswordViewModel @Inject constructor(
try {
repository.changePassword(current = s.current, next = s.next)
internal.update {
PasswordUiState(message = "Password changed. Your other devices have been signed out.")
PasswordUiState(message = "Password changed.")
}
} catch (
@Suppress("TooGenericExceptionCaught") e: Throwable,
@@ -1,60 +0,0 @@
package com.fabledsword.minstrel.api
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.ResponseBody.Companion.toResponseBody
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
import retrofit2.HttpException
import retrofit2.Response
import java.io.IOException
class ErrorCopyTest {
private fun httpError(status: Int, body: String): HttpException =
HttpException(
Response.error<Unit>(status, body.toResponseBody("application/json".toMediaType())),
)
@Test
fun libraryNotWritableAppendsTheServerDetail() {
val detail = "Minstrel runs as uid 1000, gid 1000 and cannot delete from /music/A " +
"(read-only file system). The library mount must be writable by that user. " +
"Nothing was deleted."
val e = httpError(409, """{"error":{"code":"library_not_writable","message":"$detail"}}""")
assertEquals(
"${ErrorCopy.messageFor("library_not_writable")} $detail",
ErrorCopy.fromThrowable(e),
)
}
@Test
fun detailCodeWithoutAMessageShowsTheCopyAlone() {
val e = httpError(409, """{"error":{"code":"library_not_writable","message":""}}""")
assertEquals(ErrorCopy.messageFor("library_not_writable"), ErrorCopy.fromThrowable(e))
}
// Server messages are usually internal detail; appending them for every
// code would leak driver errors into snackbars. This pins the scope.
@Test
fun otherCodesNeverCarryTheServerMessage() {
val e = httpError(404, """{"error":{"code":"track_not_found","message":"pgx: no rows"}}""")
assertEquals(ErrorCopy.messageFor("track_not_found"), ErrorCopy.fromThrowable(e))
}
@Test
fun anUnparseableBodyFallsBackToUnknown() {
val e = httpError(500, "not json")
assertEquals(ErrorCopy.messageFor("unknown"), ErrorCopy.fromThrowable(e))
}
@Test
fun transportFailureMapsToConnectionRefused() {
assertEquals(
ErrorCopy.messageFor("connection_refused"),
ErrorCopy.fromThrowable(IOException("refused")),
)
}
}
+8 -9
View File
@@ -12,9 +12,8 @@ git.fabledsword.com/bvandeusen/ci-go:1.26
git.fabledsword.com/bvandeusen/ci-android:36
```
- `ci-go:1.26` — the `go`, `integration` and `web` lanes, `release-assets`, and the container build (`image-release`). All CI lives in one workflow, `.gitea/workflows/release.yml`, so every publishing job can depend on every lane (rule 177).
- `golang:1.26-bookworm` (Docker Hub) — the `govulncheck` lane. Deliberately the same image as the Dockerfile's builder stage rather than `ci-go`, so the standard library it checks is the one in the shipped binary. Keep the two in step.
- `ci-android:36` — native Kotlin/Compose client: ktlint + detekt + unit tests + debug APK (the `android` lane), and the signed release APK (`android-release`).
- `ci-go:1.26` — Go server tests (`.gitea/workflows/test-go.yml`), web SPA tests (`.gitea/workflows/test-web.yml`), and the release container build (`release.yml`'s `image-release` job).
- `ci-android:36` — native Kotlin/Compose client: ktlint + detekt + unit tests + debug APK (`.gitea/workflows/android.yml`), and the signed release APK (`release.yml`'s `android-release` job).
**`ci-flutter` is no longer consumed, and `ci-flutter` can now be retired.**
The M8 rewrite replaced the Flutter client with the native Android app and
@@ -31,9 +30,9 @@ split below. The label is a scheduling handle, not a toolchain assertion.
### From `ci-go:1.26`
- **Go** (1.26 toolchain) — `go vet`, `go test -race`, `go build`, `go mod`.
- **Node + npm** — `npm ci`, `npm audit`, `npm test` and `npm run check` in the `web` lane.
- **golangci-lint** — lint pass in the `go` lane.
- **docker CLI** — bridge-IP discovery of the per-job Postgres service container in the `integration` lane (via the runner's shared `/var/run/docker.sock`).
- **Node + npm** — `npm ci` and `npm test` / `npm run check` in `test-web.yml`.
- **golangci-lint** — lint pass in `test-go.yml`.
- **docker CLI** — bridge-IP discovery of the per-job Postgres service container in `test-go.yml` integration job (via the runner's shared `/var/run/docker.sock`).
- **docker buildx** — release container build + push in `release.yml`.
- **curl** — release-asset polling / upload in `release.yml`.
@@ -46,7 +45,7 @@ split below. The label is a scheduling handle, not a toolchain assertion.
No NDK: the native client has no C/C++ sources (this is why it isn't on
`ci-flutter`).
- **ktlint + detekt** — `./gradlew ktlintCheck` and `./gradlew detekt` in
the `android` lane. Image pins track `android/gradle/libs.versions.toml` so local
`android.yml`. Image pins track `android/gradle/libs.versions.toml` so local
and CI checks agree.
- **git** — `actions/checkout@v4` baseline (and any shell git operations).
- **base64 + curl** — keystore decode + release-asset upload in `release.yml`'s
@@ -59,10 +58,10 @@ None.
## Notes
- **Label/image split.** Workflows keep `runs-on: go-ci` / `runs-on: flutter-ci` as the scheduling label per the [`ci-runners.md`](https://…/FabledRulebook/ci-runners.md) "label = scheduling handle, image = `container.image`" pattern. The labels are intentional handles, not toolchain assertions — which is why the Android jobs still schedule on `flutter-ci` while pulling `ci-android:36`. Switch them to `android-ci` if that runner label is ever registered; nothing breaks either way.
- **Integration-job docker-socket dependency.** The `integration` lane uses the runner's shared docker socket (`/var/run/docker.sock`) to bridge-IP-discover the per-job Postgres service container by name + network intersection — the dev compose's `minstrel-postgres-*` containers are explicitly skipped as belt-and-suspenders. Depends on `act_runner.valid_volumes` whitelisting the socket; if that ever stops auto-mounting, integration tests fail at the `docker inspect` step.
- **Integration-job docker-socket dependency.** `test-go.yml`'s integration job uses the runner's shared docker socket (`/var/run/docker.sock`) to bridge-IP-discover the per-job Postgres service container by name + network intersection — the dev compose's `minstrel-postgres-*` containers are explicitly skipped as belt-and-suspenders. Depends on `act_runner.valid_volumes` whitelisting the socket; if that ever stops auto-mounting, integration tests fail at the `docker inspect` step.
- **Go toolchain pin.** `go.mod` is on `go 1.25.0` because `golang.org/x/crypto v0.51.0` declares 1.25 as its minimum. `ci-go:1.26` satisfies this with headroom. Future `x/crypto` bumps that move the Go floor should be paired with an image-tag bump in this file + the workflows.
- **In-app update channel — `needs:`, not polling.** `release.yml`'s `image-release` job declares `needs: [android-release]`, so on tag pushes the signed APK is guaranteed present before the image build starts — no polling window, no race. (The old cross-workflow polling against `flutter.yml` is gone with that workflow.) On non-tag `main` pushes `android-release` is skipped and `image-release` instead pulls the most recent release's APK and reconstructs its exact `versionName`, so `:latest` never ships without an update channel. It degrades to an empty `client/` — never a wrong version — if no release, asset, or tag commit-count can be resolved.
- **Cache server reachability.** The `web` lane does NOT use `cache: 'npm'` on `actions/setup-node` — the Gitea Actions cache server isn't reachable from this runner's container network and `setup-node` was burning ~4m41s on ETIMEDOUT before failing open. With the migration to `ci-go:1.26`, `setup-node` is removed entirely (Node is in the image). The cache concern reappears if a future change re-introduces a network-dependent action.
- **Cache server reachability.** `test-web.yml` does NOT use `cache: 'npm'` on `actions/setup-node` — the Gitea Actions cache server isn't reachable from this runner's container network and `setup-node` was burning ~4m41s on ETIMEDOUT before failing open. With the migration to `ci-go:1.26`, `setup-node` is removed entirely (Node is in the image). The cache concern reappears if a future change re-introduces a network-dependent action.
- **Artifacts — stock `actions/upload-artifact@v7` and `actions/download-artifact@v8`; never `@v3`.**
```yaml
uses: actions/upload-artifact@v7
+1 -29
View File
@@ -122,15 +122,7 @@ func run() error {
}
defer pool.Close()
// Fingerprinting settings (M400 #3913): one instance, shared by the scanner,
// the fingerprint backfill, the duplicate sweep and the admin API, so a save
// reaches all of them without a restart. A load failure is logged, not fatal:
// the service falls back to the shipped defaults.
fpSettings, fpErr := library.NewFingerprintSettingsService(ctx, pool)
if fpErr != nil {
logger.Warn("fingerprint settings: using defaults", "err", fpErr)
}
scanner := library.New(pool, logger, cfg.Library.ScanPaths, fpSettings)
scanner := library.New(pool, logger, cfg.Library.ScanPaths)
contact := cfg.Library.ContactEmail
if contact == "" {
@@ -222,17 +214,6 @@ func run() error {
// SQL, no external calls; empty on single-user servers.
go coplay.NewWorker(pool, logger.With("component", "coplay")).Run(ctx)
// Fingerprint backfill (M400 #3908): fingerprints the tracks the scan never
// will — everything imported before fingerprinting existed, and rows derived
// by an older method. A worker of its own rather than a scan stage; see
// internal/library/fingerprint_backfill.go for why.
go library.NewFingerprintBackfillWorker(pool, logger.With("component", "fingerprint_backfill"), fpSettings).Run(ctx)
// Duplicate sweep (M400 #3910): proposes groups of tracks holding one
// recording, from the fingerprints above. Sweeps only when fingerprints have
// changed since the last sweep.
go library.NewDuplicateSweepWorker(pool, logger.With("component", "duplicate_sweep"), fpSettings).Run(ctx)
// Start the tag-enrichment worker (#1490). Reconciles the compiled-in
// tag providers with tag_provider_settings, bumps the sources version if
// the provider set changed (re-opening settled rows), then drains tracks
@@ -376,20 +357,11 @@ func run() error {
srv.PlaylistScheduler = playlistScheduler
srv.RecSettings = recSettings
srv.TagSettings = tagSettings
srv.FingerprintSettings = fpSettings
// The sweeper above holds this same instance, so a save from the admin
// card changes what it does on its next tick (#3936).
srv.ReacqSettings = reacqSettings
srv.StreamSecret = cfg.StreamSecret
httpServer := &http.Server{
Addr: cfg.Server.Address,
Handler: srv.Router(),
ReadHeaderTimeout: 10 * time.Second,
// Closes keep-alive connections nobody is using. Deliberately no
// ReadTimeout or WriteTimeout: either would cut off audio streams and
// the SSE event stream. Request bodies get their own deadline in the
// server's limitRequestBody middleware instead.
IdleTimeout: 120 * time.Second,
}
errCh := make(chan error, 1)
+1 -1
View File
@@ -8,7 +8,7 @@ version: "3.9"
# tests at it:
# make test-integration
# (CI runs the same suite against its own ephemeral Postgres service —
# see the `integration` job in .gitea/workflows/release.yml.)
# see .gitea/workflows/test-go.yml.)
#
# Full stack (server + db):
# docker compose up --build
+2 -2
View File
@@ -25,7 +25,7 @@ require (
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
golang.org/x/sync v0.21.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.44.0 // indirect
golang.org/x/text v0.39.0 // indirect
golang.org/x/text v0.37.0 // indirect
)
+4 -4
View File
@@ -95,12 +95,12 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus=
golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-45
View File
@@ -1,12 +1,10 @@
package api
import (
"context"
"net/http"
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
)
// coverageRollupResp is the wire shape for GET /api/admin/library/coverage.
@@ -38,46 +36,3 @@ func (h *handlers) handleGetLibraryCoverage(w http.ResponseWriter, r *http.Reque
PendingNoMbid: row.PendingNoMbid,
})
}
// fingerprintCoverageResp is the wire shape for GET /api/admin/library/fingerprints.
// fingerprinted + rejected + pending = total. Missing tracks are not counted:
// there is no file to fingerprint. Enabled travels with the counts because with
// fingerprinting off (#3913) pending never shrinks, and a gauge that implies
// progress would be promising work nothing is doing.
type fingerprintCoverageResp struct {
Total int64 `json:"total"`
Fingerprinted int64 `json:"fingerprinted"`
Rejected int64 `json:"rejected"`
Pending int64 `json:"pending"`
Enabled bool `json:"enabled"`
}
// fingerprintCoverage reads the gauge against the current settings: a print at
// another length counts as pending, because the backfill will re-derive it.
func (h *handlers) fingerprintCoverage(ctx context.Context) (fingerprintCoverageResp, error) {
cfg := h.fingerprintSettings.Get()
row, err := library.FingerprintCoverage(ctx, h.pool, cfg)
if err != nil {
return fingerprintCoverageResp{}, err
}
return fingerprintCoverageResp{
Total: row.Total,
Fingerprinted: row.Fingerprinted,
Rejected: row.Rejected,
Pending: row.Pending,
Enabled: cfg.Enabled,
}, nil
}
// handleGetFingerprintCoverage implements GET /api/admin/library/fingerprints:
// how far the fingerprint backfill (#3908) has got. The backfill is its own
// worker spanning many passes, with no scan run to attach a tally to, so its
// progress is read live here. Always 200; zeros on an empty library.
func (h *handlers) handleGetFingerprintCoverage(w http.ResponseWriter, r *http.Request) {
cov, err := h.fingerprintCoverage(r.Context())
if err != nil {
writeErrWithLog(w, h.logger, "admin: get fingerprint coverage", apierror.InternalMsg("lookup failed", err))
return
}
writeJSON(w, http.StatusOK, cov)
}
-316
View File
@@ -1,316 +0,0 @@
package api
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"github.com/go-chi/chi/v5"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
)
// duplicateMemberView is one copy in a proposed duplicate group. LikeCount and
// PlayCount span every user: the report is admin-only, and what a copy carries
// is the fact the operator weighs when choosing which to keep.
type duplicateMemberView struct {
TrackID string `json:"track_id"`
Title string `json:"title"`
ArtistName string `json:"artist_name"`
AlbumID string `json:"album_id"`
AlbumTitle string `json:"album_title"`
FilePath string `json:"file_path"`
FileFormat string `json:"file_format"`
FileSize int64 `json:"file_size"`
DurationSec int32 `json:"duration_sec"`
AddedAt string `json:"added_at"`
LikeCount int64 `json:"like_count"`
PlayCount int64 `json:"play_count"`
}
// duplicateGroupView is one proposal. SurvivorTrackID and SurvivorReason are
// the copy the report proposes keeping and the rule that chose it
// (library.ProposeSurvivor) — a default the merge (#3911) lets the operator
// override.
type duplicateGroupView struct {
ID string `json:"id"`
Tier string `json:"tier"`
WorstBitErrorRate *float32 `json:"worst_bit_error_rate"`
DetectedAt string `json:"detected_at"`
SurvivorTrackID string `json:"survivor_track_id"`
SurvivorReason string `json:"survivor_reason"`
Members []duplicateMemberView `json:"members"`
}
// duplicateSweepView is the latest sweep. State is "never" when none has run,
// which is what lets the page tell an empty report apart from a sweep that
// found nothing.
type duplicateSweepView struct {
State string `json:"state"`
StartedAt *string `json:"started_at"`
FinishedAt *string `json:"finished_at"`
Candidates *int32 `json:"candidates"`
GroupsFound *int32 `json:"groups_found"`
OversizeClusters *int32 `json:"oversize_clusters"`
ErrorMessage *string `json:"error_message"`
}
// adminDuplicatesResponse is the paged report. Total counts groups.
type adminDuplicatesResponse struct {
Sweep duplicateSweepView `json:"sweep"`
Fingerprints fingerprintCoverageResp `json:"fingerprints"`
Total int64 `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
Groups []duplicateGroupView `json:"groups"`
}
// handleListDuplicates implements GET /api/admin/library/duplicates (#3912).
//
// Read-only. The sweep's state and the fingerprint backfill's progress travel
// with the groups because an empty report means three different things — still
// fingerprinting, never swept, or swept and clean — and the page has to say which.
func (h *handlers) handleListDuplicates(w http.ResponseWriter, r *http.Request) {
limit, offset, err := parsePaging(r.URL.Query())
if err != nil {
writeAdminJSONErr(w, http.StatusBadRequest, "invalid_paging")
return
}
ctx := r.Context()
q := dbq.New(h.pool)
sweep := duplicateSweepView{State: "never"}
last, err := q.GetLatestDuplicateSweep(ctx)
switch {
case err == nil:
sweep = duplicateSweepViewOf(last)
case !errors.Is(err, pgx.ErrNoRows):
h.logger.Error("admin: latest duplicate sweep", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
return
}
cov, err := h.fingerprintCoverage(ctx)
if err != nil {
h.logger.Error("admin: fingerprint coverage", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
return
}
total, err := q.CountPendingDuplicateGroups(ctx)
if err != nil {
h.logger.Error("admin: count duplicate groups", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
return
}
rows, err := q.ListPendingDuplicateGroupMembers(ctx, dbq.ListPendingDuplicateGroupMembersParams{
PageLimit: int32(limit), PageOffset: int32(offset),
})
if err != nil {
h.logger.Error("admin: list duplicate groups", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
return
}
writeJSON(w, http.StatusOK, adminDuplicatesResponse{
Sweep: sweep,
Fingerprints: cov,
Total: total,
Limit: limit,
Offset: offset,
Groups: foldDuplicateGroups(rows),
})
}
func duplicateSweepViewOf(s dbq.DuplicateSweep) duplicateSweepView {
v := duplicateSweepView{
State: "running",
Candidates: s.Candidates,
GroupsFound: s.GroupsFound,
OversizeClusters: s.OversizeClusters,
ErrorMessage: s.ErrorMessage,
}
started := formatTimestamp(s.StartedAt)
v.StartedAt = &started
if s.FinishedAt.Valid {
finished := formatTimestamp(s.FinishedAt)
v.FinishedAt = &finished
v.State = "finished"
}
return v
}
// foldDuplicateGroups folds the one-row-per-member query result into groups and
// proposes each group's survivor. It relies on the query ordering members of a
// group together, so a run-length fold is enough and the page order holds.
func foldDuplicateGroups(rows []dbq.ListPendingDuplicateGroupMembersRow) []duplicateGroupView {
groups := make([]duplicateGroupView, 0, 8)
var candidates [][]library.SurvivorCandidate
for _, row := range rows {
id := uuidToString(row.GroupID)
if n := len(groups); n == 0 || groups[n-1].ID != id {
groups = append(groups, duplicateGroupView{
ID: id,
Tier: row.Tier,
WorstBitErrorRate: row.WorstBitErrorRate,
DetectedAt: formatTimestamp(row.DetectedAt),
})
candidates = append(candidates, nil)
}
n := len(groups) - 1
trackID := uuidToString(row.TrackID)
groups[n].Members = append(groups[n].Members, duplicateMemberView{
TrackID: trackID,
Title: row.Title,
ArtistName: row.ArtistName,
AlbumID: uuidToString(row.AlbumID),
AlbumTitle: row.AlbumTitle,
FilePath: row.FilePath,
FileFormat: row.FileFormat,
FileSize: row.FileSize,
DurationSec: row.DurationMs / 1000,
AddedAt: formatTimestamp(row.AddedAt),
LikeCount: row.LikeCount,
PlayCount: row.PlayCount,
})
candidates[n] = append(candidates[n], library.SurvivorCandidate{
TrackID: trackID, FileFormat: row.FileFormat, FileSize: row.FileSize, AddedAt: row.AddedAt.Time,
})
}
for i := range groups {
groups[i].SurvivorTrackID, groups[i].SurvivorReason = library.ProposeSurvivor(candidates[i])
}
return groups
}
// handleRunDuplicateSweep implements POST /api/admin/library/duplicates/sweep:
// 202 when a sweep starts, 409 sweep_in_progress when one is already running.
// The sweep outlives the request, so it runs on a background context, as
// handleTriggerScan's scan does.
func (h *handlers) handleRunDuplicateSweep(w http.ResponseWriter, _ *http.Request) {
// Runs whatever the sweep interval says: the interval paces the automatic
// sweep, and an operator pressing the button has already decided.
started, err := library.TryStartDuplicateSweep(
context.Background(), h.pool, h.logger.With("source", "manual"), h.fingerprintSettings.Get(),
)
if err != nil {
h.logger.Error("admin: start duplicate sweep", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
return
}
if !started {
writeAdminJSONErr(w, http.StatusConflict, "sweep_in_progress")
return
}
writeJSON(w, http.StatusAccepted, map[string]bool{"started": true})
}
// handleDismissDuplicateGroup implements POST
// /api/admin/library/duplicates/{id}/dismiss: "these are not duplicates". The
// sweep keeps the dismissal and will not propose that set of tracks again. 404
// duplicate_group_not_pending when the group was already resolved or is gone.
func (h *handlers) handleDismissDuplicateGroup(w http.ResponseWriter, r *http.Request) {
id, ok := parseUUID(chi.URLParam(r, "id"))
if !ok {
writeAdminJSONErr(w, http.StatusBadRequest, "invalid_id")
return
}
n, err := dbq.New(h.pool).DismissDuplicateGroup(r.Context(), id)
if err != nil {
h.logger.Error("admin: dismiss duplicate group", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
return
}
if n == 0 {
writeAdminJSONErr(w, http.StatusNotFound, "duplicate_group_not_pending")
return
}
writeJSON(w, http.StatusOK, map[string]string{"status": "dismissed"})
}
// mergeDuplicateRequest chooses the copy to keep. An empty survivor_track_id
// keeps the report's proposal.
type mergeDuplicateRequest struct {
SurvivorTrackID string `json:"survivor_track_id"`
Unmonitor bool `json:"unmonitor"`
}
// mergeDuplicateResponse reports what the merge removed. RemovedPaths are files
// deleted from disk; the operator reads them to know exactly what went.
type mergeDuplicateResponse struct {
SurvivorTrackID string `json:"survivor_track_id"`
RemovedPaths []string `json:"removed_paths"`
LidarrUnmonitorFailed *bool `json:"lidarr_unmonitor_failed,omitempty"`
}
// mergeRequestBodyLimit bounds the request body. It holds one id and a flag.
const mergeRequestBodyLimit = 1 << 16
// handleMergeDuplicateGroup implements POST /api/admin/library/duplicates/{id}/merge
// (#3911): keep one copy, move the others' likes, plays and playlist entries onto
// it, and delete the others' files and rows.
//
// Errors:
// - 409 library_not_writable / 500 file_delete_failed when a file could not be
// removed — nothing was changed (fileRemoveAPIError)
// - 404 duplicate_group_not_pending when the group was already resolved
// - 400 survivor_not_in_group, invalid_id, invalid_body
func (h *handlers) handleMergeDuplicateGroup(w http.ResponseWriter, r *http.Request) {
admin, ok := requireUser(w, r)
if !ok {
return
}
groupID, ok := parseUUID(chi.URLParam(r, "id"))
if !ok {
writeAdminJSONErr(w, http.StatusBadRequest, "invalid_id")
return
}
var body mergeDuplicateRequest
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, mergeRequestBodyLimit)).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
writeAdminJSONErr(w, http.StatusBadRequest, "invalid_body")
return
}
var survivorID pgtype.UUID // invalid: keep the proposal
if body.SurvivorTrackID != "" {
if survivorID, ok = parseUUID(body.SurvivorTrackID); !ok {
writeAdminJSONErr(w, http.StatusBadRequest, "invalid_id")
return
}
}
res, unmonitorFailed, err := h.tracks.MergeDuplicates(r.Context(), groupID, survivorID, admin.ID, body.Unmonitor)
if err != nil {
if apiErr, ok := fileRemoveAPIError(err); ok {
logFileRemoveFailure(h.logger, apiErr, "group_id", uuidToString(groupID))
writeErr(w, apiErr)
return
}
switch {
case errors.Is(err, library.ErrDuplicateGroupNotPending):
writeAdminJSONErr(w, http.StatusNotFound, "duplicate_group_not_pending")
case errors.Is(err, library.ErrSurvivorNotInGroup):
writeAdminJSONErr(w, http.StatusBadRequest, "survivor_not_in_group")
default:
h.logger.Error("admin: merge duplicate group", "group_id", uuidToString(groupID), "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
}
return
}
resp := mergeDuplicateResponse{
SurvivorTrackID: uuidToString(res.Survivor.TrackID),
RemovedPaths: make([]string, 0, len(res.Removed)),
}
for _, c := range res.Removed {
resp.RemovedPaths = append(resp.RemovedPaths, c.FilePath)
}
if body.Unmonitor && unmonitorFailed {
failed := true
resp.LidarrUnmonitorFailed = &failed
}
writeJSON(w, http.StatusOK, resp)
}
-65
View File
@@ -1,65 +0,0 @@
package api
import (
"testing"
"time"
"github.com/jackc/pgx/v5/pgtype"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
func dupUUID(b byte) pgtype.UUID {
var u pgtype.UUID
u.Bytes[15] = b
u.Valid = true
return u
}
func dupTS(t time.Time) pgtype.Timestamptz { return pgtype.Timestamptz{Time: t, Valid: true} }
// Rows arrive one per member, members of a group together. The fold must keep
// groups apart, keep the query's order, and propose each group's survivor from
// its own members only.
func TestFoldDuplicateGroups(t *testing.T) {
older := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
newer := older.Add(48 * time.Hour)
ber := float32(0.04)
rows := []dbq.ListPendingDuplicateGroupMembersRow{
// Group 1: identical audio, sizes tie, the older copy should be kept.
{GroupID: dupUUID(1), Tier: "exact", DetectedAt: dupTS(newer), TrackID: dupUUID(10),
Title: "WWW", FileFormat: "mp3", FileSize: 6_900_000, DurationMs: 215_400, AddedAt: dupTS(newer), PlayCount: 3},
{GroupID: dupUUID(1), Tier: "exact", DetectedAt: dupTS(newer), TrackID: dupUUID(11),
Title: "WWW", FileFormat: "mp3", FileSize: 6_900_000, DurationMs: 215_400, AddedAt: dupTS(older), LikeCount: 1},
// Group 2: the same recording, FLAC against MP3.
{GroupID: dupUUID(2), Tier: "acoustic", WorstBitErrorRate: &ber, DetectedAt: dupTS(older), TrackID: dupUUID(20),
Title: "Lovesick", FileFormat: "mp3", FileSize: 9_000_000, DurationMs: 198_000, AddedAt: dupTS(older)},
{GroupID: dupUUID(2), Tier: "acoustic", WorstBitErrorRate: &ber, DetectedAt: dupTS(older), TrackID: dupUUID(21),
Title: "Lovesick", FileFormat: "flac", FileSize: 30_000_000, DurationMs: 198_000, AddedAt: dupTS(newer)},
}
got := foldDuplicateGroups(rows)
if len(got) != 2 {
t.Fatalf("folded %d groups, want 2", len(got))
}
g1, g2 := got[0], got[1]
if g1.ID != uuidToString(dupUUID(1)) || len(g1.Members) != 2 || g1.WorstBitErrorRate != nil {
t.Fatalf("group 1 = %+v, want the exact pair with no score", g1)
}
if g1.SurvivorTrackID != uuidToString(dupUUID(11)) || g1.SurvivorReason != "in the library longest" {
t.Errorf("group 1 survivor = (%s, %q), want the older copy", g1.SurvivorTrackID, g1.SurvivorReason)
}
if g1.Members[0].DurationSec != 215 || g1.Members[0].PlayCount != 3 || g1.Members[1].LikeCount != 1 {
t.Errorf("group 1 members lost their facts: %+v", g1.Members)
}
if g2.Tier != "acoustic" || g2.WorstBitErrorRate == nil || *g2.WorstBitErrorRate != ber {
t.Fatalf("group 2 = %+v, want the acoustic pair with its score", g2)
}
// Chosen from group 2's own members: a survivor leaking across groups is
// exactly what a wrong fold boundary would produce.
if g2.SurvivorTrackID != uuidToString(dupUUID(21)) || g2.SurvivorReason != "lossless (flac)" {
t.Errorf("group 2 survivor = (%s, %q), want the FLAC copy", g2.SurvivorTrackID, g2.SurvivorReason)
}
}
@@ -1,67 +0,0 @@
package api
import (
"encoding/json"
"errors"
"net/http"
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
)
// fingerprintSettingsBody is the wire shape for GET and PUT
// /api/admin/library/fingerprint-settings (M400 #3913). The threshold travels as
// the bit-error rate the matcher uses; the card presents it as a match percentage.
type fingerprintSettingsBody struct {
Enabled bool `json:"enabled"`
ChromaprintLengthSec int32 `json:"chromaprint_length_sec"`
AcousticMaxBitErrorRate float64 `json:"acoustic_max_bit_error_rate"`
BackfillConcurrency int32 `json:"backfill_concurrency"`
SweepIntervalHours int32 `json:"sweep_interval_hours"`
}
func fingerprintSettingsBodyOf(s library.FingerprintSettings) fingerprintSettingsBody {
return fingerprintSettingsBody{
Enabled: s.Enabled,
ChromaprintLengthSec: s.ChromaprintLengthSec,
AcousticMaxBitErrorRate: s.AcousticMaxBitErrorRate,
BackfillConcurrency: s.BackfillConcurrency,
SweepIntervalHours: s.SweepIntervalHours,
}
}
// handleGetFingerprintSettings implements GET /api/admin/library/fingerprint-settings.
func (h *handlers) handleGetFingerprintSettings(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, fingerprintSettingsBodyOf(h.fingerprintSettings.Get()))
}
// handleUpdateFingerprintSettings implements PUT /api/admin/library/fingerprint-settings.
//
// A whole-row write. A body that leaves a field out decodes it as zero, which no
// field accepts, so a partial save is refused rather than zeroing what it omitted.
// The saved settings reach the scanner and both workers at once: they share the
// service instance.
func (h *handlers) handleUpdateFingerprintSettings(w http.ResponseWriter, r *http.Request) {
var req fingerprintSettingsBody
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeErr(w, apierror.BadRequest("invalid_body", "malformed JSON"))
return
}
saved, err := h.fingerprintSettings.Set(r.Context(), library.FingerprintSettings{
Enabled: req.Enabled,
ChromaprintLengthSec: req.ChromaprintLengthSec,
AcousticMaxBitErrorRate: req.AcousticMaxBitErrorRate,
BackfillConcurrency: req.BackfillConcurrency,
SweepIntervalHours: req.SweepIntervalHours,
})
if err != nil {
// Validation mirrors migration 0061's CHECKs and names the field.
if errors.Is(err, library.ErrFingerprintSettingOutOfRange) {
writeErr(w, apierror.BadRequest("invalid_setting", err.Error()))
return
}
writeErrWithLog(w, h.logger, "admin fingerprint settings: update failed", apierror.Internal(err))
return
}
writeJSON(w, http.StatusOK, fingerprintSettingsBodyOf(saved))
}
@@ -1,55 +0,0 @@
package api
import (
"encoding/json"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
)
func TestGetFingerprintSettings_ServesDefaultsWithoutAService(t *testing.T) {
h := &handlers{logger: slog.New(slog.NewTextHandler(io.Discard, nil))}
rec := httptest.NewRecorder()
h.handleGetFingerprintSettings(rec, httptest.NewRequest(http.MethodGet, "/api/admin/library/fingerprint-settings", nil))
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
}
var got fingerprintSettingsBody
if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
if want := fingerprintSettingsBodyOf(library.DefaultFingerprintSettings); got != want {
t.Fatalf("body = %+v, want the defaults %+v", got, want)
}
}
func TestUpdateFingerprintSettings_Rejects(t *testing.T) {
h := &handlers{logger: slog.New(slog.NewTextHandler(io.Discard, nil))}
for name, tc := range map[string]struct {
body string
code string
mentions string
}{
"a value out of range, naming the field": {
body: `{"enabled":true,"chromaprint_length_sec":5,"acoustic_max_bit_error_rate":0.15,"backfill_concurrency":2,"sweep_interval_hours":1}`,
code: "invalid_setting",
mentions: "chromaprint_length_sec",
},
// A partial body would otherwise zero every field it left out.
"a body missing fields": {body: `{"enabled":false}`, code: "invalid_setting"},
"malformed JSON": {body: `{"enabled":`, code: "invalid_body"},
} {
rec := httptest.NewRecorder()
h.handleUpdateFingerprintSettings(rec, httptest.NewRequest(
http.MethodPut, "/api/admin/library/fingerprint-settings", strings.NewReader(tc.body)))
body := rec.Body.String()
if rec.Code != http.StatusBadRequest || !strings.Contains(body, `"`+tc.code+`"`) || !strings.Contains(body, tc.mentions) {
t.Errorf("%s: status %d body %s; want 400 %s mentioning %q", name, rec.Code, body, tc.code, tc.mentions)
}
}
}
+7 -38
View File
@@ -23,16 +23,10 @@ type networkSettingsResp struct {
// arrived and count them rather than guess.
ForwardedChain string `json:"forwarded_chain"`
RemoteAddr string `json:"remote_addr"`
// PublicURL is where users reach Minstrel; reset emails link to it and
// are not sent while it is empty.
PublicURL string `json:"public_url"`
}
// Both fields are optional so the proxy card and the public-address card can
// each save their own value without overwriting the other's.
type updateNetworkSettingsReq struct {
TrustedProxyHops *int `json:"trusted_proxy_hops"`
PublicURL *string `json:"public_url"`
TrustedProxyHops int `json:"trusted_proxy_hops"`
}
func (h *handlers) handleGetNetworkSettings(w http.ResponseWriter, r *http.Request) {
@@ -45,38 +39,14 @@ func (h *handlers) handleUpdateNetworkSettings(w http.ResponseWriter, r *http.Re
writeErr(w, apierror.BadRequest("invalid_body", "malformed JSON"))
return
}
if req.TrustedProxyHops == nil && req.PublicURL == nil {
writeErr(w, apierror.BadRequest("invalid_body", "nothing to update"))
if err := h.netSettings.SetHops(r.Context(), req.TrustedProxyHops); err != nil {
if errors.Is(err, netsettings.ErrHopsOutOfRange) {
writeErr(w, apierror.BadRequest("invalid_hops", err.Error()))
return
}
writeErrWithLog(w, h.logger, "admin network: update failed", apierror.Internal(err))
return
}
// Validate everything before writing anything, so a bad public URL can't
// leave the hops half-saved.
if req.TrustedProxyHops != nil && (*req.TrustedProxyHops < 0 || *req.TrustedProxyHops > netsettings.MaxTrustedProxyHops) {
writeErr(w, apierror.BadRequest("invalid_hops", netsettings.ErrHopsOutOfRange.Error()))
return
}
if req.PublicURL != nil {
if _, err := netsettings.NormalizePublicURL(*req.PublicURL); err != nil {
writeErr(w, apierror.BadRequest("invalid_public_url", err.Error()))
return
}
}
if req.TrustedProxyHops != nil {
if err := h.netSettings.SetHops(r.Context(), *req.TrustedProxyHops); err != nil {
if errors.Is(err, netsettings.ErrHopsOutOfRange) {
writeErr(w, apierror.BadRequest("invalid_hops", err.Error()))
return
}
writeErrWithLog(w, h.logger, "admin network: update failed", apierror.Internal(err))
return
}
}
if req.PublicURL != nil {
if err := h.netSettings.SetPublicURL(r.Context(), *req.PublicURL); err != nil {
writeErrWithLog(w, h.logger, "admin network: public URL update failed", apierror.Internal(err))
return
}
}
// Echo the payload recomputed under the NEW value, so the card can show
// immediately what the change did to this request's own address rather
// than making the operator reload to find out.
@@ -91,6 +61,5 @@ func (h *handlers) networkSettingsPayload(r *http.Request) networkSettingsResp {
DetectedClientIP: auth.ClientIP(r, hops),
ForwardedChain: r.Header.Get("X-Forwarded-For"),
RemoteAddr: r.RemoteAddr,
PublicURL: h.netSettings.PublicURL(),
}
}
-7
View File
@@ -133,13 +133,6 @@ func (h *handlers) handleDeleteQuarantineFile(w http.ResponseWriter, r *http.Req
}
action, err := h.lidarrQuarantine.DeleteFile(r.Context(), id, admin.ID)
if err != nil {
// Written in the enveloped shape, not writeAdminJSONErr's bare code: the
// message is the part that tells the operator which directory and uid.
if apiErr, ok := fileRemoveAPIError(err); ok {
logFileRemoveFailure(h.logger, apiErr, "track_id", uuidToString(id))
writeErr(w, apiErr)
return
}
switch {
case errors.Is(err, lidarrquarantine.ErrTrackNotFound):
writeAdminJSONErr(w, http.StatusNotFound, "track_not_found")
+1 -1
View File
@@ -69,7 +69,7 @@ func installQuarantineClientFn(t *testing.T, h *handlers) {
}
return lidarr.NewClient(c.BaseURL, c.APIKey)
}
h.lidarrQuarantine = lidarrquarantine.NewService(h.pool, cfg, clientFn, h.dataDir)
h.lidarrQuarantine = lidarrquarantine.NewService(h.pool, cfg, clientFn)
}
// flagDirect bypasses the HTTP handler to seed a quarantine row via the
+4 -11
View File
@@ -23,17 +23,15 @@ type removeTrackResponse struct {
// handleRemoveTrack implements DELETE /api/admin/tracks/{id}?unmonitor=true|false.
//
// Admin-only (gated by auth.RequireAdmin on the /admin route group). Deletes the
// file, then the DB row, and runs the album/artist cascade tidy-up — and deletes
// nothing at all when the file cannot be removed (#3918). When
// Admin-only (gated by auth.RequireAdmin on the /admin route group). Always
// deletes the file + DB row and runs the album/artist cascade tidy-up. When
// unmonitor=true and the track has an mbid, also calls Lidarr.UnmonitorTrack
// — failure there is non-fatal (the destructive part already completed) and
// surfaces as `lidarr_unmonitor_failed: true` in the success envelope.
//
// Per spec §5, Lidarr-side errors during the unmonitor step do NOT map to
// wire error codes. The codes this handler emits are not_found,
// library_not_writable (409) and file_delete_failed when the file could not be
// removed, server_error, plus the auth codes the middleware emits upstream.
// wire error codes; the only error codes this handler emits are not_found,
// server_error, plus the auth codes the middleware emits upstream.
func (h *handlers) handleRemoveTrack(w http.ResponseWriter, r *http.Request) {
idStr := chi.URLParam(r, "id")
trackID, ok := parseUUID(idStr)
@@ -68,11 +66,6 @@ func (h *handlers) handleRemoveTrack(w http.ResponseWriter, r *http.Request) {
writeErr(w, &apierror.Error{Status: http.StatusNotFound, Code: "not_found", Message: "track not found"})
return
}
if apiErr, ok := fileRemoveAPIError(err); ok {
logFileRemoveFailure(h.logger, apiErr, "track_id", idStr)
writeErr(w, apiErr)
return
}
h.logger.Error("api: remove track failed", "err", err, "track_id", idStr)
writeErr(w, apierror.InternalMsg("remove failed", err))
return
+1 -14
View File
@@ -169,7 +169,7 @@ func (h *handlers) handleAdminCreateUser(w http.ResponseWriter, r *http.Request)
user, err := q.CreateUserAdmin(r.Context(), dbq.CreateUserAdminParams{
Username: req.Username,
PasswordHash: string(hash),
ApiTokenHash: auth.HashAPIToken(apiToken),
ApiToken: apiToken,
IsAdmin: req.IsAdmin,
DisplayName: req.DisplayName,
})
@@ -285,19 +285,6 @@ func (h *handlers) handleAdminResetPassword(w http.ResponseWriter, r *http.Reque
return
}
// The target's existing sessions end with the old password. An admin
// resetting their own password keeps the session they're using, the
// same as a self-service change.
sessID, ownSession := auth.SessionIDFromContext(r.Context())
if targetID == caller.ID && ownSession {
_, err = q.DeleteOtherSessionsForUser(r.Context(), dbq.DeleteOtherSessionsForUserParams{UserID: targetID, ID: sessID})
} else {
_, err = q.DeleteSessionsForUser(r.Context(), targetID)
}
if err != nil {
h.logger.Error("admin reset password: revoke sessions failed", "err", err)
}
audit.WriteOrLog(r.Context(), h.pool, h.logger, caller.ID, targetID, audit.ActionPasswordResetAdmin, nil)
w.WriteHeader(http.StatusNoContent)
+22 -66
View File
@@ -7,7 +7,6 @@ package api
import (
"log/slog"
"math/rand"
"time"
"github.com/go-chi/chi/v5"
"github.com/jackc/pgx/v5/pgxpool"
@@ -34,51 +33,35 @@ import (
// Mount attaches /api/* handlers to r. Public endpoints (login) are outside
// RequireUser; everything else is gated by the middleware. The events writer
// is shared with the Subsonic mount so /rest/scrobble feeds the same store.
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, lidarrCfg *lidarrconfig.Service, lidarrReqs *lidarrrequests.Service, lidarrQuar *lidarrquarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService, fpSettings *library.FingerprintSettingsService) {
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, lidarrCfg *lidarrconfig.Service, lidarrReqs *lidarrrequests.Service, lidarrQuar *lidarrquarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService) {
rng := rand.New(rand.NewSource(rand.Int63()))
setupToken, err := auth.NewSetupToken()
if err != nil {
// crypto/rand failing means the platform can't make secrets at all;
// sessions would be minted from the same source. Nothing to degrade to.
panic("api: mint setup token: " + err.Error())
}
logSetupTokenIfNeeded(pool, logger, setupToken)
h := &handlers{
pool: pool, logger: logger, events: events, recCfg: recCfg,
recSettings: recSettings,
rng: rng.Float64,
lidarrCfg: lidarrCfg,
lidarrRequests: lidarrReqs,
lidarrQuarantine: lidarrQuar,
tracks: tracksSvc,
playlists: playlistsSvc,
coverart: coverEnricher,
coverSettings: coverSettings,
tagSettings: tagSettings,
scanner: scanner,
scanCfg: scanCfg,
dataDir: dataDir,
mailer: sender,
eventbus: bus,
playlistScheduler: playlistScheduler,
streamSecret: streamSecret,
netSettings: netSettings,
reacqSettings: reacqSettings,
fingerprintSettings: fpSettings,
librarySize: recommendation.NewLibrarySize(nil),
loginGuard: auth.NewLoginGuard(),
setupToken: setupToken,
requireSetupToken: true,
registerLimit: auth.NewAttemptLimiter(registerPerAddressMax, time.Hour),
forgotAddressLimit: auth.NewAttemptLimiter(forgotPerAddressMax, time.Hour),
forgotEmailLimit: auth.NewAttemptLimiter(forgotPerEmailMax, time.Hour),
resetLimit: auth.NewAttemptLimiter(resetFailuresPerAddressMax, 15*time.Minute),
recSettings: recSettings,
rng: rng.Float64,
lidarrCfg: lidarrCfg,
lidarrRequests: lidarrReqs,
lidarrQuarantine: lidarrQuar,
tracks: tracksSvc,
playlists: playlistsSvc,
coverart: coverEnricher,
coverSettings: coverSettings,
tagSettings: tagSettings,
scanner: scanner,
scanCfg: scanCfg,
dataDir: dataDir,
mailer: sender,
eventbus: bus,
playlistScheduler: playlistScheduler,
streamSecret: streamSecret,
netSettings: netSettings,
reacqSettings: reacqSettings,
librarySize: recommendation.NewLibrarySize(nil),
}
r.Route("/api", func(api chi.Router) {
api.Post("/auth/login", h.handleLogin)
api.Post("/auth/register", h.handleRegister)
api.Get("/auth/setup-status", h.handleSetupStatus)
api.Post("/auth/forgot-password", h.handleForgotPassword)
api.Post("/auth/reset-password", h.handleResetPassword)
@@ -108,6 +91,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
authed.Put("/me/password", h.handleChangePassword)
authed.Put("/me/profile", h.handleUpdateMyProfile)
authed.Put("/me/timezone", h.handlePutTimezone)
authed.Get("/me/api-token", h.handleGetMyAPIToken)
authed.Post("/me/api-token", h.handleRegenerateMyAPIToken)
authed.Get("/me/sessions", h.handleListMySessions)
authed.Delete("/me/sessions/{id}", h.handleRevokeMySession)
@@ -231,16 +215,6 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
admin.Get("/library/missing", h.handleListMissingTracks)
admin.Get("/library/coverage", h.handleGetLibraryCoverage)
admin.Get("/library/fingerprints", h.handleGetFingerprintCoverage)
admin.Get("/library/fingerprint-settings", h.handleGetFingerprintSettings)
admin.Put("/library/fingerprint-settings", h.handleUpdateFingerprintSettings)
// Duplicates report (#3912): proposals from the duplicate sweep, a
// trigger to sweep now, dismissal, and the merge (#3911), which deletes
// the removed copies' files after moving their history onto the kept one.
admin.Get("/library/duplicates", h.handleListDuplicates)
admin.Post("/library/duplicates/sweep", h.handleRunDuplicateSweep)
admin.Post("/library/duplicates/{id}/dismiss", h.handleDismissDuplicateGroup)
admin.Post("/library/duplicates/{id}/merge", h.handleMergeDuplicateGroup)
admin.Get("/invites", h.handleListInvites)
admin.Post("/invites", h.handleCreateInvite)
@@ -324,24 +298,6 @@ type handlers struct {
// missing files (milestone #290) — grace window, backoff, attempt caps.
// Cached in the service, so the admin card reads it without a query.
reacqSettings *reacquisition.SettingsService
// fingerprintSettings is the fingerprinting policy (M400 #3913), the same
// instance the scanner and the fingerprint workers read, so a save from the
// admin card reaches them without a restart. Nil serves the defaults.
fingerprintSettings *library.FingerprintSettingsService
// setupToken must accompany the first registration while no users exist
// (see auth.SetupToken). requireSetupToken is set by Mount, the only
// production constructor; tests that build handlers directly leave it
// off unless they are testing it.
setupToken *auth.SetupToken
requireSetupToken bool
// loginGuard throttles failed logins per account and per address, and
// the limiters below cap the other unauthenticated auth routes. All are
// nil-safe, so tests that build handlers directly run unthrottled.
loginGuard *auth.LoginGuard
registerLimit *auth.AttemptLimiter
forgotAddressLimit *auth.AttemptLimiter
forgotEmailLimit *auth.AttemptLimiter
resetLimit *auth.AttemptLimiter
// netSettings caches the trusted reverse-proxy depth read by the auth
// middleware on every request and edited from the admin network card.
netSettings *netsettings.Service
+2 -30
View File
@@ -19,20 +19,6 @@ import (
// so an abandoned laptop doesn't stay logged in forever.
const sessionCookieMaxAge = 30 * 24 * time.Hour
// Limits on the unauthenticated auth routes other than login (which uses
// auth.LoginGuard). Per address, per window as wired in Mount.
const (
// registerPerAddressMax: 10 registrations an hour. Every attempt counts,
// typos included, which is still far above a household's need.
registerPerAddressMax = 10
// forgotPerAddressMax / forgotPerEmailMax: reset emails an hour. The
// per-email cap is what keeps one inbox from being mailbombed.
forgotPerAddressMax = 5
forgotPerEmailMax = 3
// resetFailuresPerAddressMax: wrong or expired reset tokens per 15 min.
resetFailuresPerAddressMax = 20
)
func (h *handlers) handleLogout(w http.ResponseWriter, r *http.Request) {
// The session token can be on the cookie OR bearer header — RequireUser
// accepted either. Re-resolve it here so we can delete the row.
@@ -84,22 +70,10 @@ func (h *handlers) handleLogin(w http.ResponseWriter, r *http.Request) {
return
}
// Checked before any lookup or bcrypt, so a throttled guess costs the
// server nothing and learns nothing.
addr := auth.ClientIP(r, h.netSettings.Hops())
if blocked, wait := h.loginGuard.Blocked(req.Username, addr); blocked {
writeRateLimited(w, wait)
return
}
q := dbq.New(h.pool)
user, err := q.GetUserByUsername(r.Context(), req.Username)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
// Same bcrypt time as a wrong password, so timing doesn't say
// which usernames exist.
auth.DummyVerify(req.Password)
h.loginGuard.Fail(req.Username, addr)
writeErr(w, apierror.Unauthorized("invalid_credentials", "invalid username or password"))
return
}
@@ -108,11 +82,9 @@ func (h *handlers) handleLogin(w http.ResponseWriter, r *http.Request) {
return
}
if !auth.VerifyPassword(user.PasswordHash, req.Password) {
h.loginGuard.Fail(req.Username, addr)
writeErr(w, apierror.Unauthorized("invalid_credentials", "invalid username or password"))
return
}
h.loginGuard.Succeed(req.Username)
token, err := auth.MintSessionToken()
if err != nil {
@@ -128,7 +100,7 @@ func (h *handlers) handleLogin(w http.ResponseWriter, r *http.Request) {
// the active-sessions surface: a session that was born somewhere the
// user recognises but is being used from somewhere they don't is the
// case this whole surface exists to surface.
Ip: addr,
Ip: auth.ClientIP(r, h.netSettings.Hops()),
}); err != nil {
h.logger.Error("api: insert session failed", "err", err)
writeErr(w, apierror.InternalMsg("insert failed", err))
@@ -140,7 +112,7 @@ func (h *handlers) handleLogin(w http.ResponseWriter, r *http.Request) {
Value: token,
Path: "/",
HttpOnly: true,
Secure: auth.IsHTTPS(r, h.netSettings.Hops()), // plain HTTP and LAN stay functional (rule 94)
Secure: r.TLS != nil, // dev over http stays functional; prod over https gets Secure
SameSite: http.SameSiteStrictMode,
MaxAge: int(sessionCookieMaxAge.Seconds()),
})
+12 -33
View File
@@ -14,7 +14,6 @@ import (
"github.com/jackc/pgx/v5/pgtype"
"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
)
@@ -48,19 +47,6 @@ func (h *handlers) handleForgotPassword(w http.ResponseWriter, r *http.Request)
}
email := strings.ToLower(strings.TrimSpace(req.Email))
// Throttled per address (a spray) and per email (a mailbomb of one
// inbox). Applied whether or not the email matches, so a 429 says
// nothing about which addresses are registered.
addr := auth.ClientIP(r, h.netSettings.Hops())
blockedAddr, waitAddr := h.forgotAddressLimit.Blocked(addr)
blockedEmail, waitEmail := h.forgotEmailLimit.Blocked(email)
if blockedAddr || blockedEmail {
writeRateLimited(w, max(waitAddr, waitEmail))
return
}
h.forgotAddressLimit.Record(addr)
h.forgotEmailLimit.Record(email)
q := dbq.New(h.pool)
var matched bool
var auditTarget pgtype.UUID
@@ -69,7 +55,7 @@ func (h *handlers) handleForgotPassword(w http.ResponseWriter, r *http.Request)
if err == nil && user.Email != nil && *user.Email != "" {
matched = true
auditTarget = user.ID
if sendErr := h.sendResetEmail(r.Context(), user); sendErr != nil {
if sendErr := h.sendResetEmail(r.Context(), r, user); sendErr != nil {
h.logger.Warn("forgot-password: send failed",
"email", email, "err", sendErr)
// fall through; response is still 200
@@ -89,7 +75,7 @@ func (h *handlers) handleForgotPassword(w http.ResponseWriter, r *http.Request)
// sendResetEmail generates a token, persists it, renders + sends the
// email. Returns the underlying error (caller logs it but doesn't
// surface to the HTTP response).
func (h *handlers) sendResetEmail(ctx context.Context, user dbq.User) error {
func (h *handlers) sendResetEmail(ctx context.Context, r *http.Request, user dbq.User) error {
if h.mailer == nil {
return errors.New("forgot-password: no mailer configured")
}
@@ -109,10 +95,7 @@ func (h *handlers) sendResetEmail(ctx context.Context, user dbq.User) error {
return err
}
resetURL, err := buildResetURL(h.netSettings.PublicURL(), token)
if err != nil {
return err
}
resetURL := buildResetURL(r, token)
textBody, htmlBody, err := mailer.RenderResetEmail(mailer.ResetEmailVars{
Username: user.Username,
ResetURL: resetURL,
@@ -130,18 +113,14 @@ func (h *handlers) sendResetEmail(ctx context.Context, user dbq.User) error {
return h.mailer.Send(ctx, *user.Email, mailer.ResetEmailSubject, textBody, htmlBody)
}
// errNoPublicURL stops a reset email from going out before the operator has
// said where Minstrel lives. It surfaces in the log, not the response, which
// stays the same opaque 200 either way.
var errNoPublicURL = errors.New("forgot-password: no public URL set (Admin → Integrations → Public address); reset email not sent")
// buildResetURL builds the emailed link from the operator-set public URL.
// It deliberately ignores the request: the Host header is whatever the
// requester sent, and building from it let a forged Host plant a real reset
// token on a link to someone else's server.
func buildResetURL(publicURL, token string) (string, error) {
if publicURL == "" {
return "", errNoPublicURL
func buildResetURL(r *http.Request, token string) string {
scheme := "http"
if r.TLS != nil {
scheme = "https"
}
return publicURL + "/reset-password/" + token, nil
host := r.Host
if host == "" {
host = "localhost"
}
return scheme + "://" + host + "/reset-password/" + token
}
+1 -57
View File
@@ -7,59 +7,11 @@ import (
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
"git.fabledsword.com/bvandeusen/minstrel/internal/netsettings"
)
// withPublicURL gives h a network-settings service holding url, restoring an
// empty value afterwards so other tests see the default.
func withPublicURL(t *testing.T, h *handlers, pool *pgxpool.Pool, url string) {
t.Helper()
ns, err := netsettings.New(context.Background(), pool, nil)
if err != nil {
t.Fatalf("netsettings: %v", err)
}
if err := ns.SetPublicURL(context.Background(), url); err != nil {
t.Fatalf("set public url: %v", err)
}
t.Cleanup(func() { _ = ns.SetPublicURL(context.Background(), "") })
h.netSettings = ns
}
// With no public URL set, a reset email is not sent at all, and the response
// is still the same opaque 200.
func TestForgotPassword_NoPublicURL_SendsNothing(t *testing.T) {
if os.Getenv("MINSTREL_TEST_DATABASE_URL") == "" {
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
}
h, pool := testHandlers(t)
fake := &mailer.FakeSender{}
h.mailer = fake
withPublicURL(t, h, pool, "")
user := seedUser(t, pool, "nourl", "pw", false)
if _, err := pool.Exec(context.Background(),
"UPDATE users SET email = 'nourl@example.com' WHERE id = $1", user.ID); err != nil {
t.Fatalf("seed email: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/api/auth/forgot-password",
bytes.NewReader([]byte(`{"email":"nourl@example.com"}`)))
rec := httptest.NewRecorder()
h.handleForgotPassword(rec, req)
if rec.Code != http.StatusOK {
t.Errorf("status = %d, want 200", rec.Code)
}
if len(fake.Sent) != 0 {
t.Errorf("sent %d emails with no public URL set, want 0", len(fake.Sent))
}
}
func TestForgotPassword_KnownEmail_SendsMail(t *testing.T) {
if os.Getenv("MINSTREL_TEST_DATABASE_URL") == "" {
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
@@ -67,7 +19,6 @@ func TestForgotPassword_KnownEmail_SendsMail(t *testing.T) {
h, pool := testHandlers(t)
fake := &mailer.FakeSender{}
h.mailer = fake
withPublicURL(t, h, pool, "https://music.example.com")
user := seedUser(t, pool, "forgotuser", "pw", false)
if _, err := pool.Exec(context.Background(),
@@ -78,8 +29,7 @@ func TestForgotPassword_KnownEmail_SendsMail(t *testing.T) {
body := `{"email":"forgot@example.com"}`
req := httptest.NewRequest(http.MethodPost, "/api/auth/forgot-password",
bytes.NewReader([]byte(body)))
// A forged Host must not reach the link: it comes from the public URL.
req.Host = "attacker.example.net"
req.Host = "minstrel.example.com"
rec := httptest.NewRecorder()
h.handleForgotPassword(rec, req)
@@ -93,12 +43,6 @@ func TestForgotPassword_KnownEmail_SendsMail(t *testing.T) {
if got.To != "forgot@example.com" {
t.Errorf("To = %q, want forgot@example.com", got.To)
}
if !strings.Contains(got.TextBody, "https://music.example.com/reset-password/") {
t.Errorf("reset link not built from the public URL:\n%s", got.TextBody)
}
if strings.Contains(got.TextBody+got.HTMLBody, "attacker.example.net") {
t.Error("the request's Host header reached the emailed link")
}
// Token row was inserted.
var tokenCount int
if err := pool.QueryRow(context.Background(),
+6 -62
View File
@@ -1,10 +1,8 @@
package api
import (
"context"
"encoding/json"
"errors"
"log/slog"
"net/http"
"regexp"
"time"
@@ -12,7 +10,6 @@ import (
"github.com/jackc/pgerrcode"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"golang.org/x/crypto/bcrypt"
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
@@ -29,12 +26,9 @@ var usernameRe = regexp.MustCompile(`^[a-zA-Z0-9_-]{3,32}$`)
const minPasswordLength = 8
type registerReq struct {
Username string `json:"username"`
Password string `json:"password"`
InviteToken string `json:"invite_token"`
// SetupToken is required only for the very first account; see
// auth.SetupToken.
SetupToken string `json:"setup_token"`
Username string `json:"username"`
Password string `json:"password"`
InviteToken string `json:"invite_token"`
DisplayName *string `json:"display_name"`
}
@@ -61,15 +55,6 @@ type registerReq struct {
// - 409 username_taken (PG unique violation on users.username)
// - 500 server_error otherwise
func (h *handlers) handleRegister(w http.ResponseWriter, r *http.Request) {
// Every attempt counts, not only failures: a successful registration is
// the thing being sprayed when the mode is open.
addr := auth.ClientIP(r, h.netSettings.Hops())
if blocked, wait := h.registerLimit.Blocked(addr); blocked {
writeRateLimited(w, wait)
return
}
h.registerLimit.Record(addr)
var req registerReq
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeErr(w, apierror.BadRequest("invalid_body", "invalid JSON body"))
@@ -94,19 +79,6 @@ func (h *handlers) handleRegister(w http.ResponseWriter, r *http.Request) {
return
}
// The first account becomes admin, so it must prove it can read the
// server log. Checked before the invite logic, which the empty-users
// state skips.
if userCount == 0 && h.requireSetupToken && !h.setupToken.Matches(req.SetupToken) {
// Repeat the token in the log at the moment someone needs it: the
// boot line may have scrolled away, or users may have been deleted
// since boot.
h.logger.Warn("register: first-admin registration needs the setup token",
"setup_token", h.setupToken.Value())
writeErr(w, apierror.Forbidden("setup_token_invalid", "the setup token from the server log is required to create the first account"))
return
}
// Validate invite (skipped on empty-users state; skipped in 'open' mode).
usedInviteToken := ""
if userCount > 0 {
@@ -161,7 +133,7 @@ func (h *handlers) handleRegister(w http.ResponseWriter, r *http.Request) {
user, err := q.CreateUserFirstAdminRace(r.Context(), dbq.CreateUserFirstAdminRaceParams{
Username: req.Username,
PasswordHash: string(hash),
ApiTokenHash: auth.HashAPIToken(apiToken),
ApiToken: apiToken,
DisplayName: req.DisplayName,
})
if err != nil {
@@ -203,7 +175,7 @@ func (h *handlers) handleRegister(w http.ResponseWriter, r *http.Request) {
UserID: user.ID,
TokenHash: auth.HashSessionToken(sessionToken),
UserAgent: r.UserAgent(),
Ip: addr,
Ip: auth.ClientIP(r, h.netSettings.Hops()),
}); err != nil {
h.logger.Error("register: insert session failed", "err", err)
writeErr(w, apierror.Internal(err))
@@ -214,7 +186,7 @@ func (h *handlers) handleRegister(w http.ResponseWriter, r *http.Request) {
Value: sessionToken,
Path: "/",
HttpOnly: true,
Secure: auth.IsHTTPS(r, h.netSettings.Hops()), // plain HTTP and LAN stay functional (rule 94)
Secure: r.TLS != nil,
SameSite: http.SameSiteStrictMode,
MaxAge: int(sessionCookieMaxAge.Seconds()),
})
@@ -251,31 +223,3 @@ func (h *handlers) handleRegister(w http.ResponseWriter, r *http.Request) {
},
})
}
// handleSetupStatus implements GET /api/auth/setup-status. It tells the
// register screen whether to ask for the setup token, i.e. whether no
// account exists yet. Public, since it is needed before anyone can sign in;
// "this server has no users" is not worth hiding from someone who could
// simply try to register.
func (h *handlers) handleSetupStatus(w http.ResponseWriter, r *http.Request) {
n, err := dbq.New(h.pool).CountUsers(r.Context())
if err != nil {
writeErrWithLog(w, h.logger, "setup status: count users failed", apierror.Internal(err))
return
}
writeJSON(w, http.StatusOK, map[string]bool{"setup_required": n == 0})
}
// logSetupTokenIfNeeded writes the setup token to the log at boot when the
// instance has no accounts yet, with the instruction for using it.
func logSetupTokenIfNeeded(pool *pgxpool.Pool, logger *slog.Logger, token *auth.SetupToken) {
if pool == nil || logger == nil {
return
}
n, err := dbq.New(pool).CountUsers(context.Background())
if err != nil || n > 0 {
return
}
logger.Warn("no accounts yet: open the web app, choose Create account, and enter this setup token to become the admin",
"setup_token", token.Value())
}
-72
View File
@@ -273,75 +273,3 @@ func TestRegister_FirstAdminRace(t *testing.T) {
// not "exactly one" (which would require serializable isolation).
t.Logf("admin count after race = %d (>=1 is the invariant)", adminCount)
}
// With the gate on (as Mount sets it), the first account needs the setup
// token from the log; a missing or wrong one is refused and creates nothing.
func TestRegister_FirstUserNeedsSetupToken(t *testing.T) {
if os.Getenv("MINSTREL_TEST_DATABASE_URL") == "" {
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
}
h, _ := testHandlers(t)
resetUsers(t, h)
token, err := auth.NewSetupToken()
if err != nil {
t.Fatalf("mint: %v", err)
}
h.setupToken = token
h.requireSetupToken = true
register := func(body string) int {
req := httptest.NewRequest(http.MethodPost, "/api/auth/register", bytes.NewReader([]byte(body)))
rec := httptest.NewRecorder()
h.handleRegister(rec, req)
return rec.Code
}
if code := register(`{"username":"squatter","password":"abcd1234"}`); code != http.StatusForbidden {
t.Errorf("no token: status = %d, want 403", code)
}
if code := register(`{"username":"squatter","password":"abcd1234","setup_token":"wrong"}`); code != http.StatusForbidden {
t.Errorf("wrong token: status = %d, want 403", code)
}
var n int
if err := h.pool.QueryRow(context.Background(), `SELECT count(*) FROM users`).Scan(&n); err != nil {
t.Fatalf("count: %v", err)
}
if n != 0 {
t.Fatalf("a refused registration created %d account(s)", n)
}
if code := register(`{"username":"operator","password":"abcd1234","setup_token":"` + token.Value() + `"}`); code != http.StatusOK {
t.Fatalf("right token: status = %d, want 200", code)
}
// Once an account exists the token plays no part: the second user is
// governed by registration mode, not the setup token.
if _, err := h.pool.Exec(context.Background(),
`UPDATE registration_settings SET mode = 'open' WHERE id = true`); err != nil {
t.Fatalf("open mode: %v", err)
}
t.Cleanup(func() {
_, _ = h.pool.Exec(context.Background(),
`UPDATE registration_settings SET mode = 'invite_only' WHERE id = true`)
})
if code := register(`{"username":"second","password":"abcd1234"}`); code != http.StatusOK {
t.Errorf("second user without token: status = %d, want 200", code)
}
}
func TestSetupToken_MatchesOnlyItself(t *testing.T) {
tok, err := auth.NewSetupToken()
if err != nil {
t.Fatalf("mint: %v", err)
}
if !tok.Matches(tok.Value()) {
t.Error("token does not match itself")
}
if tok.Matches("") || tok.Matches(tok.Value()+"x") {
t.Error("token matched something else")
}
var none *auth.SetupToken
if none.Matches("") || none.Matches("anything") {
t.Error("a nil token must fail closed")
}
}
-19
View File
@@ -10,7 +10,6 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
@@ -43,15 +42,6 @@ func (h *handlers) handleResetPassword(w http.ResponseWriter, r *http.Request) {
return
}
// Tokens carry 256 bits, so guessing one is hopeless; the limit is on
// failed tries per address so that nobody gets to find that out at our
// expense.
addr := auth.ClientIP(r, h.netSettings.Hops())
if blocked, wait := h.resetLimit.Blocked(addr); blocked {
writeRateLimited(w, wait)
return
}
q := dbq.New(h.pool)
// Look up the reset record so we know which user to update before
@@ -60,7 +50,6 @@ func (h *handlers) handleResetPassword(w http.ResponseWriter, r *http.Request) {
reset, err := q.GetPasswordReset(r.Context(), req.Token)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
h.resetLimit.Record(addr)
writeErr(w, apierror.BadRequest("invalid_token", ""))
return
}
@@ -80,7 +69,6 @@ func (h *handlers) handleResetPassword(w http.ResponseWriter, r *http.Request) {
return
}
if rows == 0 {
h.resetLimit.Record(addr)
writeErr(w, apierror.BadRequest("invalid_token", ""))
return
}
@@ -104,13 +92,6 @@ func (h *handlers) handleResetPassword(w http.ResponseWriter, r *http.Request) {
return
}
// End every session the account has. Whoever needed a reset isn't signed
// in anywhere they rely on, and whoever may have learned the old password
// must not stay signed in on it.
if _, err := q.DeleteSessionsForUser(r.Context(), reset.UserID); err != nil {
h.logger.Error("reset password: revoke sessions failed", "err", err)
}
audit.WriteOrLog(r.Context(), h.pool, h.logger, reset.UserID, reset.UserID, audit.ActionPasswordResetByEmail, nil)
w.WriteHeader(http.StatusNoContent)
+2 -32
View File
@@ -65,7 +65,7 @@ func testHandlers(t *testing.T) (*handlers, *pgxpool.Pool) {
}
lidarrCfg := lidarrconfig.New(pool)
lidarrReqs := lidarrrequests.NewService(pool, lidarrCfg, nil, nil)
lidarrQuar := lidarrquarantine.NewService(pool, lidarrCfg, nil, "")
lidarrQuar := lidarrquarantine.NewService(pool, lidarrCfg, nil)
// tracks.Service has no Lidarr unmonitorer in tests by default; the
// admin-tracks tests below override h.tracks via installTracksLidarrStub
// when they need a stubbed Lidarr.
@@ -91,7 +91,7 @@ func seedUser(t *testing.T, pool *pgxpool.Pool, username, password string, isAdm
u, err := dbq.New(pool).CreateUser(context.Background(), dbq.CreateUserParams{
Username: prefixed,
PasswordHash: string(hash),
ApiTokenHash: "test-api-token-" + prefixed,
ApiToken: "test-api-token-" + prefixed,
IsAdmin: isAdmin,
})
if err != nil {
@@ -179,36 +179,6 @@ func TestHandleLogin_UnknownUserReturns401(t *testing.T) {
}
}
// A guesser who reaches the account limit is refused with 429 before any
// password check, so even the right password is turned away until the window
// passes, and the response says when to come back.
func TestHandleLogin_ThrottledAfterRepeatedFailures(t *testing.T) {
h, pool := testHandlers(t)
h.loginGuard = auth.NewLoginGuard()
seedUser(t, pool, "alice", "hunter2", false)
login := func(password string) *httptest.ResponseRecorder {
body := strings.NewReader(`{"username":"test-alice","password":"` + password + `"}`)
req := httptest.NewRequest(http.MethodPost, "/api/auth/login", body)
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
h.handleLogin(w, req)
return w
}
for i := 0; i < 10; i++ {
if w := login("wrong"); w.Code != http.StatusUnauthorized {
t.Fatalf("attempt %d: status = %d, want 401", i+1, w.Code)
}
}
w := login("hunter2")
if w.Code != http.StatusTooManyRequests {
t.Fatalf("status = %d, want 429 once the account limit is reached", w.Code)
}
if w.Header().Get("Retry-After") == "" {
t.Error("429 without Retry-After")
}
}
func TestHandleLogin_MalformedBodyReturns400(t *testing.T) {
h, _ := testHandlers(t)
req := httptest.NewRequest(http.MethodPost, "/api/auth/login",
-14
View File
@@ -3,10 +3,7 @@ package api
import (
"encoding/json"
"log/slog"
"math"
"net/http"
"strconv"
"time"
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
)
@@ -39,17 +36,6 @@ func writeErr(w http.ResponseWriter, err error) {
}})
}
// writeRateLimited answers 429 with Retry-After in whole seconds, rounded
// up so a client that honours it never arrives a moment early.
func writeRateLimited(w http.ResponseWriter, wait time.Duration) {
secs := int(math.Ceil(wait.Seconds()))
if secs < 1 {
secs = 1
}
w.Header().Set("Retry-After", strconv.Itoa(secs))
writeErr(w, apierror.TooManyRequests("rate_limited", "too many attempts; try again later"))
}
// writeErrWithLog logs the error at Error level and writes the response.
// Use for 500-class errors where the operator needs the cause in logs.
func writeErrWithLog(w http.ResponseWriter, logger *slog.Logger, msg string, err error) {
-58
View File
@@ -1,58 +0,0 @@
package api
import (
"errors"
"fmt"
"log/slog"
"net/http"
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
)
// fileRemoveAPIError answers a delete that could not reach the track's file
// (#3918). Both delete endpoints use it, so the operator gets the same
// explanation from the admin remove-track action and from quarantine's Delete
// file.
//
// The unwritable case is a 409 rather than a 500 because nothing is broken: the
// request conflicts with how the library is mounted, and the fix is the
// operator's. The message names the directory — removal writes to the parent,
// not the file — and the uid/gid the process runs as, which is the half of a
// permission problem invisible from the host. Every case says nothing was
// deleted, because that is exactly what the operator will be worried about.
func fileRemoveAPIError(err error) (*apierror.Error, bool) {
var fre *library.FileRemoveError
if !errors.As(err, &fre) {
return nil, false
}
if fre.NotWritable() {
return &apierror.Error{
Status: http.StatusConflict,
Code: "library_not_writable",
Message: fmt.Sprintf(
"Minstrel runs as uid %d, gid %d and cannot delete from %s (%s). "+
"The library mount must be writable by that user. Nothing was deleted.",
fre.UID, fre.GID, fre.Dir(), fre.Reason()),
Cause: err,
}, true
}
return &apierror.Error{
Status: http.StatusInternalServerError,
Code: "file_delete_failed",
Message: fmt.Sprintf("Could not delete %s (%s). Nothing was deleted.", fre.Path, fre.Reason()),
Cause: err,
}, true
}
// logFileRemoveFailure records a delete that could not reach its file. An
// unwritable library is an environment fact the operator can fix, so it is a
// Warn; anything else is a real fault.
func logFileRemoveFailure(logger *slog.Logger, apiErr *apierror.Error, attrs ...any) {
attrs = append(attrs, "code", apiErr.Code, "err", apiErr.Cause)
if apiErr.Status == http.StatusConflict {
logger.Warn("api: track file could not be deleted", attrs...)
return
}
logger.Error("api: track file could not be deleted", attrs...)
}
-96
View File
@@ -1,96 +0,0 @@
package api
import (
"errors"
"fmt"
"io/fs"
"net/http"
"strings"
"syscall"
"testing"
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
)
const removeTestPath = "/music/Moe Shop/WWW (2020)/01 - WWW.mp3"
// removeFailure builds the error a delete service returns when the file would
// not go, wrapped the way lidarrquarantine.DeleteFile and tracks.RemoveTrack
// wrap it — the mapping has to see through that.
func removeFailure(errno syscall.Errno) error {
return fmt.Errorf("delete file: %w", &library.FileRemoveError{
Path: removeTestPath, UID: 1000, GID: 1000,
Err: &fs.PathError{Op: "remove", Path: removeTestPath, Err: errno},
})
}
func TestFileRemoveAPIError(t *testing.T) {
cases := []struct {
name string
errno syscall.Errno
wantStatus int
wantCode string
wantIn []string
}{
{
name: "read-only mount", errno: syscall.EROFS,
wantStatus: http.StatusConflict, wantCode: "library_not_writable",
wantIn: []string{"uid 1000, gid 1000", "/music/Moe Shop/WWW (2020)", "read-only file system", "Nothing was deleted"},
},
{
name: "permission denied", errno: syscall.EACCES,
wantStatus: http.StatusConflict, wantCode: "library_not_writable",
wantIn: []string{"permission denied", "Nothing was deleted"},
},
{
name: "operation not permitted", errno: syscall.EPERM,
wantStatus: http.StatusConflict, wantCode: "library_not_writable",
wantIn: []string{"operation not permitted"},
},
{
name: "i/o error", errno: syscall.EIO,
wantStatus: http.StatusInternalServerError, wantCode: "file_delete_failed",
wantIn: []string{removeTestPath, "input/output error", "Nothing was deleted"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
apiErr, ok := fileRemoveAPIError(removeFailure(tc.errno))
if !ok {
t.Fatal("a wrapped *library.FileRemoveError was not recognised")
}
if apiErr.Status != tc.wantStatus || apiErr.Code != tc.wantCode {
t.Fatalf("got %d %s, want %d %s", apiErr.Status, apiErr.Code, tc.wantStatus, tc.wantCode)
}
for _, want := range tc.wantIn {
if !strings.Contains(apiErr.Message, want) {
t.Errorf("message %q lacks %q", apiErr.Message, want)
}
}
})
}
}
// The unwritable answer must name the DIRECTORY. Removal needs write access to
// the parent, so a message naming the file would send the operator to fix the
// wrong permissions. The directory is a prefix of the file path, which is why a
// plain "contains the directory" check could never catch that regression.
func TestFileRemoveAPIError_NotWritableNamesTheDirectoryNotTheFile(t *testing.T) {
apiErr, _ := fileRemoveAPIError(removeFailure(syscall.EROFS))
if strings.Contains(apiErr.Message, "01 - WWW.mp3") {
t.Fatalf("message names the file rather than its directory: %q", apiErr.Message)
}
}
func TestFileRemoveAPIError_IgnoresOtherErrors(t *testing.T) {
for name, err := range map[string]error{
"nil": nil,
"plain error": errors.New("delete track: connection reset"),
"path error": &fs.PathError{Op: "remove", Path: removeTestPath, Err: syscall.EROFS},
"not found": library.ErrTrackNotFound,
} {
if _, ok := fileRemoveAPIError(err); ok {
t.Errorf("%s: mapped as a file-remove failure", name)
}
}
}
+1 -4
View File
@@ -465,7 +465,7 @@ func TestRoutesRegisteredInMount(t *testing.T) {
r := chi.NewRouter()
w := playevents.NewWriter(h.pool, slog.New(slog.NewTextHandler(io.Discard, nil)),
30*time.Minute, 0.5, 30000)
Mount(r, h.pool, h.logger, w, config.RecommendationConfig{RadioSize: 50, RadioSizeMax: 200, RecentlyPlayedHours: 1}, h.recSettings, h.lidarrCfg, h.lidarrRequests, h.lidarrQuarantine, h.tracks, h.playlists, h.coverart, h.coverSettings, h.tagSettings, h.scanner, h.scanCfg, h.dataDir, nil, eventbus.New(), nil, nil, h.netSettings, nil, nil)
Mount(r, h.pool, h.logger, w, config.RecommendationConfig{RadioSize: 50, RadioSizeMax: 200, RecentlyPlayedHours: 1}, h.recSettings, h.lidarrCfg, h.lidarrRequests, h.lidarrQuarantine, h.tracks, h.playlists, h.coverart, h.coverSettings, h.tagSettings, h.scanner, h.scanCfg, h.dataDir, nil, eventbus.New(), nil, nil, h.netSettings, nil)
paths := []string{
"/api/artists",
@@ -484,9 +484,6 @@ func TestRoutesRegisteredInMount(t *testing.T) {
// wired.
"/api/admin/library/missing",
"/api/admin/library/reacquisition",
"/api/admin/library/fingerprints",
"/api/admin/library/fingerprint-settings",
"/api/admin/library/duplicates",
}
for _, p := range paths {
req := httptest.NewRequest(http.MethodGet, p, nil)
-14
View File
@@ -8,7 +8,6 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
@@ -62,19 +61,6 @@ func (h *handlers) handleChangePassword(w http.ResponseWriter, r *http.Request)
return
}
// Sign out every other device. A password change is often the response
// to suspecting someone else has it, and their session would otherwise
// outlive the password it was opened with. The device making the change
// stays signed in.
if sessID, ok := auth.SessionIDFromContext(r.Context()); ok {
if _, err := q.DeleteOtherSessionsForUser(r.Context(), dbq.DeleteOtherSessionsForUserParams{
UserID: user.ID,
ID: sessID,
}); err != nil {
h.logger.Error("change password: revoke other sessions failed", "err", err)
}
}
audit.WriteOrLog(r.Context(), h.pool, h.logger, user.ID, user.ID, audit.ActionPasswordChangeSelf, nil)
w.WriteHeader(http.StatusNoContent)
+22 -9
View File
@@ -13,11 +13,23 @@ type apiTokenResp struct {
APIToken string `json:"api_token"`
}
// handleGetMyAPIToken implements GET /api/me/api-token.
func (h *handlers) handleGetMyAPIToken(w http.ResponseWriter, r *http.Request) {
user, ok := requireUser(w, r)
if !ok {
return
}
q := dbq.New(h.pool)
current, err := q.GetUserByID(r.Context(), user.ID)
if err != nil {
h.logger.Error("get api token: lookup failed", "err", err)
writeErr(w, apierror.Internal(err))
return
}
writeJSON(w, http.StatusOK, apiTokenResp{APIToken: current.ApiToken})
}
// handleRegenerateMyAPIToken implements POST /api/me/api-token.
//
// Only the key's hash is stored, so this response is the one time the key
// can be read. There is deliberately no GET: a key that has been shown and
// lost is replaced, not looked up.
func (h *handlers) handleRegenerateMyAPIToken(w http.ResponseWriter, r *http.Request) {
user, ok := requireUser(w, r)
if !ok {
@@ -30,10 +42,11 @@ func (h *handlers) handleRegenerateMyAPIToken(w http.ResponseWriter, r *http.Req
return
}
q := dbq.New(h.pool)
if err := q.RegenerateApiToken(r.Context(), dbq.RegenerateApiTokenParams{
ID: user.ID,
ApiTokenHash: auth.HashAPIToken(newToken),
}); err != nil {
updated, err := q.RegenerateApiToken(r.Context(), dbq.RegenerateApiTokenParams{
ID: user.ID,
ApiToken: newToken,
})
if err != nil {
h.logger.Error("regenerate api token: update failed", "err", err)
writeErr(w, apierror.Internal(err))
return
@@ -41,5 +54,5 @@ func (h *handlers) handleRegenerateMyAPIToken(w http.ResponseWriter, r *http.Req
audit.WriteOrLog(r.Context(), h.pool, h.logger, user.ID, user.ID, audit.ActionTokenRegenerate, nil)
writeJSON(w, http.StatusOK, apiTokenResp{APIToken: newToken})
writeJSON(w, http.StatusOK, apiTokenResp{APIToken: updated.ApiToken})
}
+38 -14
View File
@@ -8,23 +8,47 @@ import (
"os"
"testing"
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
"github.com/go-chi/chi/v5"
)
func newMeTokenRouter(h *handlers) chi.Router {
r := chi.NewRouter()
r.Get("/api/me/api-token", h.handleGetMyAPIToken)
r.Post("/api/me/api-token", h.handleRegenerateMyAPIToken)
return r
}
func TestGetAPIToken_ReturnsCurrentToken(t *testing.T) {
if os.Getenv("MINSTREL_TEST_DATABASE_URL") == "" {
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
}
h, pool := testHandlers(t)
user := seedUser(t, pool, "tok1", "pw", false)
req := httptest.NewRequest(http.MethodGet, "/api/me/api-token", nil)
req = withUser(req, user)
rec := httptest.NewRecorder()
newMeTokenRouter(h).ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
}
var resp apiTokenResp
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode: %v", err)
}
if resp.APIToken != user.ApiToken {
t.Errorf("api_token = %q, want %q", resp.APIToken, user.ApiToken)
}
}
func TestRegenerateAPIToken_IssuesNewToken(t *testing.T) {
if os.Getenv("MINSTREL_TEST_DATABASE_URL") == "" {
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
}
h, pool := testHandlers(t)
user := seedUser(t, pool, "tok2", "pw", false)
oldHash := user.ApiTokenHash
oldToken := user.ApiToken
req := httptest.NewRequest(http.MethodPost, "/api/me/api-token", nil)
req = withUser(req, user)
@@ -41,20 +65,20 @@ func TestRegenerateAPIToken_IssuesNewToken(t *testing.T) {
if resp.APIToken == "" {
t.Fatalf("api_token is empty")
}
// The DB holds the new key's hash, never the key, and the old key no
// longer matches.
var dbHash string
if resp.APIToken == oldToken {
t.Errorf("api_token unchanged after regenerate")
}
// Verify DB row has the new token and old token no longer matches.
var dbToken string
if err := pool.QueryRow(context.Background(),
"SELECT api_token_hash FROM users WHERE id = $1", user.ID).Scan(&dbHash); err != nil {
t.Fatalf("read token hash: %v", err)
"SELECT api_token FROM users WHERE id = $1", user.ID).Scan(&dbToken); err != nil {
t.Fatalf("read token: %v", err)
}
if dbHash != auth.HashAPIToken(resp.APIToken) {
t.Errorf("DB api_token_hash = %q, want hash of the returned key", dbHash)
if dbToken != resp.APIToken {
t.Errorf("DB api_token = %q, want %q", dbToken, resp.APIToken)
}
if dbHash == oldHash {
t.Errorf("old key hash still in DB after regenerate")
}
if dbHash == resp.APIToken {
t.Errorf("DB holds the raw key")
if dbToken == oldToken {
t.Errorf("old token still in DB after regenerate")
}
}
+2 -2
View File
@@ -117,7 +117,7 @@ func (h *handlers) handleGetCover(w http.ResponseWriter, r *http.Request) {
// clients skip the conditional GET for the bulk of a session, but
// stale art clears within 24h after a re-scan. ServeContent below
// still emits Last-Modified for the conditional path when needed.
w.Header().Set("Cache-Control", "private, max-age=86400, must-revalidate")
w.Header().Set("Cache-Control", "public, max-age=86400, must-revalidate")
http.ServeContent(w, r, filepath.Base(path), info.ModTime(), f)
}
@@ -197,6 +197,6 @@ func (h *handlers) handleGetStream(w http.ResponseWriter, r *http.Request) {
// max-age + immutable lets the client cache (LockCachingAudioSource
// on the Flutter side, browser cache on web) skip even the
// conditional GET on repeat plays.
w.Header().Set("Cache-Control", "private, max-age=31536000, immutable")
w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
http.ServeContent(w, r, filepath.Base(track.FilePath), info.ModTime(), f)
}
+1 -1
View File
@@ -406,7 +406,7 @@ func (h *handlers) handleGetPlaylistCover(w http.ResponseWriter, r *http.Request
// (system playlists re-rendered on rebuild, user playlists when
// modified). 5 minutes is short enough for normal edits to feel
// fresh, long enough to skip repeat fetches during a session.
w.Header().Set("Cache-Control", "private, max-age=300, must-revalidate")
w.Header().Set("Cache-Control", "public, max-age=300, must-revalidate")
http.ServeFile(w, r, full)
}
-4
View File
@@ -101,10 +101,6 @@ func (h *handlers) handleRadio(w http.ResponseWriter, r *http.Request) {
candidates, err := recommendation.LoadCandidatesFromSimilarity(
r.Context(), q, user.ID, seedID,
h.recCfg.RecentlyPlayedHours, currentVec, exclude, limits,
// A fresh seed per request (#3889): radio is a new session each time
// and SHOULD draw differently. The system mixes are the surfaces that
// promise repeatability; this is not one of them.
strconv.FormatInt(time.Now().UnixNano(), 36),
)
if err != nil {
h.logger.Warn("api: radio: similarity-pool failed; falling back to whole-library", "err", err)
-139
View File
@@ -1,139 +0,0 @@
package api
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
func sessionExists(t *testing.T, pool *pgxpool.Pool, id pgtype.UUID) bool {
t.Helper()
var exists bool
if err := pool.QueryRow(context.Background(),
`SELECT EXISTS (SELECT 1 FROM sessions WHERE id = $1)`, id,
).Scan(&exists); err != nil {
t.Fatalf("exists check: %v", err)
}
return exists
}
// Sessions expire server-side on two limits, and an expired one stops
// authenticating at once rather than when the GC sweep gets to it.
func TestSessionExpiry_IdleAndAbsoluteLimits(t *testing.T) {
_, pool := testHandlers(t)
user := seedUser(t, pool, "alice", "hunter2", false)
q := dbq.New(pool)
mint := func(setup string) []byte {
token, err := auth.MintSessionToken()
if err != nil {
t.Fatalf("mint: %v", err)
}
hash := auth.HashSessionToken(token)
sess, err := q.InsertSession(context.Background(), dbq.InsertSessionParams{
UserID: user.ID, TokenHash: hash, UserAgent: "test", Ip: "192.0.2.1",
})
if err != nil {
t.Fatalf("insert: %v", err)
}
if setup != "" {
if _, err := pool.Exec(context.Background(), `UPDATE sessions SET `+setup+` WHERE id = $1`, sess.ID); err != nil {
t.Fatalf("age session: %v", err)
}
}
return hash
}
fresh := mint("")
idle := mint(`last_seen_at = now() - interval '31 days'`)
old := mint(`created_at = now() - interval '366 days'`)
if _, err := q.GetSessionByTokenHash(context.Background(), fresh); err != nil {
t.Errorf("fresh session: %v, want found", err)
}
for name, hash := range map[string][]byte{"idle": idle, "absolute": old} {
if _, err := q.GetSessionByTokenHash(context.Background(), hash); !errors.Is(err, pgx.ErrNoRows) {
t.Errorf("%s-expired session: err = %v, want ErrNoRows", name, err)
}
}
listed, err := q.ListSessionsForUser(context.Background(), user.ID)
if err != nil {
t.Fatalf("list: %v", err)
}
if len(listed) != 1 {
t.Errorf("listed %d sessions, want only the live one", len(listed))
}
swept, err := q.GcDeleteExpiredSessions(context.Background())
if err != nil {
t.Fatalf("gc: %v", err)
}
if swept != 2 {
t.Errorf("gc swept %d, want the 2 expired rows", swept)
}
}
// Changing your password signs out every other device and keeps this one.
func TestHandleChangePassword_RevokesOtherSessions(t *testing.T) {
h, pool := testHandlers(t)
user := seedUser(t, pool, "alice", "hunter2", false)
current := seedSession(t, pool, user.ID, "192.0.2.1")
other := seedSession(t, pool, user.ID, "198.51.100.7")
body := strings.NewReader(`{"current_password":"hunter2","new_password":"correct-horse"}`)
req := httptest.NewRequest(http.MethodPut, "/api/me/password", body)
req.Header.Set("Content-Type", "application/json")
req = withSession(req, user, current)
w := httptest.NewRecorder()
h.handleChangePassword(w, req)
if w.Code != http.StatusNoContent {
t.Fatalf("status = %d body = %s", w.Code, w.Body.String())
}
if !sessionExists(t, pool, current) {
t.Error("the device that changed the password was signed out")
}
if sessionExists(t, pool, other) {
t.Error("another device's session survived the password change")
}
}
// A reset by email ends every session the account has.
func TestHandleResetPassword_RevokesAllSessions(t *testing.T) {
h, pool := testHandlers(t)
user := seedUser(t, pool, "alice", "hunter2", false)
s1 := seedSession(t, pool, user.ID, "192.0.2.1")
s2 := seedSession(t, pool, user.ID, "198.51.100.7")
const token = "reset-token-for-session-lifecycle-test"
if _, err := pool.Exec(context.Background(),
`INSERT INTO password_resets (token, user_id, expires_at) VALUES ($1, $2, now() + interval '1 hour')`,
token, user.ID,
); err != nil {
t.Fatalf("seed reset: %v", err)
}
body := strings.NewReader(`{"token":"` + token + `","new_password":"correct-horse"}`)
req := httptest.NewRequest(http.MethodPost, "/api/auth/reset-password", body)
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
h.handleResetPassword(w, req)
if w.Code != http.StatusNoContent {
t.Fatalf("status = %d body = %s", w.Code, w.Body.String())
}
if sessionExists(t, pool, s1) || sessionExists(t, pool, s2) {
t.Error("a session survived the password reset")
}
}
-4
View File
@@ -54,10 +54,6 @@ func Unauthorized(code, message string) *Error {
return &Error{Status: 401, Code: code, Message: message}
}
func TooManyRequests(code, message string) *Error {
return &Error{Status: 429, Code: code, Message: message}
}
func Internal(cause error) *Error {
return &Error{Status: 500, Code: "server_error", Message: "internal server error", Cause: cause}
}
-5
View File
@@ -55,11 +55,6 @@ const (
// exercised.
ActionSessionRevoke Action = "session_revoke"
ActionSessionRevokeOthers Action = "session_revoke_others"
// Duplicate merge (#3911). Irreversible: a copy's file and row are removed
// and its history moved onto the copy kept. The metadata names both, so the
// log can answer "where did that file go" long after the report is gone.
ActionDuplicateMerge Action = "duplicate_merge"
)
// Write inserts one audit_log row. metadata is marshaled as JSON;
-1
View File
@@ -168,7 +168,6 @@ func TestWrite_AllActionConstantsArePersisted(t *testing.T) {
audit.ActionTokenRegenerate,
audit.ActionForgotPasswordInit,
audit.ActionPasswordResetByEmail,
audit.ActionDuplicateMerge,
}
for _, a := range actions {
if err := audit.Write(context.Background(), pool, nilUUID, nilUUID, a, nil); err != nil {
-30
View File
@@ -65,36 +65,6 @@ func ClientIP(r *http.Request, trustedProxyHops int) string {
return remote
}
// IsHTTPS reports whether the client reached us over HTTPS, reading through
// trustedProxyHops reverse proxies by the same rule as ClientIP.
//
// TLS terminates at the operator's proxy (rule 94), so r.TLS is nil in every
// real deployment and only X-Forwarded-Proto can say what the client used.
// That header is believed only when hops >= 1: with no trusted proxy it is
// just something any client can send. Proxies that append rather than
// overwrite produce a comma list, read positionally like X-Forwarded-For.
//
// This decides only HTTPS-only behaviour (the cookie Secure flag, HSTS). It
// must never drive a redirect: plain-HTTP and LAN use keep working.
func IsHTTPS(r *http.Request, trustedProxyHops int) bool {
if r.TLS != nil {
return true
}
if trustedProxyHops <= 0 {
return false
}
raw := r.Header.Get("X-Forwarded-Proto")
if strings.TrimSpace(raw) == "" {
return false
}
parts := strings.Split(raw, ",")
idx := len(parts) - trustedProxyHops
if idx < 0 {
idx = 0
}
return strings.EqualFold(strings.TrimSpace(parts[idx]), "https")
}
// forwardedChain returns the X-Forwarded-For entries in wire order, or the
// single X-Real-IP value when XFF is absent.
//
-43
View File
@@ -1,7 +1,6 @@
package auth
import (
"crypto/tls"
"net/http"
"testing"
)
@@ -169,45 +168,3 @@ func TestClientIP(t *testing.T) {
})
}
}
func TestIsHTTPS(t *testing.T) {
tests := []struct {
name string
hops int
proto string
tls bool
want bool
}{
// The three cases rule 94 names.
{name: "hops 0 ignores a forwarded https", hops: 0, proto: "https", want: false},
{name: "hops 1 believes a forwarded https", hops: 1, proto: "https", want: true},
{name: "plain request is not https", hops: 1, want: false},
{name: "forwarded http is not https", hops: 1, proto: "http", want: false},
{name: "case-insensitive", hops: 1, proto: "HTTPS", want: true},
{name: "direct TLS is https whatever the hops", hops: 0, tls: true, want: true},
// An appending chain is read positionally: with one trusted hop the
// last entry is our proxy's word, and a client-supplied first entry
// cannot flip it.
{name: "appended chain reads our proxy's entry", hops: 1, proto: "https, http", want: false},
{name: "two hops read the CDN's entry", hops: 2, proto: "https, http", want: true},
{name: "over-counted hops clamp to the first entry", hops: 5, proto: "https", want: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
r, err := http.NewRequest(http.MethodGet, "/api/auth/login", nil)
if err != nil {
t.Fatalf("NewRequest: %v", err)
}
if tc.proto != "" {
r.Header.Set("X-Forwarded-Proto", tc.proto)
}
if tc.tls {
r.TLS = &tls.ConnectionState{}
}
if got := IsHTTPS(r, tc.hops); got != tc.want {
t.Errorf("IsHTTPS(hops=%d, proto=%q) = %v, want %v", tc.hops, tc.proto, got, tc.want)
}
})
}
}
-205
View File
@@ -1,205 +0,0 @@
package auth
import (
"strings"
"sync"
"time"
"golang.org/x/crypto/bcrypt"
)
// AttemptLimiter caps how many attempts a key may make inside a fixed
// window. It is the throttle in front of every password-shaped check:
// native login, register, forgot/reset password and Subsonic /rest auth.
//
// In memory on purpose. Minstrel is a single process, the counts only need
// to outlive a guessing run rather than a restart, and a table would put a
// write on every failed login. Each key costs one small struct, and expired
// keys are swept as the map grows, so a spray across many addresses cannot
// hold memory past one window.
type AttemptLimiter struct {
max int
window time.Duration
now func() time.Time
mu sync.Mutex
buckets map[string]*attemptBucket
nextSweep int
}
type attemptBucket struct {
count int
start time.Time
}
// sweepFloor is the map size below which expired keys are left in place;
// past it, a sweep runs whenever the map doubles from its last swept size.
const sweepFloor = 1024
// NewAttemptLimiter returns a limiter allowing max attempts per key per
// window.
func NewAttemptLimiter(max int, window time.Duration) *AttemptLimiter {
return &AttemptLimiter{
max: max,
window: window,
now: time.Now,
buckets: map[string]*attemptBucket{},
nextSweep: sweepFloor,
}
}
// Blocked reports whether key has used its attempts for the current window,
// and if so how long until the window resets. It records nothing, so a check
// can run before the expensive work and the outcome be recorded after.
func (l *AttemptLimiter) Blocked(key string) (bool, time.Duration) {
if l == nil || key == "" {
return false, 0
}
l.mu.Lock()
defer l.mu.Unlock()
b, ok := l.buckets[key]
if !ok {
return false, 0
}
now := l.now()
if now.Sub(b.start) >= l.window {
delete(l.buckets, key)
return false, 0
}
if b.count < l.max {
return false, 0
}
return true, b.start.Add(l.window).Sub(now)
}
// Record counts one attempt against key.
func (l *AttemptLimiter) Record(key string) {
if l == nil || key == "" {
return
}
l.mu.Lock()
defer l.mu.Unlock()
now := l.now()
b, ok := l.buckets[key]
if !ok || now.Sub(b.start) >= l.window {
l.buckets[key] = &attemptBucket{count: 1, start: now}
l.maybeSweep(now)
return
}
b.count++
}
// Reset forgets key, as after a successful login: the user who finally got
// their password right should not carry their typos into the next window.
func (l *AttemptLimiter) Reset(key string) {
if l == nil || key == "" {
return
}
l.mu.Lock()
defer l.mu.Unlock()
delete(l.buckets, key)
}
// maybeSweep drops expired buckets once the map has doubled since the last
// sweep. Callers hold l.mu.
func (l *AttemptLimiter) maybeSweep(now time.Time) {
if len(l.buckets) < l.nextSweep {
return
}
for k, b := range l.buckets {
if now.Sub(b.start) >= l.window {
delete(l.buckets, k)
}
}
l.nextSweep = max(sweepFloor, 2*len(l.buckets))
}
// LoginGuard pairs a per-account and a per-address limiter, the shape every
// password check uses. The account limit stops a slow guess at one user from
// many addresses; the address limit stops one address spraying many users.
// Only failures are recorded, so a user who signs in correctly is never
// counted at all.
type LoginGuard struct {
account *AttemptLimiter
address *AttemptLimiter
}
// Login limits: 10 failures per account and 50 per address per 15 minutes,
// the same numbers ThoughtSync settled on. Generous enough that a user
// fumbling a password manager never meets them, tight enough that an online
// guess against bcrypt gets ~1,000 tries a day per account.
const (
loginWindow = 15 * time.Minute
loginAccountMax = 10
loginAddressMax = 50
)
// NewLoginGuard returns a guard with the default login limits.
func NewLoginGuard() *LoginGuard {
return &LoginGuard{
account: NewAttemptLimiter(loginAccountMax, loginWindow),
address: NewAttemptLimiter(loginAddressMax, loginWindow),
}
}
// Blocked reports whether either the account or the address is over its
// limit, with the longer of the two waits. Check it BEFORE verifying the
// password, so a blocked guess costs no bcrypt.
func (g *LoginGuard) Blocked(account, address string) (bool, time.Duration) {
if g == nil {
return false, 0
}
aBlocked, aWait := g.account.Blocked(accountKey(account))
ipBlocked, ipWait := g.address.Blocked(address)
return aBlocked || ipBlocked, max(aWait, ipWait)
}
// Fail records a failed attempt against both the account and the address.
// An unknown username counts against its name all the same, so the limit
// gives away nothing about which accounts exist.
func (g *LoginGuard) Fail(account, address string) {
if g == nil {
return
}
g.account.Record(accountKey(account))
g.address.Record(address)
}
// Succeed clears the account's failures. The address keeps its count: one
// address that guessed fifty accounts and got one right is still spraying.
func (g *LoginGuard) Succeed(account string) {
if g == nil {
return
}
g.account.Reset(accountKey(account))
}
// accountKey folds case so "Admin" and "admin" share one budget. Usernames
// are compared exactly by the lookup, but the guesser shouldn't get a fresh
// allowance per capitalisation.
func accountKey(account string) string {
return strings.ToLower(strings.TrimSpace(account))
}
var (
dummyHashOnce sync.Once
dummyHash []byte
)
// DummyVerify spends the same bcrypt time a real password check would, for
// the path where the username doesn't exist. Without it, an unknown user
// answers in microseconds and a known one in ~50ms, and the uniform error
// message hides nothing.
func DummyVerify(plaintext string) {
dummyHashOnce.Do(func() {
// What the hash is of doesn't matter — no account carries it. Its
// cost does: DefaultCost, the same as every stored password.
h, err := bcrypt.GenerateFromPassword([]byte("minstrel-dummy-password"), bcrypt.DefaultCost)
if err == nil {
dummyHash = h
}
})
if dummyHash != nil {
_ = bcrypt.CompareHashAndPassword(dummyHash, []byte(plaintext))
}
}
-114
View File
@@ -1,114 +0,0 @@
package auth
import (
"testing"
"time"
)
// fakeClock lets a test step through a window without sleeping.
type fakeClock struct{ t time.Time }
func (c *fakeClock) now() time.Time { return c.t }
func newTestLimiter(max int, window time.Duration) (*AttemptLimiter, *fakeClock) {
c := &fakeClock{t: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)}
l := NewAttemptLimiter(max, window)
l.now = c.now
return l, c
}
func TestAttemptLimiter_BlocksAtMaxAndReportsWait(t *testing.T) {
l, c := newTestLimiter(3, 15*time.Minute)
for i := 0; i < 3; i++ {
if blocked, _ := l.Blocked("k"); blocked {
t.Fatalf("blocked after %d attempts, want allowed below max", i)
}
l.Record("k")
}
c.t = c.t.Add(5 * time.Minute)
blocked, wait := l.Blocked("k")
if !blocked {
t.Fatal("not blocked after max attempts")
}
if wait != 10*time.Minute {
t.Errorf("wait = %v, want the 10m left in the window", wait)
}
}
func TestAttemptLimiter_WindowExpiryClears(t *testing.T) {
l, c := newTestLimiter(1, time.Minute)
l.Record("k")
if blocked, _ := l.Blocked("k"); !blocked {
t.Fatal("want blocked inside the window")
}
c.t = c.t.Add(time.Minute)
if blocked, _ := l.Blocked("k"); blocked {
t.Fatal("still blocked once the window has passed")
}
}
func TestAttemptLimiter_KeysAreIndependentAndResetClears(t *testing.T) {
l, _ := newTestLimiter(1, time.Minute)
l.Record("a")
if blocked, _ := l.Blocked("b"); blocked {
t.Fatal("one key's attempts blocked another")
}
l.Reset("a")
if blocked, _ := l.Blocked("a"); blocked {
t.Fatal("Reset did not clear the key")
}
}
func TestAttemptLimiter_SweepDropsExpiredKeys(t *testing.T) {
l, c := newTestLimiter(5, time.Minute)
// One short of the floor: no sweep yet, however stale these become.
for i := 0; i < sweepFloor-1; i++ {
l.Record("k" + time.Duration(i).String())
}
c.t = c.t.Add(2 * time.Minute)
l.Record("fresh") // reaches the floor and triggers a sweep
if n := len(l.buckets); n != 1 {
t.Errorf("buckets after sweep = %d, want only the fresh key", n)
}
}
func TestAttemptLimiter_NilAndEmptyKeyAreNoOps(t *testing.T) {
var l *AttemptLimiter
l.Record("k")
if blocked, _ := l.Blocked("k"); blocked {
t.Error("nil limiter blocked")
}
real, _ := newTestLimiter(1, time.Minute)
real.Record("")
if blocked, _ := real.Blocked(""); blocked {
t.Error("empty key was counted")
}
}
func TestLoginGuard_AccountLimitSpansAddressesAndFoldsCase(t *testing.T) {
g := NewLoginGuard()
for i := 0; i < loginAccountMax; i++ {
g.Fail("Alice", "10.0.0."+string(rune('0'+i%10)))
}
if blocked, _ := g.Blocked("alice", "192.0.2.1"); !blocked {
t.Fatal("account limit should hold from a fresh address and any capitalisation")
}
g.Succeed("ALICE")
if blocked, _ := g.Blocked("alice", "192.0.2.1"); blocked {
t.Fatal("Succeed should clear the account's failures")
}
}
func TestLoginGuard_AddressLimitSpansAccounts(t *testing.T) {
g := NewLoginGuard()
for i := 0; i < loginAddressMax; i++ {
g.Fail("user"+time.Duration(i).String(), "203.0.113.9")
}
if blocked, _ := g.Blocked("someone-new", "203.0.113.9"); !blocked {
t.Fatal("address that sprayed many accounts should be blocked")
}
g.Succeed("someone-new")
if blocked, _ := g.Blocked("someone-new", "203.0.113.9"); !blocked {
t.Fatal("a success must not clear the address count")
}
}
-10
View File
@@ -5,7 +5,6 @@ import (
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"log/slog"
"net/http"
@@ -42,15 +41,6 @@ func HashSessionToken(token string) []byte {
return sum[:]
}
// HashAPIToken maps a raw API key (the OpenSubsonic apiKey) to the
// `users.api_token_hash` column: sha256, hex. The key is minted with
// MintSessionToken, so it carries the same 256 bits and the same reasoning
// as HashSessionToken applies. Hex rather than bytes so the column stays
// text and a migration can compute it in SQL from the old plaintext.
func HashAPIToken(token string) string {
return hex.EncodeToString(HashSessionToken(token))
}
// VerifyPassword is the canonical bcrypt comparison. Returns false on a
// malformed hash so callers don't need to distinguish "hash invalid" from
// "password wrong" — both are auth failures from the client's perspective.
-46
View File
@@ -1,46 +0,0 @@
package auth
import (
"crypto/rand"
"crypto/subtle"
"encoding/hex"
)
// SetupToken guards the first-admin registration. Until the first account
// exists, register makes whoever calls it the admin, so a fresh instance on a
// public address belonged to whoever found it first. Now that call also has
// to carry this token, which is generated at startup and written only to the
// server log: proof that the caller can read the operator's logs.
//
// The token lives in memory. A restart mints a new one and logs it again,
// which is the behaviour wanted: an old token from a log line someone else
// saw stops working.
type SetupToken struct {
value string
}
// NewSetupToken mints a 128-bit token.
func NewSetupToken() (*SetupToken, error) {
b := make([]byte, 16)
if _, err := rand.Read(b); err != nil {
return nil, err
}
return &SetupToken{value: hex.EncodeToString(b)}, nil
}
// Value returns the token for logging.
func (t *SetupToken) Value() string {
if t == nil {
return ""
}
return t.value
}
// Matches reports whether supplied is the token, in constant time. A nil
// token never matches anything, so a missing token fails closed.
func (t *SetupToken) Matches(supplied string) bool {
if t == nil || t.value == "" || supplied == "" {
return false
}
return subtle.ConstantTimeCompare([]byte(t.value), []byte(supplied)) == 1
}
+1 -1
View File
@@ -42,7 +42,7 @@ func discardLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Disca
func seedUser(t *testing.T, q *dbq.Queries, name string) pgtype.UUID {
t.Helper()
u, err := q.CreateUser(context.Background(), dbq.CreateUserParams{
Username: dbtest.TestUserPrefix + name, PasswordHash: "x", ApiTokenHash: name + "-tok", IsAdmin: false,
Username: dbtest.TestUserPrefix + name, PasswordHash: "x", ApiToken: name + "-tok", IsAdmin: false,
})
if err != nil {
t.Fatalf("seed user %s: %v", name, err)
-457
View File
@@ -1,457 +0,0 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: duplicates.sql
package dbq
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const addDuplicateGroupMember = `-- name: AddDuplicateGroupMember :exec
INSERT INTO duplicate_group_members (group_id, track_id)
VALUES ($1, $2)
ON CONFLICT DO NOTHING
`
type AddDuplicateGroupMemberParams struct {
GroupID pgtype.UUID
TrackID pgtype.UUID
}
func (q *Queries) AddDuplicateGroupMember(ctx context.Context, arg AddDuplicateGroupMemberParams) error {
_, err := q.db.Exec(ctx, addDuplicateGroupMember, arg.GroupID, arg.TrackID)
return err
}
const countPendingDuplicateGroups = `-- name: CountPendingDuplicateGroups :one
SELECT count(*)::bigint
FROM duplicate_groups g
WHERE g.status = 'pending'
AND (SELECT count(*) FROM duplicate_group_members m WHERE m.group_id = g.id) >= 2
`
// Proposals awaiting review. A group left with one member — its other tracks
// deleted since the sweep — is no proposal at all and is not counted; the next
// sweep retires it.
func (q *Queries) CountPendingDuplicateGroups(ctx context.Context) (int64, error) {
row := q.db.QueryRow(ctx, countPendingDuplicateGroups)
var column_1 int64
err := row.Scan(&column_1)
return column_1, err
}
const deleteStalePendingDuplicateGroups = `-- name: DeleteStalePendingDuplicateGroups :execrows
DELETE FROM duplicate_groups g
WHERE g.status = 'pending'
AND g.last_seen_sweep_id IS DISTINCT FROM $1
AND (g.last_seen_sweep_id IS NULL
OR (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = g.last_seen_sweep_id)
< (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = $1))
`
// A pending proposal this sweep did not find again no longer describes the
// library: a member was re-fingerprinted, merged away or went missing. Dismissed
// groups are kept regardless — they are the memory of a decision.
//
// Only proposals last confirmed by an EARLIER sweep go. Should two sweeps ever
// overlap (a manual trigger racing the worker), neither may delete what the other
// has just found.
func (q *Queries) DeleteStalePendingDuplicateGroups(ctx context.Context, sweepID pgtype.UUID) (int64, error) {
result, err := q.db.Exec(ctx, deleteStalePendingDuplicateGroups, sweepID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const dismissDuplicateGroup = `-- name: DismissDuplicateGroup :execrows
UPDATE duplicate_groups
SET status = 'dismissed', resolved_at = now()
WHERE id = $1 AND status = 'pending'
`
// "These are not duplicates." Only a pending group can be dismissed; zero rows
// means it was already resolved or no longer exists.
func (q *Queries) DismissDuplicateGroup(ctx context.Context, id pgtype.UUID) (int64, error) {
result, err := q.db.Exec(ctx, dismissDuplicateGroup, id)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const finishDuplicateSweep = `-- name: FinishDuplicateSweep :exec
UPDATE duplicate_sweeps
SET finished_at = now(),
candidates = $1,
groups_found = $2,
oversize_clusters = $3,
error_message = NULLIF($4::text, '')
WHERE id = $5
`
type FinishDuplicateSweepParams struct {
Candidates *int32
GroupsFound *int32
OversizeClusters *int32
ErrorMessage string
ID pgtype.UUID
}
func (q *Queries) FinishDuplicateSweep(ctx context.Context, arg FinishDuplicateSweepParams) error {
_, err := q.db.Exec(ctx, finishDuplicateSweep,
arg.Candidates,
arg.GroupsFound,
arg.OversizeClusters,
arg.ErrorMessage,
arg.ID,
)
return err
}
const getInFlightDuplicateSweep = `-- name: GetInFlightDuplicateSweep :one
SELECT id, started_at
FROM duplicate_sweeps
WHERE finished_at IS NULL
ORDER BY started_at DESC
LIMIT 1
`
type GetInFlightDuplicateSweepRow struct {
ID pgtype.UUID
StartedAt pgtype.Timestamptz
}
// The guard against two sweeps at once: "in flight" is finished_at IS NULL.
func (q *Queries) GetInFlightDuplicateSweep(ctx context.Context) (GetInFlightDuplicateSweepRow, error) {
row := q.db.QueryRow(ctx, getInFlightDuplicateSweep)
var i GetInFlightDuplicateSweepRow
err := row.Scan(&i.ID, &i.StartedAt)
return i, err
}
const getLatestDuplicateSweep = `-- name: GetLatestDuplicateSweep :one
SELECT id, started_at, finished_at, candidates, groups_found, oversize_clusters, error_message
FROM duplicate_sweeps
ORDER BY started_at DESC
LIMIT 1
`
func (q *Queries) GetLatestDuplicateSweep(ctx context.Context) (DuplicateSweep, error) {
row := q.db.QueryRow(ctx, getLatestDuplicateSweep)
var i DuplicateSweep
err := row.Scan(
&i.ID,
&i.StartedAt,
&i.FinishedAt,
&i.Candidates,
&i.GroupsFound,
&i.OversizeClusters,
&i.ErrorMessage,
)
return i, err
}
const getLatestFingerprintComputedAt = `-- name: GetLatestFingerprintComputedAt :one
SELECT max(computed_at)::timestamptz AS latest FROM track_fingerprints
`
// Whether a sweep has anything new to look at: fingerprints written since the
// last sweep started.
func (q *Queries) GetLatestFingerprintComputedAt(ctx context.Context) (pgtype.Timestamptz, error) {
row := q.db.QueryRow(ctx, getLatestFingerprintComputedAt)
var latest pgtype.Timestamptz
err := row.Scan(&latest)
return latest, err
}
const listDismissedDuplicateMemberSets = `-- name: ListDismissedDuplicateMemberSets :many
SELECT g.id, array_agg(m.track_id ORDER BY m.track_id)::uuid[] AS track_ids
FROM duplicate_groups g
JOIN duplicate_group_members m ON m.group_id = g.id
WHERE g.status = 'dismissed'
GROUP BY g.id
`
type ListDismissedDuplicateMemberSetsRow struct {
ID pgtype.UUID
TrackIds []pgtype.UUID
}
// What the operator has already said are not duplicates. A new proposal whose
// every member sat together in one of these is not proposed again.
func (q *Queries) ListDismissedDuplicateMemberSets(ctx context.Context) ([]ListDismissedDuplicateMemberSetsRow, error) {
rows, err := q.db.Query(ctx, listDismissedDuplicateMemberSets)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListDismissedDuplicateMemberSetsRow
for rows.Next() {
var i ListDismissedDuplicateMemberSetsRow
if err := rows.Scan(&i.ID, &i.TrackIds); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listDuplicateCandidates = `-- name: ListDuplicateCandidates :many
SELECT t.id, t.duration_ms, f.audio_stream_sha256, f.chromaprint
FROM tracks t
JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NULL
AND f.fingerprint_version >= $1
AND f.chromaprint IS NOT NULL
-- Only chromaprints taken at the current length: prints at two lengths are not
-- comparable, and after a length change the backfill is still re-deriving the
-- rest (#3913).
AND f.chromaprint_length_sec = $2
AND (t.duration_ms, t.id) > ($3::integer, $4::uuid)
ORDER BY t.duration_ms, t.id
LIMIT $5
`
type ListDuplicateCandidatesParams struct {
CurrentVersion int16
ChromaprintLengthSec int32
AfterDurationMs int32
AfterID pgtype.UUID
PageLimit int32
}
type ListDuplicateCandidatesRow struct {
ID pgtype.UUID
DurationMs int32
AudioStreamSha256 []byte
Chromaprint []int32
}
// The acoustic tier's input, one page at a time in (duration_ms, id) order so the
// sweep holds only a sliding window of durations. Tracks without a chromaprint
// cannot be compared acoustically and are left out; any exact duplicates among
// them come from ListExactDuplicateHashes.
func (q *Queries) ListDuplicateCandidates(ctx context.Context, arg ListDuplicateCandidatesParams) ([]ListDuplicateCandidatesRow, error) {
rows, err := q.db.Query(ctx, listDuplicateCandidates,
arg.CurrentVersion,
arg.ChromaprintLengthSec,
arg.AfterDurationMs,
arg.AfterID,
arg.PageLimit,
)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListDuplicateCandidatesRow
for rows.Next() {
var i ListDuplicateCandidatesRow
if err := rows.Scan(
&i.ID,
&i.DurationMs,
&i.AudioStreamSha256,
&i.Chromaprint,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listExactDuplicateHashes = `-- name: ListExactDuplicateHashes :many
SELECT f.audio_stream_sha256,
array_agg(t.id ORDER BY t.id)::uuid[] AS track_ids
FROM track_fingerprints f
JOIN tracks t ON t.id = f.track_id
WHERE t.missing_since IS NULL
AND f.fingerprint_version >= $1
AND f.audio_stream_sha256 IS NOT NULL
GROUP BY f.audio_stream_sha256
HAVING count(*) > 1
`
type ListExactDuplicateHashesRow struct {
AudioStreamSha256 []byte
TrackIds []pgtype.UUID
}
// The exact tier, library-wide in one pass: identical encoded audio shared by
// more than one present track.
func (q *Queries) ListExactDuplicateHashes(ctx context.Context, currentVersion int16) ([]ListExactDuplicateHashesRow, error) {
rows, err := q.db.Query(ctx, listExactDuplicateHashes, currentVersion)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListExactDuplicateHashesRow
for rows.Next() {
var i ListExactDuplicateHashesRow
if err := rows.Scan(&i.AudioStreamSha256, &i.TrackIds); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listPendingDuplicateGroupMembers = `-- name: ListPendingDuplicateGroupMembers :many
WITH page AS (
SELECT g.id, g.tier, g.worst_bit_error_rate, g.detected_at
FROM duplicate_groups g
WHERE g.status = 'pending'
AND (SELECT count(*) FROM duplicate_group_members m WHERE m.group_id = g.id) >= 2
ORDER BY g.detected_at DESC, g.id
LIMIT $2 OFFSET $1
)
SELECT p.id AS group_id,
p.tier,
p.worst_bit_error_rate,
p.detected_at,
t.id AS track_id,
t.title,
artists.name AS artist_name,
albums.id AS album_id,
albums.title AS album_title,
t.file_path,
t.file_format,
t.file_size,
t.duration_ms,
t.added_at,
(SELECT count(*) FROM general_likes l WHERE l.track_id = t.id)::bigint AS like_count,
(SELECT count(*) FROM play_events e WHERE e.track_id = t.id)::bigint AS play_count
FROM page p
JOIN duplicate_group_members m ON m.group_id = p.id
JOIN tracks t ON t.id = m.track_id
JOIN albums ON albums.id = t.album_id
JOIN artists ON artists.id = t.artist_id
ORDER BY p.detected_at DESC, p.id, t.id
`
type ListPendingDuplicateGroupMembersParams struct {
PageOffset int32
PageLimit int32
}
type ListPendingDuplicateGroupMembersRow struct {
GroupID pgtype.UUID
Tier string
WorstBitErrorRate *float32
DetectedAt pgtype.Timestamptz
TrackID pgtype.UUID
Title string
ArtistName string
AlbumID pgtype.UUID
AlbumTitle string
FilePath string
FileFormat string
FileSize int64
DurationMs int32
AddedAt pgtype.Timestamptz
LikeCount int64
PlayCount int64
}
// One page of proposals, newest first, flattened to one row per member so the
// handler folds them without a query per group. What each copy carries — likes
// and plays from every user — is here because it is what the operator weighs
// when deciding which copy to keep.
func (q *Queries) ListPendingDuplicateGroupMembers(ctx context.Context, arg ListPendingDuplicateGroupMembersParams) ([]ListPendingDuplicateGroupMembersRow, error) {
rows, err := q.db.Query(ctx, listPendingDuplicateGroupMembers, arg.PageOffset, arg.PageLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListPendingDuplicateGroupMembersRow
for rows.Next() {
var i ListPendingDuplicateGroupMembersRow
if err := rows.Scan(
&i.GroupID,
&i.Tier,
&i.WorstBitErrorRate,
&i.DetectedAt,
&i.TrackID,
&i.Title,
&i.ArtistName,
&i.AlbumID,
&i.AlbumTitle,
&i.FilePath,
&i.FileFormat,
&i.FileSize,
&i.DurationMs,
&i.AddedAt,
&i.LikeCount,
&i.PlayCount,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const startDuplicateSweep = `-- name: StartDuplicateSweep :one
INSERT INTO duplicate_sweeps DEFAULT VALUES RETURNING id, started_at
`
type StartDuplicateSweepRow struct {
ID pgtype.UUID
StartedAt pgtype.Timestamptz
}
func (q *Queries) StartDuplicateSweep(ctx context.Context) (StartDuplicateSweepRow, error) {
row := q.db.QueryRow(ctx, startDuplicateSweep)
var i StartDuplicateSweepRow
err := row.Scan(&i.ID, &i.StartedAt)
return i, err
}
const upsertDuplicateGroup = `-- name: UpsertDuplicateGroup :one
INSERT INTO duplicate_groups (member_key, tier, worst_bit_error_rate, last_seen_sweep_id)
VALUES ($1, $2, $3, $4)
ON CONFLICT (member_key) DO UPDATE
SET tier = EXCLUDED.tier,
worst_bit_error_rate = EXCLUDED.worst_bit_error_rate,
last_seen_sweep_id = EXCLUDED.last_seen_sweep_id
WHERE duplicate_groups.status = 'pending'
RETURNING id
`
type UpsertDuplicateGroupParams struct {
MemberKey string
Tier string
WorstBitErrorRate *float32
SweepID pgtype.UUID
}
// Proposes a group, or refreshes one already pending. A group already dismissed
// or merged is left exactly as it is: the WHERE on the update makes the conflict
// a no-op, and the caller sees no row.
func (q *Queries) UpsertDuplicateGroup(ctx context.Context, arg UpsertDuplicateGroupParams) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, upsertDuplicateGroup,
arg.MemberKey,
arg.Tier,
arg.WorstBitErrorRate,
arg.SweepID,
)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
}
@@ -1,72 +0,0 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: fingerprint_settings.sql
package dbq
import (
"context"
)
const getFingerprintSettings = `-- name: GetFingerprintSettings :one
SELECT id, enabled, chromaprint_length_sec, acoustic_max_bit_error_rate, backfill_concurrency, sweep_interval_hours, updated_at FROM fingerprint_settings WHERE id = true
`
func (q *Queries) GetFingerprintSettings(ctx context.Context) (FingerprintSetting, error) {
row := q.db.QueryRow(ctx, getFingerprintSettings)
var i FingerprintSetting
err := row.Scan(
&i.ID,
&i.Enabled,
&i.ChromaprintLengthSec,
&i.AcousticMaxBitErrorRate,
&i.BackfillConcurrency,
&i.SweepIntervalHours,
&i.UpdatedAt,
)
return i, err
}
const updateFingerprintSettings = `-- name: UpdateFingerprintSettings :one
UPDATE fingerprint_settings
SET enabled = $1,
chromaprint_length_sec = $2,
acoustic_max_bit_error_rate = $3,
backfill_concurrency = $4,
sweep_interval_hours = $5,
updated_at = now()
WHERE id = true
RETURNING id, enabled, chromaprint_length_sec, acoustic_max_bit_error_rate, backfill_concurrency, sweep_interval_hours, updated_at
`
type UpdateFingerprintSettingsParams struct {
Enabled bool
ChromaprintLengthSec int32
AcousticMaxBitErrorRate float64
BackfillConcurrency int32
SweepIntervalHours int32
}
// Whole-row write from the admin card; migration 0061's CHECKs are the backstop
// behind the service's own validation.
func (q *Queries) UpdateFingerprintSettings(ctx context.Context, arg UpdateFingerprintSettingsParams) (FingerprintSetting, error) {
row := q.db.QueryRow(ctx, updateFingerprintSettings,
arg.Enabled,
arg.ChromaprintLengthSec,
arg.AcousticMaxBitErrorRate,
arg.BackfillConcurrency,
arg.SweepIntervalHours,
)
var i FingerprintSetting
err := row.Scan(
&i.ID,
&i.Enabled,
&i.ChromaprintLengthSec,
&i.AcousticMaxBitErrorRate,
&i.BackfillConcurrency,
&i.SweepIntervalHours,
&i.UpdatedAt,
)
return i, err
}
-172
View File
@@ -1,172 +0,0 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: fingerprints.sql
package dbq
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const deleteTrackFingerprint = `-- name: DeleteTrackFingerprint :exec
DELETE FROM track_fingerprints WHERE track_id = $1
`
// A file changed but could not be fingerprinted, for a reason unrelated to the
// file. The stored row describes the OLD bytes, so it goes and the backfill
// re-derives it — nothing may keep trusting a stale identity.
func (q *Queries) DeleteTrackFingerprint(ctx context.Context, trackID pgtype.UUID) error {
_, err := q.db.Exec(ctx, deleteTrackFingerprint, trackID)
return err
}
const getFingerprintCoverage = `-- name: GetFingerprintCoverage :one
SELECT count(*)::bigint AS total,
count(*) FILTER (
WHERE f.fingerprint_version >= $1
AND f.chromaprint_length_sec = $2
AND f.audio_stream_sha256 IS NOT NULL AND f.chromaprint IS NOT NULL
)::bigint AS fingerprinted,
count(*) FILTER (
WHERE f.fingerprint_version >= $1
AND f.chromaprint_length_sec = $2
AND (f.audio_stream_sha256 IS NULL OR f.chromaprint IS NULL)
)::bigint AS rejected,
count(*) FILTER (
WHERE f.track_id IS NULL
OR f.fingerprint_version < $1
OR f.chromaprint_length_sec <> $2
)::bigint AS pending
FROM tracks t
LEFT JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NULL
`
type GetFingerprintCoverageParams struct {
CurrentVersion int16
ChromaprintLengthSec int32
}
type GetFingerprintCoverageRow struct {
Total int64
Fingerprinted int64
Rejected int64
Pending int64
}
// The admin gauge for the backfill. fingerprinted + rejected + pending = total.
// "Current" means derived by the current method AT the current length: a row at
// another length is pending, because the backfill will re-derive it. rejected is
// a current row with a NULL half: a tool ran and refused the file, which is
// settled rather than waiting. Missing tracks are excluded, or the gauge could
// never reach the end.
func (q *Queries) GetFingerprintCoverage(ctx context.Context, arg GetFingerprintCoverageParams) (GetFingerprintCoverageRow, error) {
row := q.db.QueryRow(ctx, getFingerprintCoverage, arg.CurrentVersion, arg.ChromaprintLengthSec)
var i GetFingerprintCoverageRow
err := row.Scan(
&i.Total,
&i.Fingerprinted,
&i.Rejected,
&i.Pending,
)
return i, err
}
const listTracksNeedingFingerprint = `-- name: ListTracksNeedingFingerprint :many
SELECT t.id, t.file_path
FROM tracks t
LEFT JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NULL
-- A row taken at another length is as stale as one from an older method:
-- chromaprints at two lengths cannot be compared (#3913).
AND (f.track_id IS NULL
OR f.fingerprint_version < $1
OR f.chromaprint_length_sec <> $2)
AND t.id > $3
ORDER BY t.id
LIMIT $4
`
type ListTracksNeedingFingerprintParams struct {
CurrentVersion int16
ChromaprintLengthSec int32
AfterID pgtype.UUID
BatchLimit int32
}
type ListTracksNeedingFingerprintRow struct {
ID pgtype.UUID
FilePath string
}
// The backfill's work queue (#3908): tracks with no fingerprint, or one derived
// by an older method. Keyset-paged on id so a pass visits each track at most
// once. That cursor is load-bearing: an inconclusive attempt writes no row, so
// without it a file that keeps timing out would be listed again straight away
// and retried in a tight loop. Missing tracks are skipped — there is no file to
// read.
func (q *Queries) ListTracksNeedingFingerprint(ctx context.Context, arg ListTracksNeedingFingerprintParams) ([]ListTracksNeedingFingerprintRow, error) {
rows, err := q.db.Query(ctx, listTracksNeedingFingerprint,
arg.CurrentVersion,
arg.ChromaprintLengthSec,
arg.AfterID,
arg.BatchLimit,
)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListTracksNeedingFingerprintRow
for rows.Next() {
var i ListTracksNeedingFingerprintRow
if err := rows.Scan(&i.ID, &i.FilePath); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const upsertTrackFingerprint = `-- name: UpsertTrackFingerprint :exec
INSERT INTO track_fingerprints (
track_id, audio_stream_sha256, chromaprint, fingerprint_version, chromaprint_length_sec
) VALUES (
$1, $2, $3,
$4, $5
)
ON CONFLICT (track_id) DO UPDATE SET
audio_stream_sha256 = EXCLUDED.audio_stream_sha256,
chromaprint = EXCLUDED.chromaprint,
fingerprint_version = EXCLUDED.fingerprint_version,
chromaprint_length_sec = EXCLUDED.chromaprint_length_sec,
computed_at = now()
`
type UpsertTrackFingerprintParams struct {
TrackID pgtype.UUID
AudioStreamSha256 []byte
Chromaprint []int32
FingerprintVersion int16
ChromaprintLengthSec int32
}
// Written whenever a track's fingerprint is derived: by the scan when a file is
// new or its bytes changed, and by the backfill (#3908) for rows derived by an
// older method. Replaces the row wholesale — a fingerprint of the old bytes has
// no standing once the file has changed.
func (q *Queries) UpsertTrackFingerprint(ctx context.Context, arg UpsertTrackFingerprintParams) error {
_, err := q.db.Exec(ctx, upsertTrackFingerprint,
arg.TrackID,
arg.AudioStreamSha256,
arg.Chromaprint,
arg.FingerprintVersion,
arg.ChromaprintLengthSec,
)
return err
}
-16
View File
@@ -84,22 +84,6 @@ func (q *Queries) GcDeleteExpiredPasswordResets(ctx context.Context) (int64, err
return result.RowsAffected(), nil
}
const gcDeleteExpiredSessions = `-- name: GcDeleteExpiredSessions :execrows
DELETE FROM sessions
WHERE last_seen_at <= now() - interval '30 days'
OR created_at <= now() - interval '365 days'
`
// Sessions past their idle (30 days) or absolute (1 year) limit. They already
// fail auth through GetSessionByTokenHash's filter; this only clears the rows.
func (q *Queries) GcDeleteExpiredSessions(ctx context.Context) (int64, error) {
result, err := q.db.Exec(ctx, gcDeleteExpiredSessions)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const gcExpireScrobbleQueueFailedRows = `-- name: GcExpireScrobbleQueueFailedRows :execrows
DELETE FROM scrobble_queue
WHERE status = 'failed'
-339
View File
@@ -1,339 +0,0 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: merge.sql
package dbq
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const listDuplicateGroupMergeMembers = `-- name: ListDuplicateGroupMergeMembers :many
SELECT t.id, t.file_path, t.file_format, t.file_size, t.added_at, t.album_id,
t.mbid, albums.mbid AS album_mbid
FROM duplicate_group_members m
JOIN tracks t ON t.id = m.track_id
JOIN albums ON albums.id = t.album_id
WHERE m.group_id = $1
ORDER BY t.id
`
type ListDuplicateGroupMergeMembersRow struct {
ID pgtype.UUID
FilePath string
FileFormat string
FileSize int64
AddedAt pgtype.Timestamptz
AlbumID pgtype.UUID
Mbid *string
AlbumMbid *string
}
func (q *Queries) ListDuplicateGroupMergeMembers(ctx context.Context, groupID pgtype.UUID) ([]ListDuplicateGroupMergeMembersRow, error) {
rows, err := q.db.Query(ctx, listDuplicateGroupMergeMembers, groupID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListDuplicateGroupMergeMembersRow
for rows.Next() {
var i ListDuplicateGroupMergeMembersRow
if err := rows.Scan(
&i.ID,
&i.FilePath,
&i.FileFormat,
&i.FileSize,
&i.AddedAt,
&i.AlbumID,
&i.Mbid,
&i.AlbumMbid,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const lockDuplicateGroupForMerge = `-- name: LockDuplicateGroupForMerge :one
SELECT id, tier, status
FROM duplicate_groups
WHERE id = $1
FOR UPDATE
`
type LockDuplicateGroupForMergeRow struct {
ID pgtype.UUID
Tier string
Status string
}
// Duplicate merge (Scribe #3911). Every statement here runs inside the one
// transaction library.MergeDuplicateGroup opens, after the removed copy's file
// is already gone. The loser's own track row is deleted last with DeleteTrack;
// what these do is move everything it carries onto the survivor first, so that
// delete's CASCADE finds nothing left to destroy.
// Locks the group for the rest of the transaction, so two merges of one group
// cannot run at once.
func (q *Queries) LockDuplicateGroupForMerge(ctx context.Context, id pgtype.UUID) (LockDuplicateGroupForMergeRow, error) {
row := q.db.QueryRow(ctx, lockDuplicateGroupForMerge, id)
var i LockDuplicateGroupForMergeRow
err := row.Scan(&i.ID, &i.Tier, &i.Status)
return i, err
}
const markDuplicateGroupMerged = `-- name: MarkDuplicateGroupMerged :execrows
UPDATE duplicate_groups
SET status = 'merged', resolved_at = now()
WHERE id = $1 AND status = 'pending'
`
func (q *Queries) MarkDuplicateGroupMerged(ctx context.Context, id pgtype.UUID) (int64, error) {
result, err := q.db.Exec(ctx, markDuplicateGroupMerged, id)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const mergeCopyGeneralLikes = `-- name: MergeCopyGeneralLikes :many
INSERT INTO general_likes (user_id, track_id, liked_at)
SELECT user_id, $1::uuid, liked_at
FROM general_likes
WHERE track_id = $2::uuid
ON CONFLICT (user_id, track_id) DO UPDATE
SET liked_at = LEAST(general_likes.liked_at, EXCLUDED.liked_at)
RETURNING user_id
`
type MergeCopyGeneralLikesParams struct {
SurvivorID pgtype.UUID
LoserID pgtype.UUID
}
// Collision-safe merges: a unique key includes track_id, so the survivor may
// already hold a matching row. Copy what it lacks; DeleteTrack's CASCADE then
// removes the loser's originals.
// One like per user. A user who liked both copies keeps a single like, dated to
// the earlier of the two.
func (q *Queries) MergeCopyGeneralLikes(ctx context.Context, arg MergeCopyGeneralLikesParams) ([]pgtype.UUID, error) {
rows, err := q.db.Query(ctx, mergeCopyGeneralLikes, arg.SurvivorID, arg.LoserID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []pgtype.UUID
for rows.Next() {
var user_id pgtype.UUID
if err := rows.Scan(&user_id); err != nil {
return nil, err
}
items = append(items, user_id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const mergeCopyTrackSimilarity = `-- name: MergeCopyTrackSimilarity :execrows
INSERT INTO track_similarity (track_a_id, track_b_id, score, source, fetched_at)
SELECT CASE WHEN track_a_id = $1::uuid THEN $2::uuid ELSE track_a_id END,
CASE WHEN track_b_id = $1::uuid THEN $2::uuid ELSE track_b_id END,
score, source, fetched_at
FROM track_similarity
WHERE (track_a_id = $1::uuid OR track_b_id = $1::uuid)
AND (CASE WHEN track_a_id = $1::uuid THEN $2::uuid ELSE track_a_id END)
<> (CASE WHEN track_b_id = $1::uuid THEN $2::uuid ELSE track_b_id END)
ON CONFLICT (track_a_id, track_b_id, source) DO NOTHING
`
type MergeCopyTrackSimilarityParams struct {
LoserID pgtype.UUID
SurvivorID pgtype.UUID
}
// Rewrites the loser to the survivor on either side of an edge. An edge between
// the two copies would become a track similar to itself — the table forbids
// that, and it means nothing — so it is dropped. An edge the survivor already
// has from the same source is kept as it is.
func (q *Queries) MergeCopyTrackSimilarity(ctx context.Context, arg MergeCopyTrackSimilarityParams) (int64, error) {
result, err := q.db.Exec(ctx, mergeCopyTrackSimilarity, arg.LoserID, arg.SurvivorID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const mergeCopyTrackTags = `-- name: MergeCopyTrackTags :execrows
INSERT INTO track_tags (track_id, tag, weight)
SELECT $1::uuid, tag, weight
FROM track_tags
WHERE track_id = $2::uuid
ON CONFLICT (track_id, tag) DO NOTHING
`
type MergeCopyTrackTagsParams struct {
SurvivorID pgtype.UUID
LoserID pgtype.UUID
}
func (q *Queries) MergeCopyTrackTags(ctx context.Context, arg MergeCopyTrackTagsParams) (int64, error) {
result, err := q.db.Exec(ctx, mergeCopyTrackTags, arg.SurvivorID, arg.LoserID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const mergeInheritTrackMbid = `-- name: MergeInheritTrackMbid :exec
UPDATE tracks AS survivor
SET mbid = loser.mbid
FROM tracks AS loser
WHERE survivor.id = $1::uuid
AND loser.id = $2::uuid
AND survivor.mbid IS NULL
AND loser.mbid IS NOT NULL
`
type MergeInheritTrackMbidParams struct {
SurvivorID pgtype.UUID
LoserID pgtype.UUID
}
// A recording MBID is what the similarity pipeline keys on. If only the removed
// copy carried one, the survivor takes it rather than going dark to similarity.
func (q *Queries) MergeInheritTrackMbid(ctx context.Context, arg MergeInheritTrackMbidParams) error {
_, err := q.db.Exec(ctx, mergeInheritTrackMbid, arg.SurvivorID, arg.LoserID)
return err
}
const mergeRepointContextualLikes = `-- name: MergeRepointContextualLikes :execrows
UPDATE contextual_likes SET track_id = $1::uuid WHERE track_id = $2::uuid
`
type MergeRepointContextualLikesParams struct {
SurvivorID pgtype.UUID
LoserID pgtype.UUID
}
func (q *Queries) MergeRepointContextualLikes(ctx context.Context, arg MergeRepointContextualLikesParams) (int64, error) {
result, err := q.db.Exec(ctx, mergeRepointContextualLikes, arg.SurvivorID, arg.LoserID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const mergeRepointLidarrRequests = `-- name: MergeRepointLidarrRequests :execrows
UPDATE lidarr_requests SET matched_track_id = $1::uuid
WHERE matched_track_id = $2::uuid
`
type MergeRepointLidarrRequestsParams struct {
SurvivorID pgtype.UUID
LoserID pgtype.UUID
}
func (q *Queries) MergeRepointLidarrRequests(ctx context.Context, arg MergeRepointLidarrRequestsParams) (int64, error) {
result, err := q.db.Exec(ctx, mergeRepointLidarrRequests, arg.SurvivorID, arg.LoserID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const mergeRepointPlayEvents = `-- name: MergeRepointPlayEvents :execrows
UPDATE play_events SET track_id = $1::uuid WHERE track_id = $2::uuid
`
type MergeRepointPlayEventsParams struct {
SurvivorID pgtype.UUID
LoserID pgtype.UUID
}
// Plain repoints: no unique key involves track_id, so moving rows cannot collide.
func (q *Queries) MergeRepointPlayEvents(ctx context.Context, arg MergeRepointPlayEventsParams) (int64, error) {
result, err := q.db.Exec(ctx, mergeRepointPlayEvents, arg.SurvivorID, arg.LoserID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const mergeRepointPlaybackErrors = `-- name: MergeRepointPlaybackErrors :execrows
UPDATE playback_errors SET track_id = $1::uuid WHERE track_id = $2::uuid
`
type MergeRepointPlaybackErrorsParams struct {
SurvivorID pgtype.UUID
LoserID pgtype.UUID
}
func (q *Queries) MergeRepointPlaybackErrors(ctx context.Context, arg MergeRepointPlaybackErrorsParams) (int64, error) {
result, err := q.db.Exec(ctx, mergeRepointPlaybackErrors, arg.SurvivorID, arg.LoserID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const mergeRepointPlaylistTracks = `-- name: MergeRepointPlaylistTracks :many
UPDATE playlist_tracks SET track_id = $1::uuid
WHERE track_id = $2::uuid
RETURNING playlist_id
`
type MergeRepointPlaylistTracksParams struct {
SurvivorID pgtype.UUID
LoserID pgtype.UUID
}
// playlist_tracks is keyed by (playlist_id, position), so repointing keeps every
// entry exactly where it was. A playlist that held both copies simply holds the
// survivor twice — the user put two entries there, and both stay.
func (q *Queries) MergeRepointPlaylistTracks(ctx context.Context, arg MergeRepointPlaylistTracksParams) ([]pgtype.UUID, error) {
rows, err := q.db.Query(ctx, mergeRepointPlaylistTracks, arg.SurvivorID, arg.LoserID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []pgtype.UUID
for rows.Next() {
var playlist_id pgtype.UUID
if err := rows.Scan(&playlist_id); err != nil {
return nil, err
}
items = append(items, playlist_id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const mergeRepointSkipEvents = `-- name: MergeRepointSkipEvents :execrows
UPDATE skip_events SET track_id = $1::uuid WHERE track_id = $2::uuid
`
type MergeRepointSkipEventsParams struct {
SurvivorID pgtype.UUID
LoserID pgtype.UUID
}
func (q *Queries) MergeRepointSkipEvents(ctx context.Context, arg MergeRepointSkipEventsParams) (int64, error) {
result, err := q.db.Exec(ctx, mergeRepointSkipEvents, arg.SurvivorID, arg.LoserID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
+1 -47
View File
@@ -297,42 +297,6 @@ type DiscoverTuning struct {
UpdatedAt pgtype.Timestamptz
}
type DuplicateGroup struct {
ID pgtype.UUID
MemberKey string
Tier string
WorstBitErrorRate *float32
Status string
DetectedAt pgtype.Timestamptz
LastSeenSweepID pgtype.UUID
ResolvedAt pgtype.Timestamptz
}
type DuplicateGroupMember struct {
GroupID pgtype.UUID
TrackID pgtype.UUID
}
type DuplicateSweep struct {
ID pgtype.UUID
StartedAt pgtype.Timestamptz
FinishedAt pgtype.Timestamptz
Candidates *int32
GroupsFound *int32
OversizeClusters *int32
ErrorMessage *string
}
type FingerprintSetting struct {
ID bool
Enabled bool
ChromaprintLengthSec int32
AcousticMaxBitErrorRate float64
BackfillConcurrency int32
SweepIntervalHours int32
UpdatedAt pgtype.Timestamptz
}
type GeneralLike struct {
UserID pgtype.UUID
TrackID pgtype.UUID
@@ -430,7 +394,6 @@ type MissingReacquisition struct {
type NetworkSetting struct {
ID bool
TrustedProxyHops int32
PublicUrl string
}
type PasswordReset struct {
@@ -704,15 +667,6 @@ type Track struct {
MissingSince pgtype.Timestamptz
}
type TrackFingerprint struct {
TrackID pgtype.UUID
AudioStreamSha256 []byte
Chromaprint []int32
FingerprintVersion int16
ComputedAt pgtype.Timestamptz
ChromaprintLengthSec int32
}
type TrackSimilarity struct {
TrackAID pgtype.UUID
TrackBID pgtype.UUID
@@ -731,6 +685,7 @@ type User struct {
ID pgtype.UUID
Username string
PasswordHash string
ApiToken string
IsAdmin bool
CreatedAt pgtype.Timestamptz
SubsonicPassword *string
@@ -742,7 +697,6 @@ type User struct {
Timezone string
TimezoneUpdatedAt pgtype.Timestamptz
DebugModeEnabled bool
ApiTokenHash string
}
type UserInvite struct {
+4 -15
View File
@@ -10,34 +10,23 @@ import (
)
const getNetworkSettings = `-- name: GetNetworkSettings :one
SELECT id, trusted_proxy_hops, public_url FROM network_settings WHERE id = true
SELECT id, trusted_proxy_hops FROM network_settings WHERE id = true
`
func (q *Queries) GetNetworkSettings(ctx context.Context) (NetworkSetting, error) {
row := q.db.QueryRow(ctx, getNetworkSettings)
var i NetworkSetting
err := row.Scan(&i.ID, &i.TrustedProxyHops, &i.PublicUrl)
return i, err
}
const updatePublicURL = `-- name: UpdatePublicURL :one
UPDATE network_settings SET public_url = $1 WHERE id = true RETURNING id, trusted_proxy_hops, public_url
`
func (q *Queries) UpdatePublicURL(ctx context.Context, publicUrl string) (NetworkSetting, error) {
row := q.db.QueryRow(ctx, updatePublicURL, publicUrl)
var i NetworkSetting
err := row.Scan(&i.ID, &i.TrustedProxyHops, &i.PublicUrl)
err := row.Scan(&i.ID, &i.TrustedProxyHops)
return i, err
}
const updateTrustedProxyHops = `-- name: UpdateTrustedProxyHops :one
UPDATE network_settings SET trusted_proxy_hops = $1 WHERE id = true RETURNING id, trusted_proxy_hops, public_url
UPDATE network_settings SET trusted_proxy_hops = $1 WHERE id = true RETURNING id, trusted_proxy_hops
`
func (q *Queries) UpdateTrustedProxyHops(ctx context.Context, trustedProxyHops int32) (NetworkSetting, error) {
row := q.db.QueryRow(ctx, updateTrustedProxyHops, trustedProxyHops)
var i NetworkSetting
err := row.Scan(&i.ID, &i.TrustedProxyHops, &i.PublicUrl)
err := row.Scan(&i.ID, &i.TrustedProxyHops)
return i, err
}
+16 -32
View File
@@ -829,7 +829,7 @@ similar_artists AS (
JOIN seed_artist sa ON asim.artist_a_id = sa.artist_id
WHERE asim.source = 'listenbrainz'
AND t.id NOT IN (SELECT id FROM excluded_ids)
ORDER BY asim.score DESC, md5(t.id::text || $12::text)
ORDER BY asim.score DESC, random()
LIMIT $6
),
tag_overlap AS (
@@ -857,7 +857,7 @@ likes_overlap AS (
WHERE t.id = gl.track_id
AND trim(g_overlap.g) IN (SELECT tag FROM seed_tags)
)
ORDER BY md5(gl.track_id::text || $12::text)
ORDER BY random()
LIMIT $8
),
taste_overlap AS (
@@ -884,7 +884,7 @@ coplay_artists AS (
WHERE asim.source = 'user_cooccurrence'
AND t.id NOT IN (SELECT id FROM excluded_ids)
AND t.id <> $2
ORDER BY asim.score DESC, md5(t.id::text || $12::text)
ORDER BY asim.score DESC, random()
LIMIT $11
),
random_fill AS (
@@ -900,7 +900,7 @@ random_fill AS (
UNION SELECT track_id FROM taste_overlap
UNION SELECT track_id FROM coplay_artists
)
ORDER BY md5(t.id::text || $12::text)
ORDER BY random()
LIMIT $9
)
SELECT
@@ -938,18 +938,17 @@ GROUP BY t.id, t.title, t.album_id, t.artist_id, t.duration_ms, t.file_path,
`
type LoadRadioCandidatesV2Params struct {
UserID pgtype.UUID
ID pgtype.UUID
Column3 interface{}
Column4 []pgtype.UUID
Limit int32
Limit_2 int32
Limit_3 int32
Limit_4 int32
Limit_5 int32
Limit_6 int32
Limit_7 int32
Column12 string
UserID pgtype.UUID
ID pgtype.UUID
Column3 interface{}
Column4 []pgtype.UUID
Limit int32
Limit_2 int32
Limit_3 int32
Limit_4 int32
Limit_5 int32
Limit_6 int32
Limit_7 int32
}
type LoadRadioCandidatesV2Row struct {
@@ -972,22 +971,8 @@ type LoadRadioCandidatesV2Row struct {
// enter the pool even when the similarity/random arms miss them; scored
// in Go via TasteMatch, so sim_score here is 0 pool-inclusion),
// $11 coplay_artists K (#1533 — tracks by artists co-played across the
// instance with the seed's artist; source='user_cooccurrence'),
// $12 order_seed (text) — see below.
// instance with the seed's artist; source='user_cooccurrence').
//
// $12 REPLACES `ORDER BY random()` IN FOUR ARMS (#3889). Those arms returned
// a stable set only while their LIMIT exceeded the rows eligible for them: at
// that point they returned all of them and the order stopped mattering,
// because the caller sorts by track id before scoring. Below that threshold
// they returned a random SUBSET, and two builds on the same day drew
// different ones — so "daily determinism" held by accident, and only for
// libraries smaller than the limits.
//
// md5(id || seed) keeps the intent — an arbitrary spread that changes when
// the seed does — while making it reproducible for a given seed. The CALLER
// decides what that means: system mixes pass a per-(user, day) string and get
// the determinism they promise; radio passes a fresh value per request and
// keeps varying, which is what a radio should do.
// Returns same shape as LoadRadioCandidates plus similarity_score column.
func (q *Queries) LoadRadioCandidatesV2(ctx context.Context, arg LoadRadioCandidatesV2Params) ([]LoadRadioCandidatesV2Row, error) {
rows, err := q.db.Query(ctx, loadRadioCandidatesV2,
@@ -1002,7 +987,6 @@ func (q *Queries) LoadRadioCandidatesV2(ctx context.Context, arg LoadRadioCandid
arg.Limit_5,
arg.Limit_6,
arg.Limit_7,
arg.Column12,
)
if err != nil {
return nil, err
+2 -36
View File
@@ -69,38 +69,10 @@ func (q *Queries) DeleteSessionForUser(ctx context.Context, arg DeleteSessionFor
return result.RowsAffected(), nil
}
const deleteSessionsForUser = `-- name: DeleteSessionsForUser :execrows
DELETE FROM sessions WHERE user_id = $1
`
// Every session the user has, the caller's included. A password reset uses
// it: whoever forgot the password is not signed in anywhere they trust, and
// whoever may have learned it must be signed out everywhere.
func (q *Queries) DeleteSessionsForUser(ctx context.Context, userID pgtype.UUID) (int64, error) {
result, err := q.db.Exec(ctx, deleteSessionsForUser, userID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const getSessionByTokenHash = `-- name: GetSessionByTokenHash :one
SELECT id, user_id, token_hash, user_agent, created_at, last_seen_at, created_ip, last_ip FROM sessions
WHERE token_hash = $1
AND last_seen_at > now() - interval '30 days'
AND created_at > now() - interval '365 days'
SELECT id, user_id, token_hash, user_agent, created_at, last_seen_at, created_ip, last_ip FROM sessions WHERE token_hash = $1
`
// Expired sessions are invisible here, so they fail auth the moment they
// lapse rather than whenever the GC sweep next runs. Two limits:
//
// idle 30 days — a token nobody has used in a month is abandoned, and
// matches the web cookie's lifetime;
// absolute 1 year — even a token in daily use is re-issued yearly, so a
// stolen one that is being used quietly does not live
// forever.
//
// Keep in step with ListSessionsForUser and GcDeleteExpiredSessions.
func (q *Queries) GetSessionByTokenHash(ctx context.Context, tokenHash []byte) (Session, error) {
row := q.db.QueryRow(ctx, getSessionByTokenHash, tokenHash)
var i Session
@@ -155,17 +127,11 @@ func (q *Queries) InsertSession(ctx context.Context, arg InsertSessionParams) (S
}
const listSessionsForUser = `-- name: ListSessionsForUser :many
SELECT id, user_id, token_hash, user_agent, created_at, last_seen_at, created_ip, last_ip FROM sessions
WHERE user_id = $1
AND last_seen_at > now() - interval '30 days'
AND created_at > now() - interval '365 days'
ORDER BY last_seen_at DESC
SELECT id, user_id, token_hash, user_agent, created_at, last_seen_at, created_ip, last_ip FROM sessions WHERE user_id = $1 ORDER BY last_seen_at DESC
`
// Most-recently-active first: the row a user is most likely to act on is the
// one that moved last, and an unfamiliar entry at the top is the alarm.
// Expired rows are left out: they no longer authenticate, so listing them
// as active would be wrong. Same limits as GetSessionByTokenHash.
func (q *Queries) ListSessionsForUser(ctx context.Context, userID pgtype.UUID) ([]Session, error) {
rows, err := q.db.Query(ctx, listSessionsForUser, userID)
if err != nil {
+19 -16
View File
@@ -148,36 +148,39 @@ func (q *Queries) DeleteTrack(ctx context.Context, id pgtype.UUID) (DeleteTrackR
return i, err
}
const findMissingTrackByAudioHash = `-- name: FindMissingTrackByAudioHash :many
SELECT t.id, t.file_path
FROM tracks t
JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NOT NULL
AND f.audio_stream_sha256 = $1
const findMissingTrackByFingerprint = `-- name: FindMissingTrackByFingerprint :many
SELECT id, file_path FROM tracks
WHERE missing_since IS NOT NULL
AND file_size = $1
AND duration_ms = $2
LIMIT 2
`
type FindMissingTrackByAudioHashRow struct {
type FindMissingTrackByFingerprintParams struct {
FileSize int64
DurationMs int32
}
type FindMissingTrackByFingerprintRow struct {
ID pgtype.UUID
FilePath string
}
// Move detection fallback for files with no MBID (#2528, #3914). The audio stream
// hash identifies the encoded audio itself, so it survives a rename, a move and a
// retag — anything short of a re-encode. It replaced (file_size, duration_ms),
// which could pair two unrelated files that happened to share a byte count and a
// duration, and missed a file retagged in place, whose size changes.
// Move detection fallback for files with no MBID (#2528). Exact byte size AND
// exact decoded duration is a strong pair: a plain move or rename preserves
// both, while a re-encode changes at least one — and a re-encode genuinely is a
// different file, so failing to match there is correct rather than a gap.
//
// Same missing-only constraint and same LIMIT 2 rationale as the MBID variant.
func (q *Queries) FindMissingTrackByAudioHash(ctx context.Context, audioStreamSha256 []byte) ([]FindMissingTrackByAudioHashRow, error) {
rows, err := q.db.Query(ctx, findMissingTrackByAudioHash, audioStreamSha256)
func (q *Queries) FindMissingTrackByFingerprint(ctx context.Context, arg FindMissingTrackByFingerprintParams) ([]FindMissingTrackByFingerprintRow, error) {
rows, err := q.db.Query(ctx, findMissingTrackByFingerprint, arg.FileSize, arg.DurationMs)
if err != nil {
return nil, err
}
defer rows.Close()
var items []FindMissingTrackByAudioHashRow
var items []FindMissingTrackByFingerprintRow
for rows.Next() {
var i FindMissingTrackByAudioHashRow
var i FindMissingTrackByFingerprintRow
if err := rows.Scan(&i.ID, &i.FilePath); err != nil {
return nil, err
}
+63 -45
View File
@@ -52,15 +52,15 @@ func (q *Queries) CountUsers(ctx context.Context) (int64, error) {
}
const createUser = `-- name: CreateUser :one
INSERT INTO users (username, password_hash, api_token_hash, is_admin, display_name)
INSERT INTO users (username, password_hash, api_token, is_admin, display_name)
VALUES ($1, $2, $3, $4, $5)
RETURNING id, username, password_hash, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled, api_token_hash
RETURNING id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled
`
type CreateUserParams struct {
Username string
PasswordHash string
ApiTokenHash string
ApiToken string
IsAdmin bool
DisplayName *string
}
@@ -69,7 +69,7 @@ func (q *Queries) CreateUser(ctx context.Context, arg CreateUserParams) (User, e
row := q.db.QueryRow(ctx, createUser,
arg.Username,
arg.PasswordHash,
arg.ApiTokenHash,
arg.ApiToken,
arg.IsAdmin,
arg.DisplayName,
)
@@ -78,6 +78,7 @@ func (q *Queries) CreateUser(ctx context.Context, arg CreateUserParams) (User, e
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
@@ -89,21 +90,20 @@ func (q *Queries) CreateUser(ctx context.Context, arg CreateUserParams) (User, e
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
&i.ApiTokenHash,
)
return i, err
}
const createUserAdmin = `-- name: CreateUserAdmin :one
INSERT INTO users (username, password_hash, api_token_hash, is_admin, display_name)
INSERT INTO users (username, password_hash, api_token, is_admin, display_name)
VALUES ($1, $2, $3, $4, $5)
RETURNING id, username, password_hash, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled, api_token_hash
RETURNING id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled
`
type CreateUserAdminParams struct {
Username string
PasswordHash string
ApiTokenHash string
ApiToken string
IsAdmin bool
DisplayName *string
}
@@ -115,7 +115,7 @@ func (q *Queries) CreateUserAdmin(ctx context.Context, arg CreateUserAdminParams
row := q.db.QueryRow(ctx, createUserAdmin,
arg.Username,
arg.PasswordHash,
arg.ApiTokenHash,
arg.ApiToken,
arg.IsAdmin,
arg.DisplayName,
)
@@ -124,6 +124,7 @@ func (q *Queries) CreateUserAdmin(ctx context.Context, arg CreateUserAdminParams
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
@@ -135,25 +136,24 @@ func (q *Queries) CreateUserAdmin(ctx context.Context, arg CreateUserAdminParams
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
&i.ApiTokenHash,
)
return i, err
}
const createUserFirstAdminRace = `-- name: CreateUserFirstAdminRace :one
INSERT INTO users (username, password_hash, api_token_hash, is_admin, display_name)
INSERT INTO users (username, password_hash, api_token, is_admin, display_name)
VALUES (
$1, $2, $3,
(SELECT NOT EXISTS (SELECT 1 FROM users)),
$4
)
RETURNING id, username, password_hash, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled, api_token_hash
RETURNING id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled
`
type CreateUserFirstAdminRaceParams struct {
Username string
PasswordHash string
ApiTokenHash string
ApiToken string
DisplayName *string
}
@@ -174,7 +174,7 @@ func (q *Queries) CreateUserFirstAdminRace(ctx context.Context, arg CreateUserFi
row := q.db.QueryRow(ctx, createUserFirstAdminRace,
arg.Username,
arg.PasswordHash,
arg.ApiTokenHash,
arg.ApiToken,
arg.DisplayName,
)
var i User
@@ -182,6 +182,7 @@ func (q *Queries) CreateUserFirstAdminRace(ctx context.Context, arg CreateUserFi
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
@@ -193,7 +194,6 @@ func (q *Queries) CreateUserFirstAdminRace(ctx context.Context, arg CreateUserFi
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
&i.ApiTokenHash,
)
return i, err
}
@@ -240,7 +240,7 @@ func (q *Queries) GetListenBrainzConfig(ctx context.Context, id pgtype.UUID) (Ge
}
const getOldestAdmin = `-- name: GetOldestAdmin :one
SELECT id, username, password_hash, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled, api_token_hash FROM users
SELECT id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled FROM users
WHERE is_admin = true
ORDER BY created_at, id
LIMIT 1
@@ -260,6 +260,7 @@ func (q *Queries) GetOldestAdmin(ctx context.Context) (User, error) {
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
@@ -271,22 +272,22 @@ func (q *Queries) GetOldestAdmin(ctx context.Context) (User, error) {
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
&i.ApiTokenHash,
)
return i, err
}
const getUserByAPITokenHash = `-- name: GetUserByAPITokenHash :one
SELECT id, username, password_hash, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled, api_token_hash FROM users WHERE api_token_hash = $1
const getUserByAPIToken = `-- name: GetUserByAPIToken :one
SELECT id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled FROM users WHERE api_token = $1
`
func (q *Queries) GetUserByAPITokenHash(ctx context.Context, apiTokenHash string) (User, error) {
row := q.db.QueryRow(ctx, getUserByAPITokenHash, apiTokenHash)
func (q *Queries) GetUserByAPIToken(ctx context.Context, apiToken string) (User, error) {
row := q.db.QueryRow(ctx, getUserByAPIToken, apiToken)
var i User
err := row.Scan(
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
@@ -298,13 +299,12 @@ func (q *Queries) GetUserByAPITokenHash(ctx context.Context, apiTokenHash string
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
&i.ApiTokenHash,
)
return i, err
}
const getUserByEmail = `-- name: GetUserByEmail :one
SELECT id, username, password_hash, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled, api_token_hash FROM users WHERE lower(email) = lower($1)
SELECT id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled FROM users WHERE lower(email) = lower($1)
`
// Used by forgot-password lookup. Lowercase comparison both sides
@@ -317,6 +317,7 @@ func (q *Queries) GetUserByEmail(ctx context.Context, lower string) (User, error
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
@@ -328,13 +329,12 @@ func (q *Queries) GetUserByEmail(ctx context.Context, lower string) (User, error
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
&i.ApiTokenHash,
)
return i, err
}
const getUserByID = `-- name: GetUserByID :one
SELECT id, username, password_hash, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled, api_token_hash FROM users WHERE id = $1
SELECT id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled FROM users WHERE id = $1
`
func (q *Queries) GetUserByID(ctx context.Context, id pgtype.UUID) (User, error) {
@@ -344,6 +344,7 @@ func (q *Queries) GetUserByID(ctx context.Context, id pgtype.UUID) (User, error)
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
@@ -355,13 +356,12 @@ func (q *Queries) GetUserByID(ctx context.Context, id pgtype.UUID) (User, error)
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
&i.ApiTokenHash,
)
return i, err
}
const getUserByUsername = `-- name: GetUserByUsername :one
SELECT id, username, password_hash, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled, api_token_hash FROM users WHERE username = $1
SELECT id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled FROM users WHERE username = $1
`
func (q *Queries) GetUserByUsername(ctx context.Context, username string) (User, error) {
@@ -371,6 +371,7 @@ func (q *Queries) GetUserByUsername(ctx context.Context, username string) (User,
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
@@ -382,7 +383,6 @@ func (q *Queries) GetUserByUsername(ctx context.Context, username string) (User,
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
&i.ApiTokenHash,
)
return i, err
}
@@ -471,21 +471,39 @@ func (q *Queries) ListUsers(ctx context.Context) ([]ListUsersRow, error) {
return items, nil
}
const regenerateApiToken = `-- name: RegenerateApiToken :exec
UPDATE users SET api_token_hash = $2 WHERE id = $1
const regenerateApiToken = `-- name: RegenerateApiToken :one
UPDATE users SET api_token = $2 WHERE id = $1
RETURNING id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled
`
type RegenerateApiTokenParams struct {
ID pgtype.UUID
ApiTokenHash string
ID pgtype.UUID
ApiToken string
}
// Self-service: caller wants a new API token. Used by the /settings
// API Token card's "Regenerate" button. Only the hash is stored; the
// handler returns the raw key once.
func (q *Queries) RegenerateApiToken(ctx context.Context, arg RegenerateApiTokenParams) error {
_, err := q.db.Exec(ctx, regenerateApiToken, arg.ID, arg.ApiTokenHash)
return err
// API Token card's "Regenerate" button.
func (q *Queries) RegenerateApiToken(ctx context.Context, arg RegenerateApiTokenParams) (User, error) {
row := q.db.QueryRow(ctx, regenerateApiToken, arg.ID, arg.ApiToken)
var i User
err := row.Scan(
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
&i.ListenbrainzToken,
&i.ListenbrainzEnabled,
&i.DisplayName,
&i.AutoApproveRequests,
&i.Email,
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
)
return i, err
}
const resetUserPassword = `-- name: ResetUserPassword :exec
@@ -512,7 +530,7 @@ const setDebugMode = `-- name: SetDebugMode :one
UPDATE users
SET debug_mode_enabled = $2
WHERE id = $1
RETURNING id, username, password_hash, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled, api_token_hash
RETURNING id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled
`
type SetDebugModeParams struct {
@@ -530,6 +548,7 @@ func (q *Queries) SetDebugMode(ctx context.Context, arg SetDebugModeParams) (Use
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
@@ -541,7 +560,6 @@ func (q *Queries) SetDebugMode(ctx context.Context, arg SetDebugModeParams) (Use
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
&i.ApiTokenHash,
)
return i, err
}
@@ -599,7 +617,7 @@ const updateUserAdmin = `-- name: UpdateUserAdmin :one
UPDATE users
SET is_admin = $2
WHERE id = $1
RETURNING id, username, password_hash, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled, api_token_hash
RETURNING id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled
`
type UpdateUserAdminParams struct {
@@ -616,6 +634,7 @@ func (q *Queries) UpdateUserAdmin(ctx context.Context, arg UpdateUserAdminParams
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
@@ -627,7 +646,6 @@ func (q *Queries) UpdateUserAdmin(ctx context.Context, arg UpdateUserAdminParams
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
&i.ApiTokenHash,
)
return i, err
}
@@ -636,7 +654,7 @@ const updateUserAutoApprove = `-- name: UpdateUserAutoApprove :one
UPDATE users
SET auto_approve_requests = $2
WHERE id = $1
RETURNING id, username, password_hash, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled, api_token_hash
RETURNING id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled
`
type UpdateUserAutoApproveParams struct {
@@ -652,6 +670,7 @@ func (q *Queries) UpdateUserAutoApprove(ctx context.Context, arg UpdateUserAutoA
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
@@ -663,7 +682,6 @@ func (q *Queries) UpdateUserAutoApprove(ctx context.Context, arg UpdateUserAutoA
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
&i.ApiTokenHash,
)
return i, err
}
@@ -673,7 +691,7 @@ UPDATE users
SET display_name = $2,
email = $3
WHERE id = $1
RETURNING id, username, password_hash, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled, api_token_hash
RETURNING id, username, password_hash, api_token, is_admin, created_at, subsonic_password, listenbrainz_token, listenbrainz_enabled, display_name, auto_approve_requests, email, timezone, timezone_updated_at, debug_mode_enabled
`
type UpdateUserProfileParams struct {
@@ -691,6 +709,7 @@ func (q *Queries) UpdateUserProfile(ctx context.Context, arg UpdateUserProfilePa
&i.ID,
&i.Username,
&i.PasswordHash,
&i.ApiToken,
&i.IsAdmin,
&i.CreatedAt,
&i.SubsonicPassword,
@@ -702,7 +721,6 @@ func (q *Queries) UpdateUserProfile(ctx context.Context, arg UpdateUserProfilePa
&i.Timezone,
&i.TimezoneUpdatedAt,
&i.DebugModeEnabled,
&i.ApiTokenHash,
)
return i, err
}
@@ -1 +0,0 @@
DROP TABLE track_fingerprints;
@@ -1,38 +0,0 @@
-- 0058_track_fingerprints.up.sql — an acoustic identity per track (Scribe
-- milestone #400: #3905, #3906).
--
-- A table of its own rather than columns on tracks, for the hot path's sake:
-- tracks is read with SELECT * by eight queries, among them ListTracksByAlbum,
-- SearchTracks and GetTracksByIDs — album pages, search, the Subsonic surface.
-- A ~4 KB chromaprint column on tracks would be de-TOASTed on every one of
-- those reads to carry a value only the duplicate sweep ever looks at.
--
-- What a row means, which the backfill depends on:
-- no row never fingerprinted
-- fingerprint_version < current derived by an older method; re-derive it
-- fingerprint_version = current attempted; a NULL value means that tool
-- failed on this file, and it is not retried
-- until the file changes
-- A failure that says nothing about the file — a timeout, a cancelled scan, a
-- missing binary — writes no row at all, so the backfill tries again.
CREATE TABLE track_fingerprints (
-- CASCADE is right here, unlike for the likes and play history M400's
-- merge has to carry across: a fingerprint describes one file's bytes and
-- means nothing once that file's row is gone.
track_id uuid PRIMARY KEY REFERENCES tracks (id) ON DELETE CASCADE,
-- SHA-256 of the ENCODED audio packets (ffmpeg -c:a copy -f hash), not of
-- decoded samples. internal/library/fingerprint.go says why.
audio_stream_sha256 bytea
CHECK (audio_stream_sha256 IS NULL OR octet_length(audio_stream_sha256) = 32),
-- fpcalc -raw -signed: the same 32 bits per item, stored signed because
-- integer is.
chromaprint integer[],
fingerprint_version smallint NOT NULL,
computed_at timestamptz NOT NULL DEFAULT now()
);
-- The exact duplicate tier is an equality match on this column. Partial
-- because a NULL is never looked up — it only means the hash was not taken.
CREATE INDEX track_fingerprints_audio_stream_sha256
ON track_fingerprints (audio_stream_sha256)
WHERE audio_stream_sha256 IS NOT NULL;
@@ -1,4 +0,0 @@
DROP INDEX IF EXISTS tracks_duration_id_idx;
DROP TABLE duplicate_group_members;
DROP TABLE duplicate_groups;
DROP TABLE duplicate_sweeps;
@@ -1,54 +0,0 @@
-- 0059_duplicate_groups.up.sql — proposed duplicates and the sweeps that find
-- them (Scribe milestone #400: #3910).
--
-- The sweep compares fingerprints (track_fingerprints, 0058) and proposes groups
-- of tracks that hold one recording. Nothing here merges anything: a group is a
-- proposal the operator reviews, and the merge (#3911) is a separate act.
-- One row per sweep. Lets the report tell "the sweep has never run" apart from
-- "it ran and found nothing", and gives the in-flight guard something to check,
-- the same way scan_runs does for the library scan.
CREATE TABLE duplicate_sweeps (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
started_at timestamptz NOT NULL DEFAULT now(),
finished_at timestamptz,
candidates integer,
groups_found integer,
oversize_clusters integer,
error_message text
);
CREATE INDEX duplicate_sweeps_started_at_idx ON duplicate_sweeps (started_at DESC);
CREATE TABLE duplicate_groups (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
-- The group's identity: its member track ids, sorted and joined. A sweep
-- that finds the same tracks again updates this row rather than proposing
-- them twice, and a dismissal stays attached to the set it was made about.
member_key text NOT NULL UNIQUE,
-- Rule 36: a new value for either CHECK swaps the constraint in the same
-- migration.
tier text NOT NULL CHECK (tier IN ('exact', 'acoustic')),
-- Largest disagreement between any two members; NULL for exact groups,
-- which have no score.
worst_bit_error_rate real,
status text NOT NULL DEFAULT 'pending'
CHECK (status IN ('pending', 'dismissed', 'merged')),
detected_at timestamptz NOT NULL DEFAULT now(),
last_seen_sweep_id uuid REFERENCES duplicate_sweeps (id) ON DELETE SET NULL,
resolved_at timestamptz
);
CREATE INDEX duplicate_groups_status_idx ON duplicate_groups (status);
CREATE TABLE duplicate_group_members (
group_id uuid NOT NULL REFERENCES duplicate_groups (id) ON DELETE CASCADE,
-- CASCADE is right here: a track that genuinely leaves the library has no
-- place in a proposal about its duplicates.
track_id uuid NOT NULL REFERENCES tracks (id) ON DELETE CASCADE,
PRIMARY KEY (group_id, track_id)
);
CREATE INDEX duplicate_group_members_track_idx ON duplicate_group_members (track_id);
-- The sweep streams candidates in (duration_ms, id) order, keyset-paged, so it
-- only ever holds a few seconds' worth of durations in memory. Without this each
-- page would sort the whole library again.
CREATE INDEX tracks_duration_id_idx ON tracks (duration_ms, id);
@@ -1 +0,0 @@
DROP INDEX IF EXISTS play_events_track_idx;
@@ -1,8 +0,0 @@
-- 0060_play_events_track_index.up.sql — play_events by track (Scribe #3912, #3911).
--
-- play_events is indexed by (user_id, started_at) and (user_id, track_id), both
-- led by user. Nothing reached it by track alone until the duplicates report,
-- which shows each copy's play count — a scan of the whole table per copy — and
-- the merge (#3911), which repoints a duplicate's play history onto the copy
-- being kept. Both ask "every play of this track", whoever played it.
CREATE INDEX play_events_track_idx ON play_events (track_id);
@@ -1,2 +0,0 @@
ALTER TABLE track_fingerprints DROP COLUMN chromaprint_length_sec;
DROP TABLE fingerprint_settings;
@@ -1,50 +0,0 @@
-- 0061_fingerprint_settings.up.sql — fingerprinting's knobs, in admin Settings
-- (Scribe #3913, milestone #400). Rule 25: anything an operator might tune is a
-- database row, changed without a restart. Singleton in the style of
-- reacquisition_settings (0056).
CREATE TABLE fingerprint_settings (
id boolean PRIMARY KEY DEFAULT true,
-- Fingerprinting new files, the backfill, and the duplicate sweep. Off stops
-- the decode work entirely — the reason to turn it off is a slow NAS, and
-- that is the operator's call. On by default: a library that cannot tell its
-- duplicates apart is what milestone #400 exists to end.
enabled boolean NOT NULL DEFAULT true,
-- Seconds of audio fpcalc fingerprints. Chromaprints taken at different
-- lengths cannot be compared, which is why track_fingerprints records the
-- length each row was taken at (below): change this and every chromaprint is
-- re-derived, and until then only rows at the new length are compared.
chromaprint_length_sec integer NOT NULL DEFAULT 120,
-- The most disagreement two aligned fingerprints may show and still be
-- proposed as one recording. Unrelated audio sits near 0.5, so the ceiling
-- stays well clear of it.
acoustic_max_bit_error_rate double precision NOT NULL DEFAULT 0.15,
-- Files the backfill decodes at once. Decoding competes with playback
-- transcoding for CPU and with streaming for the mount.
backfill_concurrency integer NOT NULL DEFAULT 2,
-- The least time between duplicate sweeps. A sweep still runs only when
-- fingerprints have changed since the last one.
sweep_interval_hours integer NOT NULL DEFAULT 1,
-- When the settings were last saved. A new threshold or length can change
-- what a sweep finds, so a save makes a sweep due.
updated_at timestamptz NOT NULL DEFAULT now(),
CONSTRAINT fingerprint_settings_singleton CHECK (id = true),
CONSTRAINT fingerprint_settings_length_range
CHECK (chromaprint_length_sec >= 30 AND chromaprint_length_sec <= 600),
CONSTRAINT fingerprint_settings_threshold_range
CHECK (acoustic_max_bit_error_rate >= 0.01 AND acoustic_max_bit_error_rate <= 0.35),
CONSTRAINT fingerprint_settings_concurrency_range
CHECK (backfill_concurrency >= 1 AND backfill_concurrency <= 8),
CONSTRAINT fingerprint_settings_sweep_interval_range
CHECK (sweep_interval_hours >= 1 AND sweep_interval_hours <= 168)
);
INSERT INTO fingerprint_settings (id) VALUES (true) ON CONFLICT (id) DO NOTHING;
-- Every row written so far was taken at fpcalc's default length.
ALTER TABLE track_fingerprints ADD COLUMN chromaprint_length_sec integer NOT NULL DEFAULT 120;
@@ -1 +0,0 @@
ALTER TABLE network_settings DROP COLUMN IF EXISTS public_url;
@@ -1,8 +0,0 @@
-- The address users reach Minstrel at, e.g. https://music.example.com.
--
-- Password-reset links used to be built from the request's Host header,
-- which the requester controls: a forgot-password call with a forged Host
-- would email the victim a real reset token on a link to the attacker's
-- server. Links are now built only from this operator-set value, and none
-- are sent while it is empty (M462 #4981).
ALTER TABLE network_settings ADD COLUMN public_url text NOT NULL DEFAULT '';
@@ -1,7 +0,0 @@
-- The keys cannot be recovered from their hashes. Rolling back gives every
-- user a new random key; Subsonic clients using apiKey need the new one.
ALTER TABLE users ADD COLUMN api_token text;
UPDATE users SET api_token = md5(random()::text || id::text || clock_timestamp()::text);
ALTER TABLE users ALTER COLUMN api_token SET NOT NULL;
ALTER TABLE users ADD CONSTRAINT users_api_token_key UNIQUE (api_token);
ALTER TABLE users DROP COLUMN api_token_hash;
@@ -1,10 +0,0 @@
-- API keys (the OpenSubsonic apiKey) are stored as their sha256, hex, the
-- same way session tokens are. A leaked database row or backup no longer
-- hands out working keys. Existing keys are hashed in place, so every
-- Subsonic client keeps working; the key itself can no longer be shown
-- again, only replaced (M462 #4983).
ALTER TABLE users ADD COLUMN api_token_hash text;
UPDATE users SET api_token_hash = encode(sha256(convert_to(api_token, 'UTF8')), 'hex');
ALTER TABLE users ALTER COLUMN api_token_hash SET NOT NULL;
ALTER TABLE users ADD CONSTRAINT users_api_token_hash_key UNIQUE (api_token_hash);
ALTER TABLE users DROP COLUMN api_token;
-156
View File
@@ -1,156 +0,0 @@
-- name: StartDuplicateSweep :one
INSERT INTO duplicate_sweeps DEFAULT VALUES RETURNING id, started_at;
-- name: FinishDuplicateSweep :exec
UPDATE duplicate_sweeps
SET finished_at = now(),
candidates = sqlc.arg(candidates),
groups_found = sqlc.arg(groups_found),
oversize_clusters = sqlc.arg(oversize_clusters),
error_message = NULLIF(sqlc.arg(error_message)::text, '')
WHERE id = sqlc.arg(id);
-- name: GetInFlightDuplicateSweep :one
-- The guard against two sweeps at once: "in flight" is finished_at IS NULL.
SELECT id, started_at
FROM duplicate_sweeps
WHERE finished_at IS NULL
ORDER BY started_at DESC
LIMIT 1;
-- name: GetLatestDuplicateSweep :one
SELECT id, started_at, finished_at, candidates, groups_found, oversize_clusters, error_message
FROM duplicate_sweeps
ORDER BY started_at DESC
LIMIT 1;
-- name: GetLatestFingerprintComputedAt :one
-- Whether a sweep has anything new to look at: fingerprints written since the
-- last sweep started.
SELECT max(computed_at)::timestamptz AS latest FROM track_fingerprints;
-- name: ListExactDuplicateHashes :many
-- The exact tier, library-wide in one pass: identical encoded audio shared by
-- more than one present track.
SELECT f.audio_stream_sha256,
array_agg(t.id ORDER BY t.id)::uuid[] AS track_ids
FROM track_fingerprints f
JOIN tracks t ON t.id = f.track_id
WHERE t.missing_since IS NULL
AND f.fingerprint_version >= sqlc.arg(current_version)
AND f.audio_stream_sha256 IS NOT NULL
GROUP BY f.audio_stream_sha256
HAVING count(*) > 1;
-- name: ListDuplicateCandidates :many
-- The acoustic tier's input, one page at a time in (duration_ms, id) order so the
-- sweep holds only a sliding window of durations. Tracks without a chromaprint
-- cannot be compared acoustically and are left out; any exact duplicates among
-- them come from ListExactDuplicateHashes.
SELECT t.id, t.duration_ms, f.audio_stream_sha256, f.chromaprint
FROM tracks t
JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NULL
AND f.fingerprint_version >= sqlc.arg(current_version)
AND f.chromaprint IS NOT NULL
-- Only chromaprints taken at the current length: prints at two lengths are not
-- comparable, and after a length change the backfill is still re-deriving the
-- rest (#3913).
AND f.chromaprint_length_sec = sqlc.arg(chromaprint_length_sec)
AND (t.duration_ms, t.id) > (sqlc.arg(after_duration_ms)::integer, sqlc.arg(after_id)::uuid)
ORDER BY t.duration_ms, t.id
LIMIT sqlc.arg(page_limit);
-- name: ListDismissedDuplicateMemberSets :many
-- What the operator has already said are not duplicates. A new proposal whose
-- every member sat together in one of these is not proposed again.
SELECT g.id, array_agg(m.track_id ORDER BY m.track_id)::uuid[] AS track_ids
FROM duplicate_groups g
JOIN duplicate_group_members m ON m.group_id = g.id
WHERE g.status = 'dismissed'
GROUP BY g.id;
-- name: UpsertDuplicateGroup :one
-- Proposes a group, or refreshes one already pending. A group already dismissed
-- or merged is left exactly as it is: the WHERE on the update makes the conflict
-- a no-op, and the caller sees no row.
INSERT INTO duplicate_groups (member_key, tier, worst_bit_error_rate, last_seen_sweep_id)
VALUES (sqlc.arg(member_key), sqlc.arg(tier), sqlc.narg(worst_bit_error_rate), sqlc.arg(sweep_id))
ON CONFLICT (member_key) DO UPDATE
SET tier = EXCLUDED.tier,
worst_bit_error_rate = EXCLUDED.worst_bit_error_rate,
last_seen_sweep_id = EXCLUDED.last_seen_sweep_id
WHERE duplicate_groups.status = 'pending'
RETURNING id;
-- name: AddDuplicateGroupMember :exec
INSERT INTO duplicate_group_members (group_id, track_id)
VALUES (sqlc.arg(group_id), sqlc.arg(track_id))
ON CONFLICT DO NOTHING;
-- name: DeleteStalePendingDuplicateGroups :execrows
-- A pending proposal this sweep did not find again no longer describes the
-- library: a member was re-fingerprinted, merged away or went missing. Dismissed
-- groups are kept regardless — they are the memory of a decision.
--
-- Only proposals last confirmed by an EARLIER sweep go. Should two sweeps ever
-- overlap (a manual trigger racing the worker), neither may delete what the other
-- has just found.
DELETE FROM duplicate_groups g
WHERE g.status = 'pending'
AND g.last_seen_sweep_id IS DISTINCT FROM sqlc.arg(sweep_id)
AND (g.last_seen_sweep_id IS NULL
OR (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = g.last_seen_sweep_id)
< (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = sqlc.arg(sweep_id)));
-- name: CountPendingDuplicateGroups :one
-- Proposals awaiting review. A group left with one member — its other tracks
-- deleted since the sweep — is no proposal at all and is not counted; the next
-- sweep retires it.
SELECT count(*)::bigint
FROM duplicate_groups g
WHERE g.status = 'pending'
AND (SELECT count(*) FROM duplicate_group_members m WHERE m.group_id = g.id) >= 2;
-- name: ListPendingDuplicateGroupMembers :many
-- One page of proposals, newest first, flattened to one row per member so the
-- handler folds them without a query per group. What each copy carries — likes
-- and plays from every user — is here because it is what the operator weighs
-- when deciding which copy to keep.
WITH page AS (
SELECT g.id, g.tier, g.worst_bit_error_rate, g.detected_at
FROM duplicate_groups g
WHERE g.status = 'pending'
AND (SELECT count(*) FROM duplicate_group_members m WHERE m.group_id = g.id) >= 2
ORDER BY g.detected_at DESC, g.id
LIMIT sqlc.arg(page_limit) OFFSET sqlc.arg(page_offset)
)
SELECT p.id AS group_id,
p.tier,
p.worst_bit_error_rate,
p.detected_at,
t.id AS track_id,
t.title,
artists.name AS artist_name,
albums.id AS album_id,
albums.title AS album_title,
t.file_path,
t.file_format,
t.file_size,
t.duration_ms,
t.added_at,
(SELECT count(*) FROM general_likes l WHERE l.track_id = t.id)::bigint AS like_count,
(SELECT count(*) FROM play_events e WHERE e.track_id = t.id)::bigint AS play_count
FROM page p
JOIN duplicate_group_members m ON m.group_id = p.id
JOIN tracks t ON t.id = m.track_id
JOIN albums ON albums.id = t.album_id
JOIN artists ON artists.id = t.artist_id
ORDER BY p.detected_at DESC, p.id, t.id;
-- name: DismissDuplicateGroup :execrows
-- "These are not duplicates." Only a pending group can be dismissed; zero rows
-- means it was already resolved or no longer exists.
UPDATE duplicate_groups
SET status = 'dismissed', resolved_at = now()
WHERE id = sqlc.arg(id) AND status = 'pending';
@@ -1,15 +0,0 @@
-- name: GetFingerprintSettings :one
SELECT * FROM fingerprint_settings WHERE id = true;
-- name: UpdateFingerprintSettings :one
-- Whole-row write from the admin card; migration 0061's CHECKs are the backstop
-- behind the service's own validation.
UPDATE fingerprint_settings
SET enabled = sqlc.arg(enabled),
chromaprint_length_sec = sqlc.arg(chromaprint_length_sec),
acoustic_max_bit_error_rate = sqlc.arg(acoustic_max_bit_error_rate),
backfill_concurrency = sqlc.arg(backfill_concurrency),
sweep_interval_hours = sqlc.arg(sweep_interval_hours),
updated_at = now()
WHERE id = true
RETURNING *;
-70
View File
@@ -1,70 +0,0 @@
-- name: UpsertTrackFingerprint :exec
-- Written whenever a track's fingerprint is derived: by the scan when a file is
-- new or its bytes changed, and by the backfill (#3908) for rows derived by an
-- older method. Replaces the row wholesale — a fingerprint of the old bytes has
-- no standing once the file has changed.
INSERT INTO track_fingerprints (
track_id, audio_stream_sha256, chromaprint, fingerprint_version, chromaprint_length_sec
) VALUES (
sqlc.arg(track_id), sqlc.narg(audio_stream_sha256), sqlc.narg(chromaprint),
sqlc.arg(fingerprint_version), sqlc.arg(chromaprint_length_sec)
)
ON CONFLICT (track_id) DO UPDATE SET
audio_stream_sha256 = EXCLUDED.audio_stream_sha256,
chromaprint = EXCLUDED.chromaprint,
fingerprint_version = EXCLUDED.fingerprint_version,
chromaprint_length_sec = EXCLUDED.chromaprint_length_sec,
computed_at = now();
-- name: DeleteTrackFingerprint :exec
-- A file changed but could not be fingerprinted, for a reason unrelated to the
-- file. The stored row describes the OLD bytes, so it goes and the backfill
-- re-derives it — nothing may keep trusting a stale identity.
DELETE FROM track_fingerprints WHERE track_id = $1;
-- name: ListTracksNeedingFingerprint :many
-- The backfill's work queue (#3908): tracks with no fingerprint, or one derived
-- by an older method. Keyset-paged on id so a pass visits each track at most
-- once. That cursor is load-bearing: an inconclusive attempt writes no row, so
-- without it a file that keeps timing out would be listed again straight away
-- and retried in a tight loop. Missing tracks are skipped — there is no file to
-- read.
SELECT t.id, t.file_path
FROM tracks t
LEFT JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NULL
-- A row taken at another length is as stale as one from an older method:
-- chromaprints at two lengths cannot be compared (#3913).
AND (f.track_id IS NULL
OR f.fingerprint_version < sqlc.arg(current_version)
OR f.chromaprint_length_sec <> sqlc.arg(chromaprint_length_sec))
AND t.id > sqlc.arg(after_id)
ORDER BY t.id
LIMIT sqlc.arg(batch_limit);
-- name: GetFingerprintCoverage :one
-- The admin gauge for the backfill. fingerprinted + rejected + pending = total.
-- "Current" means derived by the current method AT the current length: a row at
-- another length is pending, because the backfill will re-derive it. rejected is
-- a current row with a NULL half: a tool ran and refused the file, which is
-- settled rather than waiting. Missing tracks are excluded, or the gauge could
-- never reach the end.
SELECT count(*)::bigint AS total,
count(*) FILTER (
WHERE f.fingerprint_version >= sqlc.arg(current_version)
AND f.chromaprint_length_sec = sqlc.arg(chromaprint_length_sec)
AND f.audio_stream_sha256 IS NOT NULL AND f.chromaprint IS NOT NULL
)::bigint AS fingerprinted,
count(*) FILTER (
WHERE f.fingerprint_version >= sqlc.arg(current_version)
AND f.chromaprint_length_sec = sqlc.arg(chromaprint_length_sec)
AND (f.audio_stream_sha256 IS NULL OR f.chromaprint IS NULL)
)::bigint AS rejected,
count(*) FILTER (
WHERE f.track_id IS NULL
OR f.fingerprint_version < sqlc.arg(current_version)
OR f.chromaprint_length_sec <> sqlc.arg(chromaprint_length_sec)
)::bigint AS pending
FROM tracks t
LEFT JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NULL;
-7
View File
@@ -68,10 +68,3 @@ UPDATE system_playlist_runs
DELETE FROM password_resets
WHERE (used_at IS NOT NULL AND used_at < now() - INTERVAL '7 days')
OR (used_at IS NULL AND expires_at < now() - INTERVAL '1 hour');
-- name: GcDeleteExpiredSessions :execrows
-- Sessions past their idle (30 days) or absolute (1 year) limit. They already
-- fail auth through GetSessionByTokenHash's filter; this only clears the rows.
DELETE FROM sessions
WHERE last_seen_at <= now() - interval '30 days'
OR created_at <= now() - interval '365 days';
-101
View File
@@ -1,101 +0,0 @@
-- Duplicate merge (Scribe #3911). Every statement here runs inside the one
-- transaction library.MergeDuplicateGroup opens, after the removed copy's file
-- is already gone. The loser's own track row is deleted last with DeleteTrack;
-- what these do is move everything it carries onto the survivor first, so that
-- delete's CASCADE finds nothing left to destroy.
-- name: LockDuplicateGroupForMerge :one
-- Locks the group for the rest of the transaction, so two merges of one group
-- cannot run at once.
SELECT id, tier, status
FROM duplicate_groups
WHERE id = sqlc.arg(id)
FOR UPDATE;
-- name: ListDuplicateGroupMergeMembers :many
SELECT t.id, t.file_path, t.file_format, t.file_size, t.added_at, t.album_id,
t.mbid, albums.mbid AS album_mbid
FROM duplicate_group_members m
JOIN tracks t ON t.id = m.track_id
JOIN albums ON albums.id = t.album_id
WHERE m.group_id = sqlc.arg(group_id)
ORDER BY t.id;
-- Plain repoints: no unique key involves track_id, so moving rows cannot collide.
-- name: MergeRepointPlayEvents :execrows
UPDATE play_events SET track_id = sqlc.arg(survivor_id)::uuid WHERE track_id = sqlc.arg(loser_id)::uuid;
-- name: MergeRepointSkipEvents :execrows
UPDATE skip_events SET track_id = sqlc.arg(survivor_id)::uuid WHERE track_id = sqlc.arg(loser_id)::uuid;
-- name: MergeRepointContextualLikes :execrows
UPDATE contextual_likes SET track_id = sqlc.arg(survivor_id)::uuid WHERE track_id = sqlc.arg(loser_id)::uuid;
-- name: MergeRepointPlaybackErrors :execrows
UPDATE playback_errors SET track_id = sqlc.arg(survivor_id)::uuid WHERE track_id = sqlc.arg(loser_id)::uuid;
-- name: MergeRepointLidarrRequests :execrows
UPDATE lidarr_requests SET matched_track_id = sqlc.arg(survivor_id)::uuid
WHERE matched_track_id = sqlc.arg(loser_id)::uuid;
-- name: MergeRepointPlaylistTracks :many
-- playlist_tracks is keyed by (playlist_id, position), so repointing keeps every
-- entry exactly where it was. A playlist that held both copies simply holds the
-- survivor twice — the user put two entries there, and both stay.
UPDATE playlist_tracks SET track_id = sqlc.arg(survivor_id)::uuid
WHERE track_id = sqlc.arg(loser_id)::uuid
RETURNING playlist_id;
-- Collision-safe merges: a unique key includes track_id, so the survivor may
-- already hold a matching row. Copy what it lacks; DeleteTrack's CASCADE then
-- removes the loser's originals.
-- name: MergeCopyGeneralLikes :many
-- One like per user. A user who liked both copies keeps a single like, dated to
-- the earlier of the two.
INSERT INTO general_likes (user_id, track_id, liked_at)
SELECT user_id, sqlc.arg(survivor_id)::uuid, liked_at
FROM general_likes
WHERE track_id = sqlc.arg(loser_id)::uuid
ON CONFLICT (user_id, track_id) DO UPDATE
SET liked_at = LEAST(general_likes.liked_at, EXCLUDED.liked_at)
RETURNING user_id;
-- name: MergeCopyTrackTags :execrows
INSERT INTO track_tags (track_id, tag, weight)
SELECT sqlc.arg(survivor_id)::uuid, tag, weight
FROM track_tags
WHERE track_id = sqlc.arg(loser_id)::uuid
ON CONFLICT (track_id, tag) DO NOTHING;
-- name: MergeCopyTrackSimilarity :execrows
-- Rewrites the loser to the survivor on either side of an edge. An edge between
-- the two copies would become a track similar to itself — the table forbids
-- that, and it means nothing — so it is dropped. An edge the survivor already
-- has from the same source is kept as it is.
INSERT INTO track_similarity (track_a_id, track_b_id, score, source, fetched_at)
SELECT CASE WHEN track_a_id = sqlc.arg(loser_id)::uuid THEN sqlc.arg(survivor_id)::uuid ELSE track_a_id END,
CASE WHEN track_b_id = sqlc.arg(loser_id)::uuid THEN sqlc.arg(survivor_id)::uuid ELSE track_b_id END,
score, source, fetched_at
FROM track_similarity
WHERE (track_a_id = sqlc.arg(loser_id)::uuid OR track_b_id = sqlc.arg(loser_id)::uuid)
AND (CASE WHEN track_a_id = sqlc.arg(loser_id)::uuid THEN sqlc.arg(survivor_id)::uuid ELSE track_a_id END)
<> (CASE WHEN track_b_id = sqlc.arg(loser_id)::uuid THEN sqlc.arg(survivor_id)::uuid ELSE track_b_id END)
ON CONFLICT (track_a_id, track_b_id, source) DO NOTHING;
-- name: MergeInheritTrackMbid :exec
-- A recording MBID is what the similarity pipeline keys on. If only the removed
-- copy carried one, the survivor takes it rather than going dark to similarity.
UPDATE tracks AS survivor
SET mbid = loser.mbid
FROM tracks AS loser
WHERE survivor.id = sqlc.arg(survivor_id)::uuid
AND loser.id = sqlc.arg(loser_id)::uuid
AND survivor.mbid IS NULL
AND loser.mbid IS NOT NULL;
-- name: MarkDuplicateGroupMerged :execrows
UPDATE duplicate_groups
SET status = 'merged', resolved_at = now()
WHERE id = sqlc.arg(id) AND status = 'pending';
-3
View File
@@ -3,6 +3,3 @@ SELECT * FROM network_settings WHERE id = true;
-- name: UpdateTrustedProxyHops :one
UPDATE network_settings SET trusted_proxy_hops = $1 WHERE id = true RETURNING *;
-- name: UpdatePublicURL :one
UPDATE network_settings SET public_url = $1 WHERE id = true RETURNING *;
+5 -20
View File
@@ -45,22 +45,7 @@ WHERE t.id <> $2
-- enter the pool even when the similarity/random arms miss them; scored
-- in Go via TasteMatch, so sim_score here is 0 pool-inclusion),
-- $11 coplay_artists K (#1533 — tracks by artists co-played across the
-- instance with the seed's artist; source='user_cooccurrence'),
-- $12 order_seed (text) — see below.
--
-- $12 REPLACES `ORDER BY random()` IN FOUR ARMS (#3889). Those arms returned
-- a stable set only while their LIMIT exceeded the rows eligible for them: at
-- that point they returned all of them and the order stopped mattering,
-- because the caller sorts by track id before scoring. Below that threshold
-- they returned a random SUBSET, and two builds on the same day drew
-- different ones — so "daily determinism" held by accident, and only for
-- libraries smaller than the limits.
--
-- md5(id || seed) keeps the intent — an arbitrary spread that changes when
-- the seed does — while making it reproducible for a given seed. The CALLER
-- decides what that means: system mixes pass a per-(user, day) string and get
-- the determinism they promise; radio passes a fresh value per request and
-- keeps varying, which is what a radio should do.
-- instance with the seed's artist; source='user_cooccurrence').
-- Returns same shape as LoadRadioCandidates plus similarity_score column.
WITH
@@ -102,7 +87,7 @@ similar_artists AS (
JOIN seed_artist sa ON asim.artist_a_id = sa.artist_id
WHERE asim.source = 'listenbrainz'
AND t.id NOT IN (SELECT id FROM excluded_ids)
ORDER BY asim.score DESC, md5(t.id::text || $12::text)
ORDER BY asim.score DESC, random()
LIMIT $6
),
tag_overlap AS (
@@ -130,7 +115,7 @@ likes_overlap AS (
WHERE t.id = gl.track_id
AND trim(g_overlap.g) IN (SELECT tag FROM seed_tags)
)
ORDER BY md5(gl.track_id::text || $12::text)
ORDER BY random()
LIMIT $8
),
taste_overlap AS (
@@ -157,7 +142,7 @@ coplay_artists AS (
WHERE asim.source = 'user_cooccurrence'
AND t.id NOT IN (SELECT id FROM excluded_ids)
AND t.id <> $2
ORDER BY asim.score DESC, md5(t.id::text || $12::text)
ORDER BY asim.score DESC, random()
LIMIT $11
),
random_fill AS (
@@ -173,7 +158,7 @@ random_fill AS (
UNION SELECT track_id FROM taste_overlap
UNION SELECT track_id FROM coplay_artists
)
ORDER BY md5(t.id::text || $12::text)
ORDER BY random()
LIMIT $9
)
SELECT
+2 -25
View File
@@ -7,18 +7,7 @@ VALUES ($1, $2, $3, sqlc.arg(ip), sqlc.arg(ip))
RETURNING *;
-- name: GetSessionByTokenHash :one
-- Expired sessions are invisible here, so they fail auth the moment they
-- lapse rather than whenever the GC sweep next runs. Two limits:
-- idle 30 days — a token nobody has used in a month is abandoned, and
-- matches the web cookie's lifetime;
-- absolute 1 year — even a token in daily use is re-issued yearly, so a
-- stolen one that is being used quietly does not live
-- forever.
-- Keep in step with ListSessionsForUser and GcDeleteExpiredSessions.
SELECT * FROM sessions
WHERE token_hash = $1
AND last_seen_at > now() - interval '30 days'
AND created_at > now() - interval '365 days';
SELECT * FROM sessions WHERE token_hash = $1;
-- name: TouchSessionLastSeen :exec
UPDATE sessions SET last_seen_at = now(), last_ip = $2 WHERE id = $1;
@@ -26,13 +15,7 @@ UPDATE sessions SET last_seen_at = now(), last_ip = $2 WHERE id = $1;
-- name: ListSessionsForUser :many
-- Most-recently-active first: the row a user is most likely to act on is the
-- one that moved last, and an unfamiliar entry at the top is the alarm.
-- Expired rows are left out: they no longer authenticate, so listing them
-- as active would be wrong. Same limits as GetSessionByTokenHash.
SELECT * FROM sessions
WHERE user_id = $1
AND last_seen_at > now() - interval '30 days'
AND created_at > now() - interval '365 days'
ORDER BY last_seen_at DESC;
SELECT * FROM sessions WHERE user_id = $1 ORDER BY last_seen_at DESC;
-- name: DeleteSession :exec
DELETE FROM sessions WHERE id = $1;
@@ -51,9 +34,3 @@ DELETE FROM sessions WHERE id = $1 AND user_id = $2;
-- "Log out everywhere else." Excludes the caller's own session so the action
-- doesn't log them out of the page they just used to invoke it.
DELETE FROM sessions WHERE user_id = $1 AND id <> $2;
-- name: DeleteSessionsForUser :execrows
-- Every session the user has, the caller's included. A password reset uses
-- it: whoever forgot the password is not signed in anywhere they trust, and
-- whoever may have learned it must be signed out everywhere.
DELETE FROM sessions WHERE user_id = $1;
+9 -11
View File
@@ -155,19 +155,17 @@ SELECT id, file_path FROM tracks
AND mbid = sqlc.arg(mbid)::text
LIMIT 2;
-- name: FindMissingTrackByAudioHash :many
-- Move detection fallback for files with no MBID (#2528, #3914). The audio stream
-- hash identifies the encoded audio itself, so it survives a rename, a move and a
-- retag — anything short of a re-encode. It replaced (file_size, duration_ms),
-- which could pair two unrelated files that happened to share a byte count and a
-- duration, and missed a file retagged in place, whose size changes.
-- name: FindMissingTrackByFingerprint :many
-- Move detection fallback for files with no MBID (#2528). Exact byte size AND
-- exact decoded duration is a strong pair: a plain move or rename preserves
-- both, while a re-encode changes at least one — and a re-encode genuinely is a
-- different file, so failing to match there is correct rather than a gap.
--
-- Same missing-only constraint and same LIMIT 2 rationale as the MBID variant.
SELECT t.id, t.file_path
FROM tracks t
JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NOT NULL
AND f.audio_stream_sha256 = sqlc.arg(audio_stream_sha256)
SELECT id, file_path FROM tracks
WHERE missing_since IS NOT NULL
AND file_size = sqlc.arg(file_size)
AND duration_ms = sqlc.arg(duration_ms)
LIMIT 2;
-- name: AdoptTrackPath :execrows
+9 -9
View File
@@ -1,5 +1,5 @@
-- name: CreateUser :one
INSERT INTO users (username, password_hash, api_token_hash, is_admin, display_name)
INSERT INTO users (username, password_hash, api_token, is_admin, display_name)
VALUES ($1, $2, $3, $4, $5)
RETURNING *;
@@ -17,7 +17,7 @@ RETURNING *;
-- and the second caller's INSERT fails with a unique violation. The
-- caller (registration handler) can retry as a regular non-admin in
-- that case (or surface a "username taken" error to the user).
INSERT INTO users (username, password_hash, api_token_hash, is_admin, display_name)
INSERT INTO users (username, password_hash, api_token, is_admin, display_name)
VALUES (
$1, $2, $3,
(SELECT NOT EXISTS (SELECT 1 FROM users)),
@@ -28,8 +28,8 @@ RETURNING *;
-- name: GetUserByUsername :one
SELECT * FROM users WHERE username = $1;
-- name: GetUserByAPITokenHash :one
SELECT * FROM users WHERE api_token_hash = $1;
-- name: GetUserByAPIToken :one
SELECT * FROM users WHERE api_token = $1;
-- name: CountUsers :one
SELECT count(*) FROM users;
@@ -87,7 +87,7 @@ WHERE u.id = $1;
-- Admin-driven user creation. Distinct from CreateUser/CreateUserFirstAdminRace:
-- the caller (an admin) supplies all five fields explicitly including is_admin,
-- so an admin can promote on creation. Used by POST /api/admin/users.
INSERT INTO users (username, password_hash, api_token_hash, is_admin, display_name)
INSERT INTO users (username, password_hash, api_token, is_admin, display_name)
VALUES ($1, $2, $3, $4, $5)
RETURNING *;
@@ -141,11 +141,11 @@ UPDATE users
WHERE id = $1
RETURNING *;
-- name: RegenerateApiToken :exec
-- name: RegenerateApiToken :one
-- Self-service: caller wants a new API token. Used by the /settings
-- API Token card's "Regenerate" button. Only the hash is stored; the
-- handler returns the raw key once.
UPDATE users SET api_token_hash = $2 WHERE id = $1;
-- API Token card's "Regenerate" button.
UPDATE users SET api_token = $2 WHERE id = $1
RETURNING *;
-- name: GetUserByEmail :one
-- Used by forgot-password lookup. Lowercase comparison both sides
-15
View File
@@ -87,10 +87,6 @@ var dataTables = []string{
// pristine Discover knobs rather than whatever a previous test tuned.
"discover_tuning",
"recommendation_tuning_audit",
"duplicate_group_members", // M400
"duplicate_groups",
"duplicate_sweeps",
"track_fingerprints", // M400
"tracks",
"albums",
"artists",
@@ -130,15 +126,4 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) {
); err != nil {
t.Fatalf("dbtest.ResetDB reset tag-sources version: %v", err)
}
// Fingerprinting settings (M400 #3913), a singleton like the counters above.
// Every column goes back to its migration default rather than to literals
// written here, so a test can pin the Go defaults to the migration's.
if _, err := pool.Exec(ctx, `
UPDATE fingerprint_settings
SET enabled = DEFAULT, chromaprint_length_sec = DEFAULT,
acoustic_max_bit_error_rate = DEFAULT, backfill_concurrency = DEFAULT,
sweep_interval_hours = DEFAULT, updated_at = DEFAULT`,
); err != nil {
t.Fatalf("dbtest.ResetDB reset fingerprint settings: %v", err)
}
}
-2
View File
@@ -16,7 +16,6 @@
// - GcExpireScrobbleQueueFailedRows (#567)
// - GcResetStuckSystemPlaylistRuns (#574)
// - GcDeleteExpiredPasswordResets (#575)
// - GcDeleteExpiredSessions (M462 #4978 — idle 30d / absolute 1y)
// - GcPruneDiagnostics (M9 — diagnostics 30d retention)
// - GcDeleteExpiredSuggestionSnoozes (#2374 — snoozes expire, then go)
// - GcDeleteOrphanedCandidateArtistTags(+State) (#2376 — the similarity
@@ -87,7 +86,6 @@ func (w *Worker) tickOnce(ctx context.Context) {
w.runSweep(ctx, "expire_scrobble_failed", q.GcExpireScrobbleQueueFailedRows)
w.runSweep(ctx, "reset_stuck_system_runs", q.GcResetStuckSystemPlaylistRuns)
w.runSweep(ctx, "delete_expired_password_resets", q.GcDeleteExpiredPasswordResets)
w.runSweep(ctx, "delete_expired_sessions", q.GcDeleteExpiredSessions)
w.runSweep(ctx, "prune_diagnostics", q.GcPruneDiagnostics)
w.runSweep(ctx, "delete_expired_suggestion_snoozes", q.GcDeleteExpiredSuggestionSnoozes)
// Tags before state: if the process dies between the two, a candidate left
+1 -1
View File
@@ -50,7 +50,7 @@ func seedUser(t *testing.T, pool *pgxpool.Pool, name string) pgtype.UUID {
u, err := dbq.New(pool).CreateUser(context.Background(), dbq.CreateUserParams{
Username: dbtest.TestUserPrefix + name,
PasswordHash: "test-hash",
ApiTokenHash: "test-token-" + name,
ApiToken: "test-token-" + name,
IsAdmin: false,
})
if err != nil {
+32 -161
View File
@@ -5,16 +5,12 @@ import (
"errors"
"fmt"
"io/fs"
"log/slog"
"os"
"path/filepath"
"syscall"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
)
@@ -23,179 +19,54 @@ import (
// that has no row in tracks.
var ErrTrackNotFound = errors.New("library: track not found")
// removeFile is os.Remove behind a variable so a test can make removal fail the
// way a read-only mount or a wrongly-owned directory does. A chmod-based test
// cannot stand in for that: root ignores permission bits, so in a CI container
// running as root it would pass without ever exercising the failure.
var removeFile = os.Remove
// FileRemoveError reports that a track's file exists but could not be removed.
// When DeleteTrackFile returns one, NOTHING was deleted: the row, its likes, its
// play history and its playlist memberships are all intact.
type FileRemoveError struct {
Path string
// UID and GID are the identity the server process runs as — the half of a
// permission problem the operator cannot see from the host side.
UID, GID int
Err error
}
func (e *FileRemoveError) Error() string { return fmt.Sprintf("remove track file: %v", e.Err) }
func (e *FileRemoveError) Unwrap() error { return e.Err }
// Dir is the directory removal needs write access to. Unlinking a file writes to
// its PARENT, so a world-writable file inside a read-only directory still cannot
// be removed — naming the file's own permissions would send the operator to the
// wrong place.
func (e *FileRemoveError) Dir() string { return filepath.Dir(e.Path) }
// NotWritable reports whether the library is unwritable for this process — a
// read-only mount or a permission denial — rather than an I/O fault. It is the
// case the operator can fix, so callers answer it differently.
func (e *FileRemoveError) NotWritable() bool {
return errors.Is(e.Err, fs.ErrPermission) || errors.Is(e.Err, syscall.EROFS)
}
// Reason is the underlying cause without the path os.Remove already wrapped
// around it, for messages that name the directory themselves.
func (e *FileRemoveError) Reason() string {
var pathErr *fs.PathError
if errors.As(e.Err, &pathErr) {
return pathErr.Err.Error()
}
return e.Err.Error()
}
// DeletedTrack reports what a delete tidied away beyond the track itself.
type DeletedTrack struct {
// AlbumID is set when the track was its album's last, so the album went too.
AlbumID *pgtype.UUID
// ArtistID is set when that album was its artist's last, so the artist went too.
ArtistID *pgtype.UUID
}
// DeleteTrackFile removes a track's file from disk and then its row, tidying
// away an album or artist the delete leaves empty. It is the ONLY path that
// deletes a track file: the admin remove-track endpoint and quarantine's Delete
// file both come through here (#3918).
// DeleteTrackFile removes a track file from disk and its row from the
// tracks table. Album and artist rows are left untouched.
//
// Order is the whole contract. The file goes first, and if it cannot go — a
// read-only mount, a permission denial, an I/O error — nothing else happens and
// a *FileRemoveError comes back. Proceeding past that failure is how #3918 lost
// history: tracks CASCADEs to play_events, general_likes, contextual_likes,
// playlist_tracks, track_tags and playback_errors, so the row and everything
// hanging off it were destroyed while the file survived, and the next scan
// re-imported it as a brand-new track with none of it.
// Steps:
// 1. Look up the track to get its file_path.
// 2. Remove the file from disk. fs.ErrNotExist is OK — already gone.
// 3. Delete the tracks row.
//
// A file that is already gone (fs.ErrNotExist) is not a failure; the row is
// removed as asked.
// Order matters: file first, then DB. If the file delete fails (permission,
// I/O error), we leave the DB row alone so the admin can retry.
//
// This is NOT the missing-file path. That lifecycle is deliberately
// non-destructive: reconcile stamps missing_since (#2523), selection paths
// filter on it, and a returning file is un-marked or adopted (#2528). This is the
// explicit, irreversible "remove this recording", never the way to tidy up a row
// whose file merely went away.
// The reverse failure mode — file gone, DB row still present — IS reconciled
// now, and not by this function: the scan's reconcile pass stamps
// tracks.missing_since (#2523), every selection path filters on it, and a file
// that returns is un-marked or adopted at its new path (#2528). That is the
// normal life of a vanished file and it is deliberately non-destructive: the
// row, its play history and its likes survive, because a missing file is a
// track Minstrel still knows about (#2527).
//
// dataDir, when set, also clears the cached art of an artist the delete removed.
// logger may be nil.
func DeleteTrackFile(
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, dataDir string, trackID pgtype.UUID,
) (DeletedTrack, error) {
if logger == nil {
logger = slog.Default()
}
// So this function is NOT the missing-file path. It is the explicit admin
// action "remove this recording from disk and from the library", and it is
// irreversible: tracks CASCADEs to play_events, general_likes_tracks,
// contextual_likes, track_tags and playback_errors. Reach for it when the
// operator means to destroy the record, never to tidy up a row whose file
// merely went away.
func DeleteTrackFile(ctx context.Context, pool *pgxpool.Pool, trackID pgtype.UUID) error {
q := dbq.New(pool)
track, err := q.GetTrackByID(ctx, trackID)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return DeletedTrack{}, ErrTrackNotFound
return ErrTrackNotFound
}
return DeletedTrack{}, fmt.Errorf("get track: %w", err)
return fmt.Errorf("get track: %w", err)
}
if err := removeTrackFileOnDisk(track.FilePath); err != nil {
return DeletedTrack{}, err
if err := os.Remove(track.FilePath); err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove file: %w", err)
}
// The row and any album or artist it empties go together, so a failure
// partway cannot leave a deleted track with a ghost album behind it.
tx, err := pool.Begin(ctx)
if err != nil {
return DeletedTrack{}, fmt.Errorf("begin tx: %w", err)
if _, err := pool.Exec(ctx, "DELETE FROM tracks WHERE id = $1", trackID); err != nil {
return fmt.Errorf("delete row: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
tq := dbq.New(tx)
deleted, err := tq.DeleteTrack(ctx, trackID)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
// Removed by someone else between the lookup and here.
return DeletedTrack{}, ErrTrackNotFound
}
return DeletedTrack{}, fmt.Errorf("delete track: %w", err)
}
out, err := tidyEmptiedAlbum(ctx, tq, deleted.AlbumID)
if err != nil {
return DeletedTrack{}, err
}
if err := tx.Commit(ctx); err != nil {
return DeletedTrack{}, fmt.Errorf("commit: %w", err)
}
// Both of these run after the delete has committed, so neither may fail
// it: the recording is gone either way. An unlogged change leaves the track
// in offline clients' caches until the next scan touches its album; a
// leftover art directory is only disk.
// Log the change after the delete succeeds. Best-effort: a Warn-level
// failure here would leave the cache index orphaned on offline clients
// until the next scan touches the surrounding album.
if err := syncpkg.LogChange(ctx, pool, syncpkg.EntityTrack,
syncpkg.FormatUUID(trackID), syncpkg.OpDelete); err != nil {
logger.Warn("track delete: LogChange failed", "track_id", syncpkg.FormatUUID(trackID), "err", err)
}
if out.ArtistID != nil && dataDir != "" {
if err := coverart.CleanupArtistArt(dataDir, *out.ArtistID); err != nil {
logger.Warn("track delete: artist-art cleanup failed",
"artist_id", syncpkg.FormatUUID(*out.ArtistID), "err", err)
}
}
return out, nil
}
// removeTrackFileOnDisk is the one rule for removing a track's file, shared by
// DeleteTrackFile and the duplicate merge. A file already gone is fine; anything
// else comes back as a *FileRemoveError, and the caller must then change nothing
// in the database (#3918).
func removeTrackFileOnDisk(path string) error {
if err := removeFile(path); err != nil && !errors.Is(err, fs.ErrNotExist) {
return &FileRemoveError{Path: path, UID: os.Getuid(), GID: os.Getgid(), Err: err}
return fmt.Errorf("log change: %w", err)
}
return nil
}
// tidyEmptiedAlbum removes an album a track delete left with no tracks, and its
// artist if that album was the artist's last. It runs on the caller's
// transaction, so the tidy-up commits or rolls back with the delete itself.
func tidyEmptiedAlbum(ctx context.Context, tq *dbq.Queries, albumID pgtype.UUID) (DeletedTrack, error) {
var out DeletedTrack
album, err := tq.DeleteAlbumIfEmpty(ctx, albumID)
switch {
case err == nil:
id := album.ID
out.AlbumID = &id
artistID, aerr := tq.DeleteArtistIfEmpty(ctx, album.ArtistID)
switch {
case aerr == nil:
out.ArtistID = &artistID
case errors.Is(aerr, pgx.ErrNoRows):
// The artist still has other albums or stray tracks.
default:
return DeletedTrack{}, fmt.Errorf("delete artist if empty: %w", aerr)
}
case errors.Is(err, pgx.ErrNoRows):
// The album still has other tracks.
default:
return DeletedTrack{}, fmt.Errorf("delete album if empty: %w", err)
}
return out, nil
}
+6 -116
View File
@@ -4,11 +4,9 @@ import (
"context"
"errors"
"io"
"io/fs"
"log/slog"
"os"
"path/filepath"
"syscall"
"testing"
"github.com/jackc/pgx/v5/pgtype"
@@ -66,15 +64,6 @@ func seedTrack(t *testing.T, pool *pgxpool.Pool, filePath string) (dbq.Track, db
return track, album, artist
}
// stubRemoveFile makes file removal fail (or succeed) on demand for one test.
// See removeFile for why this is a seam rather than a chmod.
func stubRemoveFile(t *testing.T, fn func(string) error) {
t.Helper()
orig := removeFile
removeFile = fn
t.Cleanup(func() { removeFile = orig })
}
func TestDeleteTrackFile_HappyPath(t *testing.T) {
pool := newPool(t)
q := dbq.New(pool)
@@ -84,18 +73,9 @@ func TestDeleteTrackFile_HappyPath(t *testing.T) {
if err := os.WriteFile(path, []byte("payload"), 0o644); err != nil {
t.Fatalf("write file: %v", err)
}
track, album, artist := seedTrack(t, pool, path)
// A sibling keeps the album non-empty, so this case pins that the tidy-up
// only removes an album the delete actually emptied.
if _, err := q.UpsertTrack(context.Background(), dbq.UpsertTrackParams{
Title: "Sibling", AlbumID: album.ID, ArtistID: artist.ID,
DurationMs: 1000, FilePath: filepath.Join(dir, "sibling.mp3"), FileSize: 100, FileFormat: "mp3",
}); err != nil {
t.Fatalf("sibling: %v", err)
}
track, album, _ := seedTrack(t, pool, path)
got, err := DeleteTrackFile(context.Background(), pool, nil, "", track.ID)
if err != nil {
if err := DeleteTrackFile(context.Background(), pool, track.ID); err != nil {
t.Fatalf("DeleteTrackFile: %v", err)
}
@@ -105,99 +85,9 @@ func TestDeleteTrackFile_HappyPath(t *testing.T) {
if _, err := q.GetTrackByID(context.Background(), track.ID); err == nil {
t.Errorf("track row still exists")
}
// Album row preserved (other tracks may reference it).
if _, err := q.GetAlbumByID(context.Background(), album.ID); err != nil {
t.Errorf("album with a remaining track vanished: %v", err)
}
if got.AlbumID != nil || got.ArtistID != nil {
t.Errorf("reported tidy-up %+v for an album that still has a track", got)
}
}
func TestDeleteTrackFile_EmptiedAlbumAndArtistGoToo(t *testing.T) {
pool := newPool(t)
q := dbq.New(pool)
path := filepath.Join(t.TempDir(), "lone.mp3")
if err := os.WriteFile(path, []byte("payload"), 0o644); err != nil {
t.Fatalf("write file: %v", err)
}
track, album, artist := seedTrack(t, pool, path)
got, err := DeleteTrackFile(context.Background(), pool, nil, "", track.ID)
if err != nil {
t.Fatalf("DeleteTrackFile: %v", err)
}
if got.AlbumID == nil || *got.AlbumID != album.ID {
t.Errorf("AlbumID = %v, want %v", got.AlbumID, album.ID)
}
if got.ArtistID == nil || *got.ArtistID != artist.ID {
t.Errorf("ArtistID = %v, want %v", got.ArtistID, artist.ID)
}
if _, err := q.GetAlbumByID(context.Background(), album.ID); err == nil {
t.Errorf("emptied album row still exists")
}
if _, err := q.GetArtistByID(context.Background(), artist.ID); err == nil {
t.Errorf("emptied artist row still exists")
}
}
// The #3918 proof. A file that cannot be removed must leave EVERYTHING in place:
// the row is what carries likes, plays and playlist memberships, and the file
// surviving means the next scan would re-import it as a stranger.
func TestDeleteTrackFile_UnremovableFileDeletesNothing(t *testing.T) {
cases := []struct {
name string
errno syscall.Errno
notWritable bool
}{
{"read-only mount", syscall.EROFS, true},
{"permission denied", syscall.EACCES, true},
{"operation not permitted", syscall.EPERM, true},
{"i/o error", syscall.EIO, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
pool := newPool(t)
q := dbq.New(pool)
dir := t.TempDir()
path := filepath.Join(dir, "track.mp3")
if err := os.WriteFile(path, []byte("payload"), 0o644); err != nil {
t.Fatalf("write file: %v", err)
}
track, album, _ := seedTrack(t, pool, path)
stubRemoveFile(t, func(name string) error {
return &fs.PathError{Op: "remove", Path: name, Err: tc.errno}
})
_, err := DeleteTrackFile(context.Background(), pool, nil, "", track.ID)
var fre *FileRemoveError
if !errors.As(err, &fre) {
t.Fatalf("err = %v, want a *FileRemoveError", err)
}
if fre.NotWritable() != tc.notWritable {
t.Errorf("NotWritable = %v, want %v", fre.NotWritable(), tc.notWritable)
}
if fre.Dir() != dir {
t.Errorf("Dir = %q, want the parent directory %q", fre.Dir(), dir)
}
if fre.Reason() != tc.errno.Error() {
t.Errorf("Reason = %q, want %q", fre.Reason(), tc.errno.Error())
}
if fre.UID != os.Getuid() || fre.GID != os.Getgid() {
t.Errorf("identity = %d:%d, want this process's %d:%d", fre.UID, fre.GID, os.Getuid(), os.Getgid())
}
if _, err := q.GetTrackByID(context.Background(), track.ID); err != nil {
t.Errorf("track row was deleted although its file was not: %v", err)
}
if _, err := q.GetAlbumByID(context.Background(), album.ID); err != nil {
t.Errorf("album row was deleted although the track's file was not: %v", err)
}
if _, err := os.Stat(path); err != nil {
t.Errorf("file gone although removal was refused: %v", err)
}
})
t.Errorf("album row vanished: %v", err)
}
}
@@ -207,7 +97,7 @@ func TestDeleteTrackFile_FileAlreadyGoneSucceeds(t *testing.T) {
track, _, _ := seedTrack(t, pool, "/no/such/file/anywhere.mp3")
if _, err := DeleteTrackFile(context.Background(), pool, nil, "", track.ID); err != nil {
if err := DeleteTrackFile(context.Background(), pool, track.ID); err != nil {
t.Fatalf("DeleteTrackFile with missing file: %v", err)
}
if _, err := q.GetTrackByID(context.Background(), track.ID); err == nil {
@@ -222,7 +112,7 @@ func TestDeleteTrackFile_NotFoundReturnsErr(t *testing.T) {
bogus.Bytes = [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
bogus.Valid = true
_, err := DeleteTrackFile(context.Background(), pool, nil, "", bogus)
err := DeleteTrackFile(context.Background(), pool, bogus)
if !errors.Is(err, ErrTrackNotFound) {
t.Errorf("err = %v, want ErrTrackNotFound", err)
}
-430
View File
@@ -1,430 +0,0 @@
package library
import (
"math"
"math/bits"
"sort"
)
// Duplicate matching (M400 #3909).
//
// Pure functions over fingerprints: no database, no files. This is the part that
// decides whether two tracks in the operator's library are proposed as one
// recording, so every rule in it has to be falsifiable in a unit test.
//
// Two tiers, answering different questions:
//
// exact equal audio_stream_sha256 — the same encoded audio bytes. No score,
// no threshold, no false positives (the #3885 pair).
// acoustic chromaprint fingerprints that agree closely once aligned — the
// same recording at another bitrate or in another codec.
//
// The acoustic comparison follows the approach of AcoustID's pg_acoustid
// (acoustid_compare.c): vote on the relative offset between two fingerprints
// using items that agree in their high bits, then measure disagreement at the
// winning offset. Reimplemented from that description; no code was copied.
// The alignment window and match-bit width below are taken from it.
// maxAlignOffsetItems bounds how far apart two fingerprints may be shifted and
// still be compared: ±120 items, about 15 seconds at chromaprint's ~8 items per
// second. Covers a leading silence trimmed differently or a short intro; the
// same bound pg_acoustid uses (ACOUSTID_MAX_ALIGN_OFFSET).
const maxAlignOffsetItems = 120
// alignMatchBits is how many high bits two items must share to vote for an
// offset. Matching whole 32-bit items would miss the same recording at another
// bitrate, whose low bits are noisier; 14 is pg_acoustid's MATCH_BITS.
const alignMatchBits = 14
// minOverlapItems is the least overlap worth a verdict: about 10 seconds. A few
// items agreeing perfectly is not evidence that two recordings are one.
const minOverlapItems = 80
// minDistinctFraction rejects low-information fingerprints before they can
// match. Near-silence, a sustained tone or a click track produces the same few
// items over and over, and two such tracks agree closely without being the
// same recording. Real music is overwhelmingly distinct item to item, so this
// floor only catches the pathological case. A judgment value, not a measured
// one — revisit if the sweep reports real tracks refused for it.
const minDistinctFraction = 0.3
// defaultAcousticMaxBitErrorRate is the most disagreement two aligned
// fingerprints may show and still be proposed as one recording. Unrelated audio
// sits near 0.5; the same recording re-encoded lands well under 0.1.
//
// Deliberately conservative. The operator's stated worry is the opposite of a
// missed duplicate: "the same song can appear in different albums, usually it's
// a different recording", and an instrumental shares its vocal version's
// harmony, which chroma features capture. A false merge is the failure that
// matters, and the report is reviewed anyway. This is an unmeasured default:
// calibrate it against real pairs once the backfill (#3908) has populated the
// library, then expose it in Settings (#3913).
const defaultAcousticMaxBitErrorRate = 0.15
// durationToleranceMs is how far apart two tracks' durations may be and still be
// compared. Encoders pad and trim a little; different edits differ by more.
const durationToleranceMs = 3000
// maxAcousticGroupSize caps an acoustic group. A cluster bigger than this is far
// more likely a shared jingle, a skit or a low-information pattern than eight
// copies of one recording, and proposing it would bury the real duplicates.
// Exact-tier groups are not capped: identical bytes are identical however many.
const maxAcousticGroupSize = 8
// acousticScore is the result of comparing two fingerprints.
type acousticScore struct {
// Offset is how many items b is shifted against a: b[i+Offset] aligns with
// a[i].
Offset int
// Overlap is how many aligned items were compared.
Overlap int
// BitErrorRate is the fraction of differing bits over the overlap, 0..1.
BitErrorRate float64
}
// preparedPrint is a fingerprint with the parts every comparison needs worked
// out once. The sweep compares each track with every other track within a few
// seconds of its duration, so rebuilding the alignment index for each pair would
// dominate its cost.
type preparedPrint struct {
items []int32
index map[uint32][]int
informative bool
}
// preparePrint indexes a fingerprint's items by their high bits and records
// whether it varies enough to be compared at all.
func preparePrint(fp []int32) *preparedPrint {
// Each bucket keeps only a few positions: a value repeating many times is
// uninformative, and letting it vote once per repeat would make every
// pairing O(n²).
const keepPerBucket = 4
p := &preparedPrint{items: fp, index: make(map[uint32][]int, len(fp))}
seen := make(map[int32]struct{}, len(fp))
for i, v := range fp {
seen[v] = struct{}{}
key := alignKey(v)
if pos := p.index[key]; len(pos) < keepPerBucket {
p.index[key] = append(pos, i)
}
}
p.informative = len(fp) > 0 && float64(len(seen)) >= minDistinctFraction*float64(len(fp))
return p
}
func alignKey(v int32) uint32 { return uint32(v) >> (32 - alignMatchBits) }
// compareChromaprint aligns two raw fingerprints and measures how much they
// disagree. ok is false when no verdict is possible: no offset gathered any
// votes, the overlap at the best offset is too short, or either side carries
// too little information to mean anything.
func compareChromaprint(a, b []int32) (acousticScore, bool) {
return comparePrepared(preparePrint(a), preparePrint(b))
}
// comparePrepared is compareChromaprint over fingerprints already prepared.
func comparePrepared(a, b *preparedPrint) (acousticScore, bool) {
if len(a.items) < minOverlapItems || len(b.items) < minOverlapItems {
return acousticScore{}, false
}
if !a.informative || !b.informative {
return acousticScore{}, false
}
offset, ok := bestOffset(a, b)
if !ok {
return acousticScore{}, false
}
// a[i] aligns with b[i+offset]; walk the indices valid on both sides.
start := max(0, -offset)
end := min(len(a.items), len(b.items)-offset)
overlap := end - start
if overlap < minOverlapItems {
return acousticScore{}, false
}
errBits := 0
for i := start; i < end; i++ {
errBits += bits.OnesCount32(uint32(a.items[i]) ^ uint32(b.items[i+offset]))
}
return acousticScore{
Offset: offset,
Overlap: overlap,
BitErrorRate: float64(errBits) / float64(32*overlap),
}, true
}
// bestOffset returns the relative shift most items agree on.
func bestOffset(a, b *preparedPrint) (int, bool) {
votes := make([]int, 2*maxAlignOffsetItems+1)
for j, v := range b.items {
for _, i := range a.index[alignKey(v)] {
off := j - i
if off >= -maxAlignOffsetItems && off <= maxAlignOffsetItems {
votes[off+maxAlignOffsetItems]++
}
}
}
best, bestVotes := 0, 0
for k, n := range votes {
// Strictly greater keeps the smallest shift on a tie, which is the more
// likely truth and keeps the result deterministic.
if n > bestVotes || (n == bestVotes && n > 0 && abs(k-maxAlignOffsetItems) < abs(best)) {
best, bestVotes = k-maxAlignOffsetItems, n
}
}
return best, bestVotes > 0
}
// fingerprintCandidate is one track as the grouping sees it.
type fingerprintCandidate struct {
ID string
DurationMs int32
StreamSHA256 []byte
Chromaprint []int32
}
// duplicateTier names what a group's evidence is.
type duplicateTier string
const (
tierExact duplicateTier = "exact"
tierAcoustic duplicateTier = "acoustic"
)
// duplicateGroup is a set of tracks proposed as one recording. Members are
// sorted by ID.
type duplicateGroup struct {
Tier duplicateTier
Members []string
// WorstBitErrorRate is the largest disagreement between any two members of
// an acoustic group — the weakest evidence the group rests on. Zero for
// exact groups.
WorstBitErrorRate float64
}
// groupingResult is what one grouping pass found.
type groupingResult struct {
Groups []duplicateGroup
// OversizeClusters counts acoustic clusters discarded for exceeding
// maxAcousticGroupSize. Reported rather than silent: a sudden rise means the
// cap or the information floor needs attention.
OversizeClusters int
}
// groupUnit is one thing the acoustic pass compares: a single track, or an exact
// group standing in for all its byte-identical copies.
type groupUnit struct {
ids []string // every member, sorted
durationMs int32
sortKey string // the representative's id: ties on duration break on it
print *preparedPrint
exact bool // more than one member with identical audio
assigned bool
}
// streamGrouper is the acoustic pass over units arriving in (durationMs, sortKey)
// order. It holds only the units within durationToleranceMs of the oldest one
// not yet settled, so memory is bounded by the densest few seconds of the
// library rather than by its size — the whole library's fingerprints would be
// hundreds of megabytes.
//
// A seed can be settled as soon as a unit arrives beyond its window: everything
// it could group with has already arrived, and no later seed can reach back to
// it because seeds only look forward. That is what makes the streamed result
// identical to running the same pass over the whole sorted list.
//
// Grouping is COMPLETE-LINKAGE: a unit joins a group only if it matches every
// unit already in it, within the duration tolerance and the bit-error limit.
// Single-linkage would let a chain of near-misses — A close to B, B close to C —
// drag A and C, which are not close, into one proposed merge. Complete linkage
// also means any member can be chosen as the survivor (#3911).
//
// When an acoustic group absorbs an exact group, the result is tier acoustic: a
// group is only as certain as its weakest link.
type streamGrouper struct {
maxBitErrorRate float64
window []*groupUnit
res groupingResult
}
func newStreamGrouper(maxBitErrorRate float64) *streamGrouper {
return &streamGrouper{maxBitErrorRate: maxBitErrorRate}
}
// push adds the next unit. Units must arrive in non-decreasing
// (durationMs, sortKey) order.
func (g *streamGrouper) push(u *groupUnit) {
g.window = append(g.window, u)
for len(g.window) > 1 && u.durationMs-g.window[0].durationMs > durationToleranceMs {
g.settleOldest()
}
}
// finish settles every unit still waiting and returns what was found. Groups
// are in no particular order; callers sort with sortGroups.
func (g *streamGrouper) finish() groupingResult {
for len(g.window) > 0 {
g.settleOldest()
}
return g.res
}
func (g *streamGrouper) settleOldest() {
seed := g.window[0]
g.window[0] = nil // release it: the window's backing array outlives the slide
g.window = g.window[1:]
if seed.assigned {
return
}
group := []*groupUnit{seed}
worst := 0.0
for _, cand := range g.window {
if cand.durationMs-seed.durationMs > durationToleranceMs {
break
}
if cand.assigned {
continue
}
joined, worstWithCand := true, worst
for _, member := range group {
if abs32(cand.durationMs-member.durationMs) > durationToleranceMs {
joined = false
break
}
score, ok := comparePrepared(member.print, cand.print)
if !ok || score.BitErrorRate > g.maxBitErrorRate {
joined = false
break
}
worstWithCand = math.Max(worstWithCand, score.BitErrorRate)
}
if joined {
group = append(group, cand)
worst = worstWithCand
}
}
if len(group) == 1 {
if seed.exact {
g.res.Groups = append(g.res.Groups, duplicateGroup{Tier: tierExact, Members: seed.ids})
}
return
}
// Count units, not tracks: an absorbed exact group is one piece of acoustic
// evidence however many identical files it holds.
if len(group) > maxAcousticGroupSize {
g.res.OversizeClusters++
for _, member := range group {
member.assigned = true
// The acoustic evidence is untrustworthy; identical bytes are not.
// An exact group caught inside an oversize cluster is still proposed.
if member.exact {
g.res.Groups = append(g.res.Groups, duplicateGroup{Tier: tierExact, Members: member.ids})
}
}
return
}
var members []string
for _, member := range group {
member.assigned = true
members = append(members, member.ids...)
}
sort.Strings(members)
g.res.Groups = append(g.res.Groups, duplicateGroup{
Tier: tierAcoustic, Members: members, WorstBitErrorRate: worst,
})
}
// groupDuplicates proposes duplicate groups among candidates held in memory. It
// runs the same streamGrouper the sweep uses, so there is one grouping rule.
//
// Exact groups come first: tracks sharing an audio stream hash. Each becomes a
// single unit for the acoustic pass, represented by its member with the lowest
// (duration, id) that has a chromaprint. That is the member the sweep's
// duration-ordered stream meets first, which keeps the two identical. An exact
// group with no chromaprint at all cannot be compared acoustically and stands
// on its own.
//
// The output does not depend on input order.
func groupDuplicates(cands []fingerprintCandidate, maxBitErrorRate float64) groupingResult {
byHash := map[string][]fingerprintCandidate{}
var units []*groupUnit
var printless []duplicateGroup
for _, c := range cands {
if len(c.StreamSHA256) > 0 {
byHash[string(c.StreamSHA256)] = append(byHash[string(c.StreamSHA256)], c)
continue
}
if len(c.Chromaprint) > 0 {
units = append(units, &groupUnit{
ids: []string{c.ID}, durationMs: c.DurationMs, sortKey: c.ID, print: preparePrint(c.Chromaprint),
})
}
}
for _, group := range byHash {
ids := make([]string, len(group))
for i, m := range group {
ids[i] = m.ID
}
sort.Strings(ids)
var rep *fingerprintCandidate
for i := range group {
m := &group[i]
if len(m.Chromaprint) == 0 {
continue
}
if rep == nil || m.DurationMs < rep.DurationMs || (m.DurationMs == rep.DurationMs && m.ID < rep.ID) {
rep = m
}
}
if rep == nil {
if len(group) > 1 {
printless = append(printless, duplicateGroup{Tier: tierExact, Members: ids})
}
continue
}
units = append(units, &groupUnit{
ids: ids, durationMs: rep.DurationMs, sortKey: rep.ID,
print: preparePrint(rep.Chromaprint), exact: len(group) > 1,
})
}
sort.Slice(units, func(i, j int) bool {
if units[i].durationMs != units[j].durationMs {
return units[i].durationMs < units[j].durationMs
}
return units[i].sortKey < units[j].sortKey
})
g := newStreamGrouper(maxBitErrorRate)
for _, u := range units {
g.push(u)
}
res := g.finish()
res.Groups = append(res.Groups, printless...)
sortGroups(res.Groups)
return res
}
// sortGroups orders groups by their first member. Groups are disjoint, so that
// is a total order.
func sortGroups(groups []duplicateGroup) {
sort.Slice(groups, func(i, j int) bool { return groups[i].Members[0] < groups[j].Members[0] })
}
func abs(n int) int {
if n < 0 {
return -n
}
return n
}
func abs32(n int32) int32 {
if n < 0 {
return -n
}
return n
}
-241
View File
@@ -1,241 +0,0 @@
package library
import (
"math"
"math/rand/v2"
"reflect"
"testing"
)
// printLen is a realistic fingerprint length: fpcalc's 120s at ~8 items/second.
const printLen = 960
// randomPrint is a deterministic stand-in for one recording's fingerprint.
func randomPrint(seed uint64, n int) []int32 {
r := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))
fp := make([]int32, n)
for i := range fp {
fp[i] = int32(r.Uint32())
}
return fp
}
// withBitNoise flips exactly round(fraction × all bits) distinct bits — the
// same recording through a different encoder, at a known bit-error rate.
func withBitNoise(fp []int32, fraction float64, seed uint64) []int32 {
out := append([]int32(nil), fp...)
r := rand.New(rand.NewPCG(seed, seed^0x243f6a8885a308d3))
total := 32 * len(fp)
for _, pos := range r.Perm(total)[:int(math.Round(fraction*float64(total)))] {
out[pos/32] ^= int32(uint32(1) << (pos % 32))
}
return out
}
func constantPrint(v int32, n int) []int32 {
fp := make([]int32, n)
for i := range fp {
fp[i] = v
}
return fp
}
func TestCompareChromaprint(t *testing.T) {
base := randomPrint(1, printLen)
t.Run("identical", func(t *testing.T) {
got, ok := compareChromaprint(base, base)
if !ok || got.BitErrorRate != 0 || got.Offset != 0 || got.Overlap != printLen {
t.Fatalf("got %+v ok=%v, want an exact alignment", got, ok)
}
})
t.Run("re-encoded: known bit noise is measured exactly", func(t *testing.T) {
got, ok := compareChromaprint(base, withBitNoise(base, 0.03, 2))
if !ok {
t.Fatal("a re-encode was not comparable")
}
if want := math.Round(0.03*32*printLen) / (32 * printLen); got.BitErrorRate != want {
t.Fatalf("BitErrorRate = %v, want %v", got.BitErrorRate, want)
}
})
// b starts 40 items later in the same audio: b[j] = a[j+40], so a[i] aligns
// with b[i-40].
t.Run("offset inside the window is recovered", func(t *testing.T) {
got, ok := compareChromaprint(base, base[40:])
if !ok || got.Offset != -40 || got.BitErrorRate != 0 || got.Overlap != printLen-40 {
t.Fatalf("got %+v ok=%v, want offset -40 with no error", got, ok)
}
})
t.Run("offset beyond the window never matches", func(t *testing.T) {
got, ok := compareChromaprint(base, base[200:])
if ok && got.BitErrorRate <= defaultAcousticMaxBitErrorRate {
t.Fatalf("a 200-item shift matched: %+v", got)
}
})
t.Run("unrelated recordings sit near 0.5", func(t *testing.T) {
got, ok := compareChromaprint(base, randomPrint(99, printLen))
if ok && got.BitErrorRate < 0.4 {
t.Fatalf("unrelated fingerprints scored %v", got.BitErrorRate)
}
})
t.Run("too short an overlap gives no verdict", func(t *testing.T) {
if got, ok := compareChromaprint(base, base[:minOverlapItems-1]); ok {
t.Fatalf("a %d-item fingerprint was compared: %+v", minOverlapItems-1, got)
}
})
// Two near-silent tracks agree perfectly without being one recording. The
// information floor is the only thing standing between them and a merge.
t.Run("low-information fingerprints give no verdict", func(t *testing.T) {
silence := constantPrint(0x1234, printLen)
if got, ok := compareChromaprint(silence, silence); ok {
t.Fatalf("silence compared as a match: %+v", got)
}
})
t.Run("the threshold separates close from not close", func(t *testing.T) {
near, _ := compareChromaprint(base, withBitNoise(base, 0.10, 3))
far, _ := compareChromaprint(base, withBitNoise(base, 0.20, 4))
if near.BitErrorRate > defaultAcousticMaxBitErrorRate {
t.Errorf("10%% noise (%v) is over the threshold", near.BitErrorRate)
}
if far.BitErrorRate <= defaultAcousticMaxBitErrorRate {
t.Errorf("20%% noise (%v) is under the threshold", far.BitErrorRate)
}
})
}
func TestGroupDuplicates_ExactTier(t *testing.T) {
hash := []byte("sha256-of-www-instrumental-bytes")
res := groupDuplicates([]fingerprintCandidate{
{ID: "www-01", DurationMs: 215000, StreamSHA256: hash},
{ID: "www-02", DurationMs: 215000, StreamSHA256: hash},
{ID: "lovesick", DurationMs: 198000, StreamSHA256: []byte("another")},
}, defaultAcousticMaxBitErrorRate)
want := []duplicateGroup{{Tier: tierExact, Members: []string{"www-01", "www-02"}}}
if !reflect.DeepEqual(res.Groups, want) {
t.Fatalf("groups = %+v, want %+v", res.Groups, want)
}
}
func TestGroupDuplicates_AcousticPair(t *testing.T) {
p := randomPrint(10, printLen)
res := groupDuplicates([]fingerprintCandidate{
{ID: "album", DurationMs: 240000, Chromaprint: p},
{ID: "compilation", DurationMs: 241000, Chromaprint: withBitNoise(p, 0.05, 11)},
}, defaultAcousticMaxBitErrorRate)
if len(res.Groups) != 1 || res.Groups[0].Tier != tierAcoustic ||
!reflect.DeepEqual(res.Groups[0].Members, []string{"album", "compilation"}) {
t.Fatalf("groups = %+v, want one acoustic pair", res.Groups)
}
if got := res.Groups[0].WorstBitErrorRate; math.Abs(got-0.05) > 0.001 {
t.Fatalf("WorstBitErrorRate = %v, want about 0.05", got)
}
}
// A is close to B and B is close to C, but A and C are not close. Under
// single linkage all three would be proposed as one recording; complete linkage
// must keep C out.
func TestGroupDuplicates_NoChaining(t *testing.T) {
a := randomPrint(20, printLen)
b := withBitNoise(a, 0.10, 21)
c := withBitNoise(b, 0.10, 22)
if s, _ := compareChromaprint(a, c); s.BitErrorRate <= defaultAcousticMaxBitErrorRate {
t.Fatalf("fixture broken: A and C are close (%v), so this cannot test chaining", s.BitErrorRate)
}
res := groupDuplicates([]fingerprintCandidate{
{ID: "a", DurationMs: 200000, Chromaprint: a},
{ID: "b", DurationMs: 200000, Chromaprint: b},
{ID: "c", DurationMs: 200000, Chromaprint: c},
}, defaultAcousticMaxBitErrorRate)
if len(res.Groups) != 1 || !reflect.DeepEqual(res.Groups[0].Members, []string{"a", "b"}) {
t.Fatalf("groups = %+v, want only {a, b}", res.Groups)
}
}
func TestGroupDuplicates_DurationTolerance(t *testing.T) {
p := randomPrint(30, printLen)
res := groupDuplicates([]fingerprintCandidate{
{ID: "edit", DurationMs: 200000, Chromaprint: p},
{ID: "extended", DurationMs: 200000 + durationToleranceMs + 1, Chromaprint: p},
}, defaultAcousticMaxBitErrorRate)
if len(res.Groups) != 0 {
t.Fatalf("tracks %dms apart were grouped: %+v", durationToleranceMs+1, res.Groups)
}
}
// Nine tracks that all match are far likelier a shared jingle than nine copies
// of one recording. The cluster must be reported, not proposed.
func TestGroupDuplicates_OversizeClusterIsDiscarded(t *testing.T) {
p := randomPrint(40, printLen)
var cands []fingerprintCandidate
for i := range maxAcousticGroupSize + 1 {
cands = append(cands, fingerprintCandidate{
ID: string(rune('a' + i)), DurationMs: 30000, Chromaprint: withBitNoise(p, 0.01, uint64(100+i)),
})
}
res := groupDuplicates(cands, defaultAcousticMaxBitErrorRate)
if len(res.Groups) != 0 || res.OversizeClusters != 1 {
t.Fatalf("groups = %+v, oversize = %d; want none proposed and 1 oversize", res.Groups, res.OversizeClusters)
}
}
// Two byte-identical copies plus a re-encode of the same recording are one
// group, and it is only as certain as its weakest link.
func TestGroupDuplicates_ExactGroupAbsorbedIntoAcoustic(t *testing.T) {
p := randomPrint(50, printLen)
hash := []byte("same-bytes")
res := groupDuplicates([]fingerprintCandidate{
{ID: "x1", DurationMs: 180000, StreamSHA256: hash, Chromaprint: p},
{ID: "x2", DurationMs: 180000, StreamSHA256: hash, Chromaprint: p},
{ID: "y", DurationMs: 180000, StreamSHA256: []byte("other-bytes"), Chromaprint: withBitNoise(p, 0.03, 51)},
}, defaultAcousticMaxBitErrorRate)
want := []string{"x1", "x2", "y"}
if len(res.Groups) != 1 || res.Groups[0].Tier != tierAcoustic || !reflect.DeepEqual(res.Groups[0].Members, want) {
t.Fatalf("groups = %+v, want one acoustic group %v", res.Groups, want)
}
}
func TestGroupDuplicates_UnrelatedTracksNeverGroup(t *testing.T) {
var cands []fingerprintCandidate
for i := range 6 {
cands = append(cands, fingerprintCandidate{
ID: string(rune('a' + i)), DurationMs: 210000, Chromaprint: randomPrint(uint64(60+i), printLen),
})
}
if res := groupDuplicates(cands, defaultAcousticMaxBitErrorRate); len(res.Groups) != 0 {
t.Fatalf("unrelated recordings were grouped: %+v", res.Groups)
}
}
func TestGroupDuplicates_OrderIndependent(t *testing.T) {
p := randomPrint(70, printLen)
q := randomPrint(71, printLen)
hash := []byte("identical")
cands := []fingerprintCandidate{
{ID: "p1", DurationMs: 200000, Chromaprint: p},
{ID: "p2", DurationMs: 201000, Chromaprint: withBitNoise(p, 0.04, 72)},
{ID: "q1", DurationMs: 150000, Chromaprint: q},
{ID: "q2", DurationMs: 150500, Chromaprint: withBitNoise(q, 0.02, 73)},
{ID: "h1", DurationMs: 90000, StreamSHA256: hash},
{ID: "h2", DurationMs: 90000, StreamSHA256: hash},
{ID: "lone", DurationMs: 200000, Chromaprint: randomPrint(74, printLen)},
}
want := groupDuplicates(cands, defaultAcousticMaxBitErrorRate)
if len(want.Groups) != 3 {
t.Fatalf("fixture broken: %d groups, want 3 (p, q, h)", len(want.Groups))
}
r := rand.New(rand.NewPCG(75, 76))
for range 20 {
shuffled := append([]fingerprintCandidate(nil), cands...)
r.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] })
if got := groupDuplicates(shuffled, defaultAcousticMaxBitErrorRate); !reflect.DeepEqual(got, want) {
t.Fatalf("input order changed the result:\n got %+v\n want %+v", got, want)
}
}
}
-309
View File
@@ -1,309 +0,0 @@
package library
import (
"context"
"errors"
"fmt"
"log/slog"
"sort"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
)
// Duplicate merge (M400 #3911).
// ErrDuplicateGroupNotPending means the group was already merged or dismissed,
// no longer exists, or no longer has two members to merge.
var ErrDuplicateGroupNotPending = errors.New("library: duplicate group is not pending")
// ErrSurvivorNotInGroup means the copy chosen to keep is not a member of the group.
var ErrSurvivorNotInGroup = errors.New("library: survivor is not a member of the group")
// MergedCopy is one copy a merge kept or removed.
type MergedCopy struct {
TrackID pgtype.UUID
FilePath string
TrackMbid *string
AlbumMbid *string
}
// MergeResult says what a merge did.
type MergeResult struct {
Tier string
Survivor MergedCopy
Removed []MergedCopy
// What moved onto the survivor — reported so the operator, and the audit
// log, can see that the history was kept rather than take it on trust.
PlayEvents int64
SkipEvents int64
Likes int // users whose like now sits on the survivor
PlaylistEntries int
DeletedAlbumIDs []pgtype.UUID
DeletedArtistIDs []pgtype.UUID
}
// MergeDuplicateGroup keeps one copy of a duplicate group and removes the rest,
// carrying everything the removed copies held onto the one kept.
//
// survivorID chooses the copy to keep; an invalid (zero) id takes the proposal
// from ProposeSurvivor.
//
// The danger this is built around: every table referencing tracks does so ON
// DELETE CASCADE, so deleting a duplicate's row outright silently destroys its
// likes, plays, playlist entries and tags. The merge moves all of that onto the
// survivor first, and only then deletes the now-empty row.
//
// It deletes the removed copies' FILES too, and first, before any row changes
// (#3918, note #3926). A merge that left the file behind would be undone by the
// next scan, which re-imports it as a new track with no history. If a file cannot
// be removed, the *FileRemoveError comes back and nothing in the database changes.
// With several copies to remove, one file may already be gone when a later one
// fails; that copy's row keeps all its history and is marked missing by the next
// scan, and retrying the merge picks up where it stopped.
//
// Everything else happens in one transaction, which holds a lock on the group so
// two merges of it cannot run at once. Sync changes for clients' caches are logged
// inside it, the way the playlists service logs its own.
func MergeDuplicateGroup(
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, dataDir string,
groupID, survivorID pgtype.UUID,
) (MergeResult, error) {
if logger == nil {
logger = slog.Default()
}
tx, err := pool.Begin(ctx)
if err != nil {
return MergeResult{}, fmt.Errorf("begin merge: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
tq := dbq.New(tx)
group, err := tq.LockDuplicateGroupForMerge(ctx, groupID)
if errors.Is(err, pgx.ErrNoRows) {
return MergeResult{}, ErrDuplicateGroupNotPending
}
if err != nil {
return MergeResult{}, fmt.Errorf("lock duplicate group: %w", err)
}
if group.Status != "pending" {
return MergeResult{}, ErrDuplicateGroupNotPending
}
members, err := tq.ListDuplicateGroupMergeMembers(ctx, groupID)
if err != nil {
return MergeResult{}, fmt.Errorf("list group members: %w", err)
}
if len(members) < 2 {
return MergeResult{}, ErrDuplicateGroupNotPending
}
survivor, losers, err := splitSurvivor(members, survivorID)
if err != nil {
return MergeResult{}, err
}
for _, l := range losers {
if err := removeTrackFileOnDisk(l.FilePath); err != nil {
return MergeResult{}, err
}
}
res := MergeResult{Tier: group.Tier, Survivor: mergedCopyOf(survivor)}
likers := map[string]struct{}{}
changes := mergeChanges{}
survivorKey := syncpkg.FormatUUID(survivor.ID)
for _, l := range losers {
ids := struct{ survivor, loser pgtype.UUID }{survivor.ID, l.ID}
loserKey := syncpkg.FormatUUID(l.ID)
n, err := tq.MergeRepointPlayEvents(ctx, dbq.MergeRepointPlayEventsParams{SurvivorID: ids.survivor, LoserID: ids.loser})
if err != nil {
return MergeResult{}, fmt.Errorf("move play events: %w", err)
}
res.PlayEvents += n
n, err = tq.MergeRepointSkipEvents(ctx, dbq.MergeRepointSkipEventsParams{SurvivorID: ids.survivor, LoserID: ids.loser})
if err != nil {
return MergeResult{}, fmt.Errorf("move skip events: %w", err)
}
res.SkipEvents += n
if _, err := tq.MergeRepointContextualLikes(ctx, dbq.MergeRepointContextualLikesParams{SurvivorID: ids.survivor, LoserID: ids.loser}); err != nil {
return MergeResult{}, fmt.Errorf("move contextual likes: %w", err)
}
if _, err := tq.MergeRepointPlaybackErrors(ctx, dbq.MergeRepointPlaybackErrorsParams{SurvivorID: ids.survivor, LoserID: ids.loser}); err != nil {
return MergeResult{}, fmt.Errorf("move playback errors: %w", err)
}
if _, err := tq.MergeRepointLidarrRequests(ctx, dbq.MergeRepointLidarrRequestsParams{SurvivorID: ids.survivor, LoserID: ids.loser}); err != nil {
return MergeResult{}, fmt.Errorf("move lidarr request matches: %w", err)
}
playlists, err := tq.MergeRepointPlaylistTracks(ctx, dbq.MergeRepointPlaylistTracksParams{SurvivorID: ids.survivor, LoserID: ids.loser})
if err != nil {
return MergeResult{}, fmt.Errorf("move playlist entries: %w", err)
}
res.PlaylistEntries += len(playlists)
for _, pl := range playlists {
plKey := syncpkg.FormatUUID(pl)
changes.playlistDelete(syncpkg.EncodePlaylistTrackID(plKey, loserKey))
changes.playlistUpsert(syncpkg.EncodePlaylistTrackID(plKey, survivorKey))
}
users, err := tq.MergeCopyGeneralLikes(ctx, dbq.MergeCopyGeneralLikesParams{SurvivorID: ids.survivor, LoserID: ids.loser})
if err != nil {
return MergeResult{}, fmt.Errorf("move likes: %w", err)
}
for _, u := range users {
userKey := syncpkg.FormatUUID(u)
likers[userKey] = struct{}{}
changes.likeDelete(syncpkg.EncodeLikeID(userKey, loserKey))
changes.likeUpsert(syncpkg.EncodeLikeID(userKey, survivorKey))
}
if _, err := tq.MergeCopyTrackTags(ctx, dbq.MergeCopyTrackTagsParams{SurvivorID: ids.survivor, LoserID: ids.loser}); err != nil {
return MergeResult{}, fmt.Errorf("merge tags: %w", err)
}
if _, err := tq.MergeCopyTrackSimilarity(ctx, dbq.MergeCopyTrackSimilarityParams{SurvivorID: ids.survivor, LoserID: ids.loser}); err != nil {
return MergeResult{}, fmt.Errorf("merge similarity: %w", err)
}
if err := tq.MergeInheritTrackMbid(ctx, dbq.MergeInheritTrackMbidParams{SurvivorID: ids.survivor, LoserID: ids.loser}); err != nil {
return MergeResult{}, fmt.Errorf("inherit recording mbid: %w", err)
}
// Everything the loser carried now sits on the survivor, so the CASCADE
// this delete sets off has nothing left to destroy.
deleted, err := tq.DeleteTrack(ctx, l.ID)
if err != nil {
return MergeResult{}, fmt.Errorf("delete merged copy: %w", err)
}
tidied, err := tidyEmptiedAlbum(ctx, tq, deleted.AlbumID)
if err != nil {
return MergeResult{}, err
}
if tidied.AlbumID != nil {
res.DeletedAlbumIDs = append(res.DeletedAlbumIDs, *tidied.AlbumID)
}
if tidied.ArtistID != nil {
res.DeletedArtistIDs = append(res.DeletedArtistIDs, *tidied.ArtistID)
}
res.Removed = append(res.Removed, mergedCopyOf(l))
changes.trackDelete(loserKey)
}
res.Likes = len(likers)
marked, err := tq.MarkDuplicateGroupMerged(ctx, groupID)
if err != nil {
return MergeResult{}, fmt.Errorf("mark group merged: %w", err)
}
if marked != 1 {
return MergeResult{}, ErrDuplicateGroupNotPending
}
if err := changes.log(ctx, tx); err != nil {
return MergeResult{}, err
}
if err := tx.Commit(ctx); err != nil {
return MergeResult{}, fmt.Errorf("commit merge: %w", err)
}
// After commit, like DeleteTrackFile: a leftover art directory is only disk.
if dataDir != "" {
for _, artistID := range res.DeletedArtistIDs {
if err := coverart.CleanupArtistArt(dataDir, artistID); err != nil {
logger.Warn("duplicate merge: artist-art cleanup failed",
"artist_id", syncpkg.FormatUUID(artistID), "err", err)
}
}
}
return res, nil
}
// splitSurvivor separates the copy to keep from the copies to remove. An
// invalid survivorID takes ProposeSurvivor's choice.
func splitSurvivor(
members []dbq.ListDuplicateGroupMergeMembersRow, survivorID pgtype.UUID,
) (dbq.ListDuplicateGroupMergeMembersRow, []dbq.ListDuplicateGroupMergeMembersRow, error) {
want := ""
if survivorID.Valid {
want = syncpkg.FormatUUID(survivorID)
} else {
cands := make([]SurvivorCandidate, len(members))
for i, m := range members {
cands[i] = SurvivorCandidate{
TrackID: syncpkg.FormatUUID(m.ID), FileFormat: m.FileFormat, FileSize: m.FileSize, AddedAt: m.AddedAt.Time,
}
}
want, _ = ProposeSurvivor(cands)
}
var survivor dbq.ListDuplicateGroupMergeMembersRow
found := false
var losers []dbq.ListDuplicateGroupMergeMembersRow
for _, m := range members {
if syncpkg.FormatUUID(m.ID) == want {
survivor, found = m, true
continue
}
losers = append(losers, m)
}
if !found {
return dbq.ListDuplicateGroupMergeMembersRow{}, nil, ErrSurvivorNotInGroup
}
return survivor, losers, nil
}
func mergedCopyOf(m dbq.ListDuplicateGroupMergeMembersRow) MergedCopy {
return MergedCopy{TrackID: m.ID, FilePath: m.FilePath, TrackMbid: m.Mbid, AlbumMbid: m.AlbumMbid}
}
// mergeChanges collects the sync-log entries a merge owes clients' caches, each
// once: a user who liked two removed copies still gets one upsert for the
// survivor.
type mergeChanges struct {
tracks, likeDeletes, likeUpserts, playlistDeletes, playlistUpserts map[string]struct{}
}
func addTo(set *map[string]struct{}, id string) {
if *set == nil {
*set = map[string]struct{}{}
}
(*set)[id] = struct{}{}
}
func (c *mergeChanges) trackDelete(id string) { addTo(&c.tracks, id) }
func (c *mergeChanges) likeDelete(id string) { addTo(&c.likeDeletes, id) }
func (c *mergeChanges) likeUpsert(id string) { addTo(&c.likeUpserts, id) }
func (c *mergeChanges) playlistDelete(id string) { addTo(&c.playlistDeletes, id) }
func (c *mergeChanges) playlistUpsert(id string) { addTo(&c.playlistUpserts, id) }
func (c *mergeChanges) log(ctx context.Context, tx pgx.Tx) error {
for _, entry := range []struct {
kind syncpkg.EntityType
ids map[string]struct{}
op syncpkg.Op
}{
{syncpkg.EntityTrack, c.tracks, syncpkg.OpDelete},
{syncpkg.EntityLikeTrack, c.likeDeletes, syncpkg.OpDelete},
{syncpkg.EntityLikeTrack, c.likeUpserts, syncpkg.OpUpsert},
{syncpkg.EntityPlaylistTrack, c.playlistDeletes, syncpkg.OpDelete},
{syncpkg.EntityPlaylistTrack, c.playlistUpserts, syncpkg.OpUpsert},
} {
if len(entry.ids) == 0 {
continue
}
ids := make([]string, 0, len(entry.ids))
for id := range entry.ids {
ids = append(ids, id)
}
sort.Strings(ids)
if err := syncpkg.LogChanges(ctx, tx, entry.kind, ids, entry.op); err != nil {
return fmt.Errorf("log merge changes: %w", err)
}
}
return nil
}
-281
View File
@@ -1,281 +0,0 @@
package library
import (
"context"
"errors"
"io/fs"
"os"
"path/filepath"
"syscall"
"testing"
"time"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
)
// mergeFixture is a library with one duplicate pair carrying history on both
// copies, and a neighbour track for similarity edges.
type mergeFixture struct {
pool *pgxpool.Pool
keep, remove, other dbq.Track
keepPath, removePath string
groupID pgtype.UUID
alice, bob dbq.User
playlistID pgtype.UUID
removePlaylistPos int32
aliceEarlierLikeOnRem time.Time
}
func newMergeFixture(t *testing.T) mergeFixture {
t.Helper()
pool := newPool(t)
ctx := context.Background()
q := dbq.New(pool)
dir := t.TempDir()
f := mergeFixture{pool: pool}
f.keepPath = filepath.Join(dir, "keep.flac")
f.removePath = filepath.Join(dir, "remove.mp3")
for _, p := range []string{f.keepPath, f.removePath} {
if err := os.WriteFile(p, []byte("audio"), 0o644); err != nil {
t.Fatalf("write %s: %v", p, err)
}
}
var album dbq.Album
var artist dbq.Artist
f.keep, album, artist = seedTrack(t, pool, f.keepPath)
upsert := func(title, path string) dbq.Track {
t.Helper()
tr, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
Title: title, AlbumID: album.ID, ArtistID: artist.ID,
DurationMs: 215000, FilePath: path, FileSize: 100, FileFormat: "mp3",
})
if err != nil {
t.Fatalf("track %s: %v", title, err)
}
return tr
}
f.remove = upsert("WWW (copy)", f.removePath)
f.other = upsert("Neighbour", filepath.Join(dir, "other.mp3"))
mustExec := func(sql string, args ...any) {
t.Helper()
if _, err := pool.Exec(ctx, sql, args...); err != nil {
t.Fatalf("exec %q: %v", sql, err)
}
}
// Only the copy being removed carries a recording MBID.
mustExec(`UPDATE tracks SET mbid = 'rec-www' WHERE id = $1`, f.remove.ID)
user := func(name string) dbq.User {
t.Helper()
u, err := q.CreateUser(ctx, dbq.CreateUserParams{
Username: dbtest.TestUserPrefix + name, PasswordHash: "x", ApiTokenHash: name + "-merge-token",
})
if err != nil {
t.Fatalf("user %s: %v", name, err)
}
return u
}
f.alice, f.bob = user("merge-alice"), user("merge-bob")
// Alice liked both copies, the removed one first; Bob liked only the removed one.
f.aliceEarlierLikeOnRem = time.Now().Add(-72 * time.Hour).UTC().Truncate(time.Microsecond)
mustExec(`INSERT INTO general_likes (user_id, track_id, liked_at) VALUES ($1, $2, $3), ($1, $4, now()), ($5, $2, now())`,
f.alice.ID, f.remove.ID, f.aliceEarlierLikeOnRem, f.keep.ID, f.bob.ID)
now := pgtype.Timestamptz{Time: time.Now(), Valid: true}
session, err := q.InsertPlaySession(ctx, dbq.InsertPlaySessionParams{UserID: f.alice.ID, StartedAt: now})
if err != nil {
t.Fatalf("session: %v", err)
}
for _, track := range []dbq.Track{f.remove, f.remove, f.keep} {
if _, err := q.InsertPlayEvent(ctx, dbq.InsertPlayEventParams{
UserID: f.alice.ID, TrackID: track.ID, SessionID: session.ID, StartedAt: now,
}); err != nil {
t.Fatalf("play event: %v", err)
}
}
if _, err := q.InsertSkipEvent(ctx, dbq.InsertSkipEventParams{
UserID: f.alice.ID, TrackID: f.remove.ID, SessionID: session.ID, SkippedAt: now, PositionMs: 1000,
}); err != nil {
t.Fatalf("skip event: %v", err)
}
pl, err := q.CreatePlaylist(ctx, dbq.CreatePlaylistParams{UserID: f.alice.ID, Name: "merge-mix"})
if err != nil {
t.Fatalf("playlist: %v", err)
}
f.playlistID = pl.ID
entry, err := q.AppendPlaylistTrack(ctx, dbq.AppendPlaylistTrackParams{PlaylistID: pl.ID, TrackID: f.remove.ID})
if err != nil {
t.Fatalf("playlist entry: %v", err)
}
f.removePlaylistPos = entry.Position
mustExec(`INSERT INTO track_tags (track_id, tag, weight) VALUES ($1, 'j-pop', 1), ($1, 'house', 0.5), ($2, 'house', 0.9)`,
f.remove.ID, f.keep.ID)
mustExec(`INSERT INTO track_similarity (track_a_id, track_b_id, score, source) VALUES
($1, $3, 0.8, 'listenbrainz'),
($2, $3, 0.7, 'listenbrainz'),
($1, $2, 0.99, 'listenbrainz'),
($3, $1, 0.6, 'musicbrainz_tag')`, f.remove.ID, f.keep.ID, f.other.ID)
if err := pool.QueryRow(ctx,
`INSERT INTO duplicate_groups (member_key, tier) VALUES ('merge-fixture', 'exact') RETURNING id`,
).Scan(&f.groupID); err != nil {
t.Fatalf("group: %v", err)
}
mustExec(`INSERT INTO duplicate_group_members (group_id, track_id) VALUES ($1, $2), ($1, $3)`,
f.groupID, f.keep.ID, f.remove.ID)
return f
}
func (f mergeFixture) count(t *testing.T, sql string, args ...any) int {
t.Helper()
var n int
if err := f.pool.QueryRow(context.Background(), sql, args...).Scan(&n); err != nil {
t.Fatalf("count %q: %v", sql, err)
}
return n
}
// The #3911 proof: after a merge, every piece of history the removed copy held
// is on the copy kept, nothing is doubled, and the removed copy — row and file —
// is gone.
func TestMergeDuplicateGroup_Integration(t *testing.T) {
f := newMergeFixture(t)
ctx := context.Background()
res, err := MergeDuplicateGroup(ctx, f.pool, nil, "", f.groupID, f.keep.ID)
if err != nil {
t.Fatalf("merge: %v", err)
}
if len(res.Removed) != 1 || res.Removed[0].FilePath != f.removePath || res.Survivor.TrackID != f.keep.ID {
t.Fatalf("result = %+v, want the removed copy reported and the kept one as survivor", res)
}
if res.PlayEvents != 2 || res.SkipEvents != 1 || res.Likes != 2 || res.PlaylistEntries != 1 {
t.Errorf("moved = plays %d skips %d likes %d playlist %d, want 2, 1, 2, 1",
res.PlayEvents, res.SkipEvents, res.Likes, res.PlaylistEntries)
}
if _, err := os.Stat(f.removePath); !errors.Is(err, os.ErrNotExist) {
t.Errorf("removed copy's file still on disk: %v", err)
}
if _, err := os.Stat(f.keepPath); err != nil {
t.Errorf("kept copy's file is gone: %v", err)
}
if n := f.count(t, `SELECT count(*) FROM tracks WHERE id = $1`, f.remove.ID); n != 0 {
t.Errorf("removed copy's row still exists")
}
// Likes: one per user, Alice's dated to her earlier like.
if n := f.count(t, `SELECT count(*) FROM general_likes WHERE track_id = $1`, f.keep.ID); n != 2 {
t.Errorf("likes on the kept copy = %d, want 2 (Alice once, Bob)", n)
}
var aliceLiked time.Time
if err := f.pool.QueryRow(ctx, `SELECT liked_at FROM general_likes WHERE user_id = $1 AND track_id = $2`,
f.alice.ID, f.keep.ID).Scan(&aliceLiked); err != nil {
t.Fatalf("alice's like: %v", err)
}
if !aliceLiked.Equal(f.aliceEarlierLikeOnRem) {
t.Errorf("alice's like dated %v, want her earlier like %v", aliceLiked, f.aliceEarlierLikeOnRem)
}
// Plays and skips move exactly: none lost, none invented.
if n := f.count(t, `SELECT count(*) FROM play_events WHERE track_id = $1`, f.keep.ID); n != 3 {
t.Errorf("plays on the kept copy = %d, want 3", n)
}
if n := f.count(t, `SELECT count(*) FROM skip_events WHERE track_id = $1`, f.keep.ID); n != 1 {
t.Errorf("skips on the kept copy = %d, want 1", n)
}
// The playlist entry stays where it was and now plays the kept copy.
if n := f.count(t, `SELECT count(*) FROM playlist_tracks WHERE playlist_id = $1 AND position = $2 AND track_id = $3`,
f.playlistID, f.removePlaylistPos, f.keep.ID); n != 1 {
t.Errorf("playlist entry at position %d does not point at the kept copy", f.removePlaylistPos)
}
// Tags are a union; the kept copy's own weight wins where both had the tag.
if n := f.count(t, `SELECT count(*) FROM track_tags WHERE track_id = $1`, f.keep.ID); n != 2 {
t.Errorf("tags on the kept copy = %d, want 2 (house, j-pop)", n)
}
if n := f.count(t, `SELECT count(*) FROM track_tags WHERE track_id = $1 AND tag = 'house' AND weight = 0.9`, f.keep.ID); n != 1 {
t.Errorf("the kept copy's own house weight was overwritten")
}
// Similarity: rewritten onto the kept copy, no duplicate edge, no self-edge,
// nothing left pointing at the removed copy.
if n := f.count(t, `SELECT count(*) FROM track_similarity WHERE track_a_id = $1 AND track_b_id = $2 AND source = 'listenbrainz'`,
f.keep.ID, f.other.ID); n != 1 {
t.Errorf("listenbrainz edge keep→other = %d rows, want exactly 1", n)
}
if n := f.count(t, `SELECT count(*) FROM track_similarity WHERE track_a_id = $1 AND track_b_id = $2 AND source = 'musicbrainz_tag'`,
f.other.ID, f.keep.ID); n != 1 {
t.Errorf("musicbrainz_tag edge other→keep was not carried over")
}
if n := f.count(t, `SELECT count(*) FROM track_similarity WHERE track_a_id = track_b_id`); n != 0 {
t.Errorf("a self-edge was written")
}
// The removed copy's recording MBID is inherited; the group is closed.
if n := f.count(t, `SELECT count(*) FROM tracks WHERE id = $1 AND mbid = 'rec-www'`, f.keep.ID); n != 1 {
t.Errorf("the kept copy did not inherit the recording MBID")
}
if n := f.count(t, `SELECT count(*) FROM duplicate_groups WHERE id = $1 AND status = 'merged' AND resolved_at IS NOT NULL`, f.groupID); n != 1 {
t.Errorf("group was not marked merged")
}
// A second merge of the same group is refused rather than repeated.
if _, err := MergeDuplicateGroup(ctx, f.pool, nil, "", f.groupID, f.keep.ID); !errors.Is(err, ErrDuplicateGroupNotPending) {
t.Errorf("second merge err = %v, want ErrDuplicateGroupNotPending", err)
}
}
// When the removed copy's file cannot go, nothing may change: its likes, plays
// and row stay exactly where they were, and the group stays pending.
func TestMergeDuplicateGroup_UnremovableFileChangesNothing(t *testing.T) {
f := newMergeFixture(t)
stubRemoveFile(t, func(name string) error {
return &fs.PathError{Op: "remove", Path: name, Err: syscall.EROFS}
})
_, err := MergeDuplicateGroup(context.Background(), f.pool, nil, "", f.groupID, f.keep.ID)
var fre *FileRemoveError
if !errors.As(err, &fre) || !fre.NotWritable() {
t.Fatalf("err = %v, want a not-writable *FileRemoveError", err)
}
if n := f.count(t, `SELECT count(*) FROM tracks WHERE id = $1`, f.remove.ID); n != 1 {
t.Errorf("the copy's row was deleted although its file was not")
}
if n := f.count(t, `SELECT count(*) FROM general_likes WHERE track_id = $1`, f.remove.ID); n != 2 {
t.Errorf("likes on the copy = %d, want both still there", n)
}
if n := f.count(t, `SELECT count(*) FROM play_events WHERE track_id = $1`, f.remove.ID); n != 2 {
t.Errorf("plays on the copy = %d, want both still there", n)
}
if n := f.count(t, `SELECT count(*) FROM duplicate_groups WHERE id = $1 AND status = 'pending'`, f.groupID); n != 1 {
t.Errorf("group left pending = false, want it still pending")
}
}
func TestMergeDuplicateGroup_SurvivorMustBeAMember(t *testing.T) {
f := newMergeFixture(t)
var stranger pgtype.UUID
stranger.Bytes[15], stranger.Valid = 0xEE, true
_, err := MergeDuplicateGroup(context.Background(), f.pool, nil, "", f.groupID, stranger)
if !errors.Is(err, ErrSurvivorNotInGroup) {
t.Fatalf("err = %v, want ErrSurvivorNotInGroup", err)
}
if _, err := os.Stat(f.removePath); err != nil {
t.Errorf("a refused merge removed a file: %v", err)
}
if n := f.count(t, `SELECT count(*) FROM duplicate_groups WHERE id = $1 AND status = 'pending'`, f.groupID); n != 1 {
t.Errorf("a refused merge changed the group")
}
}
-77
View File
@@ -1,77 +0,0 @@
package library
import (
"sort"
"strings"
"time"
)
// SurvivorCandidate is what choosing which copy to keep needs to know about one
// member of a duplicate group.
type SurvivorCandidate struct {
TrackID string
FileFormat string
FileSize int64
AddedAt time.Time
}
// losslessFormats are the scanned extensions that are lossless by definition.
// m4a is left out on purpose: it holds either ALAC or AAC, and the scanner
// records only the extension, so calling it lossless would sometimes prefer an
// AAC copy over a FLAC one.
var losslessFormats = map[string]bool{"flac": true, "wav": true}
// ProposeSurvivor picks which copy of a duplicate group to keep, and gives the
// reason in words the operator reads beside it. It is a default, not a verdict:
// the report shows it and the merge (#3911) lets the operator choose another.
//
// In order:
// 1. lossless over lossy — the one difference no later step can recover
// 2. the larger file — for one recording at one duration that is the higher
// bitrate. The scanner does not record bitrate (tracks.bitrate is never
// filled), so file size is the signal that actually exists
// 3. the copy in the library longest — the one most likely to carry the play
// history and likes, so the merge moves the least
// 4. the lowest track id, so the choice is stable between page loads
func ProposeSurvivor(cands []SurvivorCandidate) (trackID, reason string) {
if len(cands) == 0 {
return "", ""
}
ranked := append([]SurvivorCandidate(nil), cands...)
sort.SliceStable(ranked, func(i, j int) bool { return survivorBefore(ranked[i], ranked[j]) })
best := ranked[0]
if len(ranked) == 1 {
return best.TrackID, "the only copy"
}
// The reason names the first rule that separated the best copy from the
// runner-up — the rule that actually decided, not every rule it passed.
next := ranked[1]
switch {
case isLossless(best) != isLossless(next):
return best.TrackID, "lossless (" + strings.ToLower(best.FileFormat) + ")"
case best.FileSize != next.FileSize:
return best.TrackID, "largest file"
case !best.AddedAt.Equal(next.AddedAt):
return best.TrackID, "in the library longest"
default:
return best.TrackID, "copies are otherwise identical"
}
}
func survivorBefore(a, b SurvivorCandidate) bool {
if isLossless(a) != isLossless(b) {
return isLossless(a)
}
if a.FileSize != b.FileSize {
return a.FileSize > b.FileSize
}
if !a.AddedAt.Equal(b.AddedAt) {
return a.AddedAt.Before(b.AddedAt)
}
return a.TrackID < b.TrackID
}
func isLossless(c SurvivorCandidate) bool {
return losslessFormats[strings.ToLower(c.FileFormat)]
}
@@ -1,91 +0,0 @@
package library
import (
"testing"
"time"
)
func TestProposeSurvivor(t *testing.T) {
older := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
newer := older.Add(24 * time.Hour)
cases := []struct {
name string
cands []SurvivorCandidate
wantID string
wantReason string
}{
{
// Lossless wins even against a much larger lossy file, and even
// when the lossy copy has been in the library longer.
name: "lossless beats larger and older",
cands: []SurvivorCandidate{
{TrackID: "mp3", FileFormat: "mp3", FileSize: 90_000_000, AddedAt: older},
{TrackID: "flac", FileFormat: "FLAC", FileSize: 30_000_000, AddedAt: newer},
},
wantID: "flac", wantReason: "lossless (flac)",
},
{
// m4a may be AAC; it must not outrank an mp3 just for being m4a.
name: "m4a is not treated as lossless",
cands: []SurvivorCandidate{
{TrackID: "m4a", FileFormat: "m4a", FileSize: 5_000_000, AddedAt: older},
{TrackID: "mp3", FileFormat: "mp3", FileSize: 9_000_000, AddedAt: newer},
},
wantID: "mp3", wantReason: "largest file",
},
{
name: "larger file wins among lossy copies",
cands: []SurvivorCandidate{
{TrackID: "128k", FileFormat: "mp3", FileSize: 3_400_000, AddedAt: older},
{TrackID: "320k", FileFormat: "mp3", FileSize: 8_600_000, AddedAt: newer},
},
wantID: "320k", wantReason: "largest file",
},
{
// The #3885 pair: identical audio, sizes equal but for the tags.
name: "the longest-standing copy wins when size ties",
cands: []SurvivorCandidate{
{TrackID: "www-02", FileFormat: "mp3", FileSize: 6_900_000, AddedAt: newer},
{TrackID: "www-01", FileFormat: "mp3", FileSize: 6_900_000, AddedAt: older},
},
wantID: "www-01", wantReason: "in the library longest",
},
{
name: "a full tie falls back to the lowest id, stably",
cands: []SurvivorCandidate{
{TrackID: "b", FileFormat: "mp3", FileSize: 1, AddedAt: older},
{TrackID: "a", FileFormat: "mp3", FileSize: 1, AddedAt: older},
},
wantID: "a", wantReason: "copies are otherwise identical",
},
{
name: "one copy",
cands: []SurvivorCandidate{{TrackID: "only", FileFormat: "mp3", FileSize: 1, AddedAt: older}},
wantID: "only", wantReason: "the only copy",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
id, reason := ProposeSurvivor(tc.cands)
if id != tc.wantID || reason != tc.wantReason {
t.Fatalf("ProposeSurvivor = (%q, %q), want (%q, %q)", id, reason, tc.wantID, tc.wantReason)
}
})
}
}
// The reason must name the rule that decided. Across three copies that is the
// comparison between first and second place, not the first rule any pair
// differs on: here the lossy copy differs from the others by format, but the
// two FLACs are separated by size.
func TestProposeSurvivor_ReasonIsTheDecidingRule(t *testing.T) {
at := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
id, reason := ProposeSurvivor([]SurvivorCandidate{
{TrackID: "mp3", FileFormat: "mp3", FileSize: 99_000_000, AddedAt: at},
{TrackID: "flac-small", FileFormat: "flac", FileSize: 20_000_000, AddedAt: at},
{TrackID: "flac-big", FileFormat: "flac", FileSize: 40_000_000, AddedAt: at},
})
if id != "flac-big" || reason != "largest file" {
t.Fatalf("got (%q, %q), want (flac-big, largest file)", id, reason)
}
}
-398
View File
@@ -1,398 +0,0 @@
package library
import (
"context"
"errors"
"fmt"
"log/slog"
"sort"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
)
// Duplicate sweep (M400 #3910).
//
// Reads fingerprints, runs them through the matcher and records what it proposes
// in duplicate_groups. It never merges or deletes anything the operator has not
// asked for: a group is a proposal, reviewed in the admin report (#3912).
//
// Blocked on duration, deliberately not on title: the #3885 pair are titled
// "WWW" and "WWW (instrumental)", so a title block would have missed the case
// that started the milestone. Candidates stream in (duration_ms, id) order and
// the grouper holds only a few seconds of durations at a time.
// duplicateCandidatePage is how many candidates one query returns. Each row
// carries a ~4 KB fingerprint, so a page is about 2 MB.
const duplicateCandidatePage = 500
// duplicateSweepTick is how often the worker checks whether a sweep is due. The
// operator's sweep interval (#3913) is the least time between sweeps; the tick
// only bounds how late past it one starts. With nothing due, a tick is two cheap
// aggregate queries.
const duplicateSweepTick = time.Hour
// sweepIntervalSlack absorbs the moment between a tick and the sweep it starts
// stamping started_at. Without it a one-hour interval checked on a one-hour tick
// would find the last sweep a moment under an hour old, and skip every other tick.
const sweepIntervalSlack = 5 * time.Minute
// staleDuplicateSweepThreshold is the age past which an in-flight sweep is
// assumed dead — a crash mid-sweep leaves finished_at NULL for ever — and another
// may start. Twice the library scan's threshold, because a sweep compares
// fingerprints across the whole library and can legitimately run long on a big
// one.
const staleDuplicateSweepThreshold = 2 * time.Hour
// duplicateSweepFinishTimeout bounds recording that a sweep ended. It runs on a
// context detached from the sweep's own, so a sweep cancelled at shutdown still
// closes its row rather than leaving it in flight until the reaper.
const duplicateSweepFinishTimeout = 10 * time.Second
// DuplicateSweepResult tallies one sweep.
type DuplicateSweepResult struct {
Candidates int // tracks with a chromaprint that were streamed
Groups int // groups the matcher found
Proposed int // written as pending, new or refreshed
Suppressed int // not proposed: already dismissed or resolved by the operator
Retired int // pending proposals this sweep did not find again, removed
Oversize int // acoustic clusters too large to propose
}
// RunDuplicateSweep runs one sweep and records it in duplicate_sweeps. cfg is a
// snapshot: one sweep applies one threshold and one length throughout.
func RunDuplicateSweep(
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, cfg FingerprintSettings,
) (DuplicateSweepResult, error) {
return runDuplicateSweep(ctx, pool, logger, cfg, duplicateCandidatePage)
}
func runDuplicateSweep(
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, cfg FingerprintSettings, pageSize int32,
) (DuplicateSweepResult, error) {
q := dbq.New(pool)
sweep, err := q.StartDuplicateSweep(ctx)
if err != nil {
return DuplicateSweepResult{}, fmt.Errorf("start duplicate sweep: %w", err)
}
res, runErr := sweepDuplicates(ctx, q, sweep.ID, cfg, pageSize)
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), duplicateSweepFinishTimeout)
defer cancel()
errMsg := ""
if runErr != nil {
errMsg = runErr.Error()
}
candidates, groups, oversize := int32(res.Candidates), int32(res.Groups), int32(res.Oversize)
if ferr := q.FinishDuplicateSweep(finishCtx, dbq.FinishDuplicateSweepParams{
ID: sweep.ID, Candidates: &candidates, GroupsFound: &groups, OversizeClusters: &oversize,
ErrorMessage: errMsg,
}); ferr != nil {
logger.Error("duplicate sweep: recording the end of the sweep failed", "err", ferr)
if runErr == nil {
runErr = fmt.Errorf("finish duplicate sweep: %w", ferr)
}
}
logger.Info("duplicate sweep complete",
"candidates", res.Candidates, "groups", res.Groups, "proposed", res.Proposed,
"suppressed", res.Suppressed, "retired", res.Retired, "oversize", res.Oversize, "err", runErr)
return res, runErr
}
func sweepDuplicates(
ctx context.Context, q *dbq.Queries, sweepID pgtype.UUID, cfg FingerprintSettings, pageSize int32,
) (DuplicateSweepResult, error) {
var res DuplicateSweepResult
// Exact tier, library-wide, in one query.
exactRows, err := q.ListExactDuplicateHashes(ctx, fingerprintVersion)
if err != nil {
return res, fmt.Errorf("list exact duplicates: %w", err)
}
exactMembers := make([][]string, len(exactRows))
exactOf := map[string]int{}
for i, row := range exactRows {
exactMembers[i] = formatUUIDs(row.TrackIds)
for _, id := range exactMembers[i] {
exactOf[id] = i
}
}
exactSeen := make([]bool, len(exactRows))
// Acoustic tier, streamed in duration order. The first member of an exact
// group the stream meets stands in for the whole group; the rest are skipped.
// Only prints at the current length are streamed: a print at another length
// cannot be compared, and is waiting on the backfill to be re-derived.
grouper := newStreamGrouper(cfg.AcousticMaxBitErrorRate)
params := dbq.ListDuplicateCandidatesParams{
CurrentVersion: fingerprintVersion,
ChromaprintLengthSec: cfg.ChromaprintLengthSec,
// Durations are never negative, and the all-zero uuid sorts first: every
// row is after this cursor. Valid must be true, or "> NULL" matches nothing.
AfterDurationMs: -1,
AfterID: pgtype.UUID{Valid: true},
PageLimit: pageSize,
}
for {
if err := ctx.Err(); err != nil {
return res, err
}
rows, err := q.ListDuplicateCandidates(ctx, params)
if err != nil {
return res, fmt.Errorf("list duplicate candidates: %w", err)
}
for _, row := range rows {
res.Candidates++
id := syncpkg.FormatUUID(row.ID)
unit := &groupUnit{ids: []string{id}, durationMs: row.DurationMs, sortKey: id}
if gi, ok := exactOf[id]; ok {
if exactSeen[gi] {
continue
}
exactSeen[gi] = true
unit.ids, unit.exact = exactMembers[gi], true
}
unit.print = preparePrint(row.Chromaprint)
grouper.push(unit)
}
if int32(len(rows)) < pageSize {
break
}
last := rows[len(rows)-1]
params.AfterDurationMs, params.AfterID = last.DurationMs, last.ID
}
found := grouper.finish()
// Exact groups none of whose members has a chromaprint never reached the
// stream. Identical bytes need no acoustic evidence.
for gi, seen := range exactSeen {
if !seen {
found.Groups = append(found.Groups, duplicateGroup{Tier: tierExact, Members: exactMembers[gi]})
}
}
sortGroups(found.Groups)
res.Groups, res.Oversize = len(found.Groups), found.OversizeClusters
dismissed, err := q.ListDismissedDuplicateMemberSets(ctx)
if err != nil {
return res, fmt.Errorf("list dismissed duplicate groups: %w", err)
}
dismissedSets := make([]map[string]struct{}, 0, len(dismissed))
for _, d := range dismissed {
set := map[string]struct{}{}
for _, id := range formatUUIDs(d.TrackIds) {
set[id] = struct{}{}
}
dismissedSets = append(dismissedSets, set)
}
for _, group := range found.Groups {
if coveredByDismissal(group.Members, dismissedSets) {
res.Suppressed++
continue
}
up := dbq.UpsertDuplicateGroupParams{
MemberKey: strings.Join(group.Members, ","),
Tier: string(group.Tier),
SweepID: sweepID,
}
if group.Tier == tierAcoustic {
worst := float32(group.WorstBitErrorRate)
up.WorstBitErrorRate = &worst
}
groupID, err := q.UpsertDuplicateGroup(ctx, up)
if errors.Is(err, pgx.ErrNoRows) {
// This exact member set was already dismissed or merged.
res.Suppressed++
continue
}
if err != nil {
return res, fmt.Errorf("upsert duplicate group: %w", err)
}
for _, id := range group.Members {
var trackID pgtype.UUID
if err := trackID.Scan(id); err != nil {
return res, fmt.Errorf("parse track id %q: %w", id, err)
}
if err := q.AddDuplicateGroupMember(ctx, dbq.AddDuplicateGroupMemberParams{
GroupID: groupID, TrackID: trackID,
}); err != nil {
return res, fmt.Errorf("add duplicate group member: %w", err)
}
}
res.Proposed++
}
// Only after a complete sweep: a sweep that failed partway has no basis for
// concluding that anything it did not reach has gone.
retired, err := q.DeleteStalePendingDuplicateGroups(ctx, sweepID)
if err != nil {
return res, fmt.Errorf("retire stale duplicate groups: %w", err)
}
res.Retired = int(retired)
return res, nil
}
// coveredByDismissal reports whether every member of a proposal sat together in
// one group the operator dismissed. A subset counts: dismissing {A, B, C} said
// none of them are copies of each other, so proposing {A, B} again would be
// asking the same question twice. A superset does not count: a new copy joining
// is new evidence, and worth asking about.
func coveredByDismissal(members []string, dismissed []map[string]struct{}) bool {
for _, set := range dismissed {
covered := true
for _, id := range members {
if _, ok := set[id]; !ok {
covered = false
break
}
}
if covered {
return true
}
}
return false
}
func formatUUIDs(ids []pgtype.UUID) []string {
out := make([]string, len(ids))
for i, id := range ids {
out[i] = syncpkg.FormatUUID(id)
}
sort.Strings(out)
return out
}
// TryStartDuplicateSweep starts a sweep in the background unless one is already
// running, reaping a sweep that has been in flight past
// staleDuplicateSweepThreshold. Mirrors TryStartScan. The sweep runs on ctx, so
// a caller answering an HTTP request must pass a context that outlives it.
func TryStartDuplicateSweep(
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, cfg FingerprintSettings,
) (bool, error) {
q := dbq.New(pool)
row, err := q.GetInFlightDuplicateSweep(ctx)
switch {
case err == nil:
age := time.Since(row.StartedAt.Time)
if age <= staleDuplicateSweepThreshold {
return false, nil
}
logger.Warn("reaping stale duplicate sweep", "id", syncpkg.FormatUUID(row.ID), "age", age)
if ferr := q.FinishDuplicateSweep(ctx, dbq.FinishDuplicateSweepParams{
ID: row.ID, ErrorMessage: "reaped (stale)",
}); ferr != nil {
return false, fmt.Errorf("reap stale duplicate sweep: %w", ferr)
}
case !errors.Is(err, pgx.ErrNoRows):
return false, fmt.Errorf("duplicate sweep in-flight check: %w", err)
}
go func() {
if _, err := RunDuplicateSweep(ctx, pool, logger, cfg); err != nil {
logger.Warn("duplicate sweep failed", "err", err)
}
}()
return true, nil
}
// DuplicateSweepWorker sweeps whenever its input has changed, at most once per
// the operator's sweep interval.
type DuplicateSweepWorker struct {
pool *pgxpool.Pool
logger *slog.Logger
settings *FingerprintSettingsService
tick time.Duration
}
// NewDuplicateSweepWorker builds a worker with the production cadence. settings
// is shared with the admin API; nil runs on defaults.
func NewDuplicateSweepWorker(
pool *pgxpool.Pool, logger *slog.Logger, settings *FingerprintSettingsService,
) *DuplicateSweepWorker {
return &DuplicateSweepWorker{pool: pool, logger: logger, settings: settings, tick: duplicateSweepTick}
}
// Run blocks until ctx is cancelled, checking once at start and then each tick.
func (w *DuplicateSweepWorker) Run(ctx context.Context) {
w.tickOnce(ctx)
t := time.NewTicker(w.tick)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
w.tickOnce(ctx)
}
}
}
// tickOnce contains one check so nothing it does can stop the next tick (rule 157).
func (w *DuplicateSweepWorker) tickOnce(ctx context.Context) {
defer func() {
if r := recover(); r != nil {
w.logger.Error("duplicate sweep: tick panicked", "panic", r)
}
}()
cfg := w.settings.Get()
due, err := duplicateSweepDue(ctx, dbq.New(w.pool), cfg, time.Now())
if err != nil {
if ctx.Err() == nil {
w.logger.Warn("duplicate sweep: due check failed", "err", err)
}
return
}
if !due {
return
}
if _, err := TryStartDuplicateSweep(ctx, w.pool, w.logger, cfg); err != nil {
w.logger.Warn("duplicate sweep: start failed", "err", err)
}
}
// duplicateSweepDue reads what sweepIsDue decides on.
func duplicateSweepDue(ctx context.Context, q *dbq.Queries, cfg FingerprintSettings, now time.Time) (bool, error) {
latest, err := q.GetLatestFingerprintComputedAt(ctx)
if err != nil {
return false, fmt.Errorf("latest fingerprint: %w", err)
}
var lastStart pgtype.Timestamptz
last, err := q.GetLatestDuplicateSweep(ctx)
switch {
case err == nil:
lastStart = last.StartedAt
case !errors.Is(err, pgx.ErrNoRows):
return false, fmt.Errorf("latest duplicate sweep: %w", err)
}
return sweepIsDue(latest, lastStart, cfg, now), nil
}
// sweepIsDue reports whether a sweep should start: something it reads has
// changed since the last sweep started, and the operator's interval has passed.
//
// Two things can change its answer. Fingerprints are its input, so any written
// after the last sweep began count; while the backfill runs that is true every
// tick, which is what the interval is for. And a settings save counts, because a
// new threshold or length changes what the same fingerprints group into.
func sweepIsDue(latestFingerprint, lastSweepStart pgtype.Timestamptz, cfg FingerprintSettings, now time.Time) bool {
if !latestFingerprint.Valid {
return false // nothing fingerprinted yet
}
if !lastSweepStart.Valid {
return true // never swept
}
interval := time.Duration(cfg.SweepIntervalHours) * time.Hour
if now.Sub(lastSweepStart.Time) < interval-sweepIntervalSlack {
return false
}
return latestFingerprint.Time.After(lastSweepStart.Time) || cfg.UpdatedAt.After(lastSweepStart.Time)
}

Some files were not shown because too many files have changed in this diff Show More